pi-braid 0.40.6 → 0.41.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/build-info.json +4 -4
- package/docs/CONTEXT-MAP.md +1 -1
- package/docs/FLEET.md +134 -0
- package/package.json +5 -1
- package/scripts/fleet/cli.mjs +540 -0
- package/scripts/fleet/driver-rpc.mjs +376 -0
- package/scripts/fleet/driver-scripted.mjs +239 -0
- package/scripts/fleet/journal.mjs +1124 -0
- package/scripts/fleet/procs.mjs +76 -0
- package/scripts/fleet/supervisor.mjs +1649 -0
- package/scripts/fleet/tool-guard.mjs +467 -0
- package/skills/release-pipeline/SKILL.md +20 -4
|
@@ -0,0 +1,1649 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fleet supervisor: one process, no model, holding a cap of node sessions.
|
|
3
|
+
*
|
|
4
|
+
* Its whole job is to keep one statement true across a crash: no completed
|
|
5
|
+
* effect is repeated and no node is silently lost. Everything else here is
|
|
6
|
+
* scheduling taste.
|
|
7
|
+
*
|
|
8
|
+
* How that statement stays true:
|
|
9
|
+
*
|
|
10
|
+
* - A turn's prompt is journalled before the child is spawned
|
|
11
|
+
* (`turns.prompt_text`), and the child's process group is journalled with
|
|
12
|
+
* it. A restart therefore knows what was sent and what to fence.
|
|
13
|
+
* - Effects are journalled from the driver's own tool events: the attempt
|
|
14
|
+
* before the tool runs, the result after it. An attempt without a result
|
|
15
|
+
* is the crash residue, and reconciliation turns it into `unknown` rather
|
|
16
|
+
* than into a repeat.
|
|
17
|
+
* - A node resumes by reopening its recorded session id. Its recovery turn
|
|
18
|
+
* carries the journal's account of the interrupted turn: which effects
|
|
19
|
+
* completed, which are unknown. The supervisor never re-sends a prompt it
|
|
20
|
+
* already recorded as sent - that single rule is the difference between
|
|
21
|
+
* resuming and restarting.
|
|
22
|
+
* - The supervisor's epoch is journalled. A restart fences the process
|
|
23
|
+
* groups of every turn its predecessor left open, so a stale drive cannot
|
|
24
|
+
* keep writing while the new one resumes the same session.
|
|
25
|
+
*
|
|
26
|
+
* The naive replay path (`naive: true`) exists to be wrong on purpose: it is
|
|
27
|
+
* the arm the crash drill measures duplication with.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { execFileSync } from "node:child_process";
|
|
31
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
32
|
+
import { chmodSync, existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
33
|
+
import { createServer, connect } from "node:net";
|
|
34
|
+
import { tmpdir } from "node:os";
|
|
35
|
+
import { join } from "node:path";
|
|
36
|
+
import { pathToFileURL } from "node:url";
|
|
37
|
+
|
|
38
|
+
import { BUSY_TIMEOUT_MS, openJournal, replayPolicy, resolveFleetRoot, SCHEMA_VERSION } from "./journal.mjs";
|
|
39
|
+
import { createRpcDriver, GUARD_EXTENSION, guardEnvironment } from "./driver-rpc.mjs";
|
|
40
|
+
import { createScriptedDriver, ASK_SENTINEL, DONE_SENTINEL } from "./driver-scripted.mjs";
|
|
41
|
+
import { groupIsAlive, pidIsAlive, residentMb } from "./procs.mjs";
|
|
42
|
+
|
|
43
|
+
export { ASK_SENTINEL, DONE_SENTINEL };
|
|
44
|
+
|
|
45
|
+
export const NODE_STATES = ["settled", "waiting_on_child", "waiting_on_human", "failed", "done"];
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Where a maintenance fact goes when the store will not take the write: beside
|
|
49
|
+
* the journal, because a fact about a wedged store is the one fact the store
|
|
50
|
+
* cannot hold. `status` reads it, and the first pass that gets through copies it
|
|
51
|
+
* into the journal and removes it.
|
|
52
|
+
*/
|
|
53
|
+
export const DEFERRED_FILE = "store-deferred.json";
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Why no future turn can carry a message for this node, or null.
|
|
57
|
+
*
|
|
58
|
+
* `done` and `failed`-at-its-attempt-bound are the two states the scheduler
|
|
59
|
+
* skips for good, so a message queued for one of them is not early, it is
|
|
60
|
+
* undeliverable. Every other state can still be driven - an open ask gets an
|
|
61
|
+
* answer, a turn in flight ends - so a message queued there is only waiting.
|
|
62
|
+
*
|
|
63
|
+
* `maxAttempts` is the caller's policy, and a caller that does not have one
|
|
64
|
+
* passes null and learns only the `done` half: the command line cannot see the
|
|
65
|
+
* bound a supervisor is running with.
|
|
66
|
+
*/
|
|
67
|
+
export function undeliverableReason(node, maxAttempts = null) {
|
|
68
|
+
if (node.state === "done") return "the node is done";
|
|
69
|
+
if (maxAttempts != null && node.state === "failed" && node.attempts >= maxAttempts) {
|
|
70
|
+
return `the node failed after ${node.attempts} attempts`;
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export const DEFAULT_ANSWER = "No decision arrived before the deadline. Continue with the charter's stated default.";
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Is this error SQLite saying "another process holds the write lock"?
|
|
79
|
+
*
|
|
80
|
+
* The primary result codes are what decide: 5 is SQLITE_BUSY, 6 is
|
|
81
|
+
* SQLITE_LOCKED, and 517 is SQLITE_BUSY_SNAPSHOT, which node:sqlite can report
|
|
82
|
+
* as an extended code. The message is the fallback for a driver that hands over
|
|
83
|
+
* a bare Error, and it matches SQLite's own strings rather than the word
|
|
84
|
+
* "locked" on its own, so a schema complaint cannot be mistaken for contention.
|
|
85
|
+
*/
|
|
86
|
+
export function isBusyError(error) {
|
|
87
|
+
const code = Number(error?.errcode);
|
|
88
|
+
if (code === 5 || code === 6 || code === 517) return true;
|
|
89
|
+
return /database is locked|database table is locked|SQLITE_BUSY/i.test(String(error?.message ?? ""));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Block the current thread for `ms`. The drain is synchronous on purpose: its
|
|
94
|
+
* claim and its queue are one transaction, and a promise there would mean the
|
|
95
|
+
* transaction spans an await, which is the shape that lets a second writer in
|
|
96
|
+
* between the two halves. So the pause between two attempts blocks.
|
|
97
|
+
*/
|
|
98
|
+
function sleepSync(ms) {
|
|
99
|
+
if (ms <= 0) return;
|
|
100
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** A JSON meta value, or null when the key is absent or unreadable. */
|
|
104
|
+
function jsonMeta(meta, key) {
|
|
105
|
+
if (!meta[key]) return null;
|
|
106
|
+
try {
|
|
107
|
+
return JSON.parse(meta[key]);
|
|
108
|
+
} catch {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The deferral a wedged or dead supervisor left beside the store, or null.
|
|
115
|
+
*
|
|
116
|
+
* Exported because `status` reads it too: when the store cannot be opened at all,
|
|
117
|
+
* this is the one fact about why the fleet is idle that is still readable.
|
|
118
|
+
*/
|
|
119
|
+
export function deferralFromDisk(root) {
|
|
120
|
+
try {
|
|
121
|
+
return JSON.parse(readFileSync(join(root, DEFERRED_FILE), "utf8"));
|
|
122
|
+
} catch {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Where the control socket lives.
|
|
129
|
+
*
|
|
130
|
+
* A unix socket path is capped (104 bytes on macOS, 108 on Linux), and a fleet
|
|
131
|
+
* root can be deeper than that - a nested project, a scratch tree, a CI
|
|
132
|
+
* workspace. The socket therefore falls back to a short hashed name in the
|
|
133
|
+
* system temp directory, and both the supervisor and the commands derive the
|
|
134
|
+
* same path from the root, so the channel does not depend on where the store
|
|
135
|
+
* happens to sit. The chosen path is journalled as `control_socket`.
|
|
136
|
+
*/
|
|
137
|
+
export function controlSocketPath(root) {
|
|
138
|
+
const absolute = resolveFleetRoot(root);
|
|
139
|
+
const local = join(absolute, "control.sock");
|
|
140
|
+
if (Buffer.byteLength(local) <= 100) return local;
|
|
141
|
+
const hash = createHash("sha256").update(absolute).digest("hex").slice(0, 12);
|
|
142
|
+
return join(tmpdir(), `pi-fleet-${hash}.sock`);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* How long the arrival probe waits for an answer before it treats the path as a
|
|
147
|
+
* departed binder's leavings. A supervisor answers a ping in microseconds; the
|
|
148
|
+
* wait is long enough that a busy tick is not mistaken for a dead channel.
|
|
149
|
+
*/
|
|
150
|
+
const CONTROL_PROBE_MS = 1_000;
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The identity of whatever file sits at `path`: a unix socket's device and
|
|
154
|
+
* inode are stable for the life of the file, and a rebind at the same path
|
|
155
|
+
* creates a different inode. Used to answer "is the path still the one this
|
|
156
|
+
* process bound?" from the filesystem rather than from memory of a name.
|
|
157
|
+
*
|
|
158
|
+
* Returns null when nothing is there, and throws when the question cannot be
|
|
159
|
+
* answered at all: a caller that cannot tell must not delete.
|
|
160
|
+
*/
|
|
161
|
+
function fileIdentity(path) {
|
|
162
|
+
try {
|
|
163
|
+
const stat = statSync(path);
|
|
164
|
+
return { dev: stat.dev, ino: stat.ino };
|
|
165
|
+
} catch (error) {
|
|
166
|
+
if (error.code === "ENOENT") return null;
|
|
167
|
+
throw error;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* One request, one response line, over a unix socket path. Resolves `null` when
|
|
173
|
+
* the path does not answer within the timeout: absent, refused, or silent. The
|
|
174
|
+
* wire format lives with the server it speaks to, so the supervisor's own
|
|
175
|
+
* arrival probe and the commands share one implementation.
|
|
176
|
+
*/
|
|
177
|
+
export function requestOnControlSocket(path, request, { timeoutMs = 10_000 } = {}) {
|
|
178
|
+
return new Promise((resolveReply) => {
|
|
179
|
+
const socket = connect(path);
|
|
180
|
+
let buffer = "";
|
|
181
|
+
const finish = (reply) => {
|
|
182
|
+
socket.destroy();
|
|
183
|
+
resolveReply(reply);
|
|
184
|
+
};
|
|
185
|
+
socket.setTimeout(timeoutMs, () => finish(null));
|
|
186
|
+
socket.on("connect", () => socket.write(JSON.stringify(request) + "\n"));
|
|
187
|
+
socket.on("data", (chunk) => {
|
|
188
|
+
buffer += chunk.toString("utf8");
|
|
189
|
+
const index = buffer.indexOf("\n");
|
|
190
|
+
if (index < 0) return;
|
|
191
|
+
try {
|
|
192
|
+
finish(JSON.parse(buffer.slice(0, index)));
|
|
193
|
+
} catch {
|
|
194
|
+
finish(null);
|
|
195
|
+
}
|
|
196
|
+
});
|
|
197
|
+
socket.on("error", () => finish(null));
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
export const CHILD_FLAGS = [
|
|
202
|
+
"--mode",
|
|
203
|
+
"rpc",
|
|
204
|
+
"--no-mcp",
|
|
205
|
+
"--no-context-files",
|
|
206
|
+
"--no-extensions",
|
|
207
|
+
"--no-skills",
|
|
208
|
+
"--no-prompt-templates",
|
|
209
|
+
"--thinking",
|
|
210
|
+
"off",
|
|
211
|
+
];
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* The queue section of one prompt: the lines it appends for the messages the
|
|
215
|
+
* turn carries, and the ids those lines are.
|
|
216
|
+
*
|
|
217
|
+
* Both halves come out of one array here, and no other code builds the section
|
|
218
|
+
* by hand, so the answer to "what did this turn carry?" is the section that went
|
|
219
|
+
* into the prompt rather than a second reading of the finished text. That is the
|
|
220
|
+
* difference between a delivery and a coincidence: a charter written beside a
|
|
221
|
+
* queued message very often repeats the operator's instruction word for word,
|
|
222
|
+
* and a body found by search is not a message the model was given.
|
|
223
|
+
*/
|
|
224
|
+
export function queueSection(header, pending) {
|
|
225
|
+
const entries = pending.map((message) => ({ msgId: message.msg_id, line: `- ${message.body}` }));
|
|
226
|
+
return { header, lines: entries.map((entry) => entry.line), entries };
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Which of the section's messages the prompt really carries.
|
|
231
|
+
*
|
|
232
|
+
* The builder already says what it appended, and the text has to agree: the
|
|
233
|
+
* section's own sentence must be there, and so must the message's own line. A
|
|
234
|
+
* turn kind that appends no section carries nothing, and a prompt that repeats a
|
|
235
|
+
* body in another section - above all a charter - is not read as one.
|
|
236
|
+
*/
|
|
237
|
+
export function carriedBy(prompt, section) {
|
|
238
|
+
if (section.entries.length === 0) return [];
|
|
239
|
+
if (!prompt.includes(section.header)) return [];
|
|
240
|
+
return section.entries.filter((entry) => prompt.includes(entry.line)).map((entry) => entry.msgId);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** A prompt assembled around its queue section: the head, the section, the tail. */
|
|
244
|
+
function withSection(head, section, tail = []) {
|
|
245
|
+
const lines = [...head];
|
|
246
|
+
if (section.entries.length > 0) {
|
|
247
|
+
if (lines.length > 0) lines.push("");
|
|
248
|
+
lines.push(section.header, ...section.lines);
|
|
249
|
+
}
|
|
250
|
+
return { prompt: [...lines, ...tail].join("\n"), section };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** The fleet preamble a node's first turn carries, before its charter body. */
|
|
254
|
+
export function charterPrompt(node, body, pending = []) {
|
|
255
|
+
const lines = [
|
|
256
|
+
`You are node ${node.node_id} of a fleet. Your charter follows.`,
|
|
257
|
+
`Work the charter with your tools. One tool call per step, in order.`,
|
|
258
|
+
`When the charter's steps are complete, end your turn with the single line ${DONE_SENTINEL}.`,
|
|
259
|
+
`If you cannot continue without a decision from a human, end your turn with one line:`,
|
|
260
|
+
`${ASK_SENTINEL} <question> | default: <the answer to use if nobody replies> | priority: <number, 0 when unsure>`,
|
|
261
|
+
"",
|
|
262
|
+
"CHARTER:",
|
|
263
|
+
body.trim(),
|
|
264
|
+
];
|
|
265
|
+
// A queued message is an operator's instruction for this node, and the first
|
|
266
|
+
// turn is a turn: it rides along instead of waiting for a later one. The
|
|
267
|
+
// charter keeps its shape, and the queue is a section of its own after it.
|
|
268
|
+
return withSection(
|
|
269
|
+
lines,
|
|
270
|
+
queueSection("The operator sent this before your first turn. It applies to this turn as well:", pending),
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/** The between-turns turn: the queued messages, and nothing else to read. */
|
|
275
|
+
export function messagePrompt(pending) {
|
|
276
|
+
return withSection([], queueSection("FLEET MESSAGE.", pending), ["", "Act on it, then end your turn."]);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* The recovery prompt is built from the journal, not from the charter. A node
|
|
281
|
+
* that sees its own completed effects listed cannot be told to start over.
|
|
282
|
+
*
|
|
283
|
+
* `note` is the journal account, and it can be suppressed: the note-removal arm
|
|
284
|
+
* measures what the resumed turn does when the prompt is silent about the
|
|
285
|
+
* effects already applied. Nothing but that arm sets it to false.
|
|
286
|
+
*/
|
|
287
|
+
export function recoveryPrompt({ node, interruptedTurn, completed, unknown, pending, note = true }) {
|
|
288
|
+
const lines = [
|
|
289
|
+
`FLEET RESUME. Node ${node.node_id} keeps the same session. Turn ${interruptedTurn.turn_id} was interrupted while it was running.`,
|
|
290
|
+
];
|
|
291
|
+
if (note && completed.length > 0) {
|
|
292
|
+
lines.push("", "Effects already applied and recorded (do NOT repeat them):");
|
|
293
|
+
for (const effect of completed) {
|
|
294
|
+
lines.push(`- seq ${effect.seq} ${effect.tool_name} completed (idempotency ${effect.idempotency_key})`);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
if (note && unknown.length > 0) {
|
|
298
|
+
lines.push("", "Effects whose outcome is unknown (the tool started and never reported):");
|
|
299
|
+
for (const effect of unknown) {
|
|
300
|
+
lines.push(`- seq ${effect.seq} ${effect.tool_name} unknown (idempotency ${effect.idempotency_key})`);
|
|
301
|
+
}
|
|
302
|
+
// The guard refuses a repeat of these, and the note has to say which gate is
|
|
303
|
+
// closed: a node told to "check the world" without being told what will stop
|
|
304
|
+
// it reads the line as a caution, and one that discovers the refusal by
|
|
305
|
+
// hitting it spends a turn learning what the prompt could have said. The
|
|
306
|
+
// release is named as an operator's decision because it is: a node that can
|
|
307
|
+
// retire its own rows is policing itself.
|
|
308
|
+
lines.push(
|
|
309
|
+
"Re-issuing one of these is refused: an effect whose outcome is unknown counts as history the guard will not let you repeat, and releasing it is an operator's decision, not this turn's. Check the world before acting on these; never repeat a completed effect.",
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
if (note && completed.length === 0 && unknown.length === 0) {
|
|
313
|
+
lines.push("", "No effect had been recorded when the turn was interrupted.");
|
|
314
|
+
}
|
|
315
|
+
return withSection(lines, queueSection("Messages waiting for you:", pending), [
|
|
316
|
+
"",
|
|
317
|
+
"Continue the charter from its first unfinished step.",
|
|
318
|
+
]);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* How much of a child's stderr one turn's log line shows. The driver keeps the
|
|
323
|
+
* last 8000 characters, and the whole of it stays on the driver object; a log
|
|
324
|
+
* line is a report, not a transcript, so it is capped here and names what it
|
|
325
|
+
* withheld rather than trailing off.
|
|
326
|
+
*/
|
|
327
|
+
const CHILD_STDERR_SHOWN = 400;
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* The log line for a child that wrote to stderr, or null when it wrote nothing.
|
|
331
|
+
*
|
|
332
|
+
* A child's stderr is where a guard that failed open says so, and a fail-open
|
|
333
|
+
* guard has no other visible product: it refuses nothing, records nothing, and
|
|
334
|
+
* the turn looks exactly like one the guard watched and found clean. A wrong
|
|
335
|
+
* FLEET_GUARD_ROOT, the likeliest way to be unguarded, leaves its only trace
|
|
336
|
+
* here, so it is reported on every turn that produced any stderr at all.
|
|
337
|
+
*
|
|
338
|
+
* Collapsed to one line and tail-only, because the alternative is a noisy child
|
|
339
|
+
* deciding how large the supervisor's log gets.
|
|
340
|
+
*/
|
|
341
|
+
export function childStderrReport(stderr, { limit = CHILD_STDERR_SHOWN } = {}) {
|
|
342
|
+
const text = String(stderr ?? "");
|
|
343
|
+
if (!text.trim()) return null;
|
|
344
|
+
const collapsed = text.replace(/\s+/g, " ").trim();
|
|
345
|
+
const over = Math.max(0, collapsed.length - limit);
|
|
346
|
+
return {
|
|
347
|
+
chars: text.length,
|
|
348
|
+
withheld: over,
|
|
349
|
+
shown: over > 0 ? `[${over} chars withheld] ${collapsed.slice(-limit)}` : collapsed,
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
export function parseAsk(text) {
|
|
354
|
+
const line = String(text ?? "")
|
|
355
|
+
.split("\n")
|
|
356
|
+
.find((candidate) => candidate.includes(ASK_SENTINEL));
|
|
357
|
+
if (!line) return null;
|
|
358
|
+
const rest = line.slice(line.indexOf(ASK_SENTINEL) + ASK_SENTINEL.length).trim();
|
|
359
|
+
const [question, ...tail] = rest.split("|");
|
|
360
|
+
let defaultAnswer = "";
|
|
361
|
+
let priority = 0;
|
|
362
|
+
for (const field of tail) {
|
|
363
|
+
const [rawKey, ...rawValue] = field.split(":");
|
|
364
|
+
const key = rawKey.trim().toLowerCase();
|
|
365
|
+
const value = rawValue.join(":").trim();
|
|
366
|
+
if (key === "default") defaultAnswer = value;
|
|
367
|
+
if (key === "priority") priority = Number(value) || 0;
|
|
368
|
+
}
|
|
369
|
+
return { question: question.trim(), defaultAnswer: defaultAnswer || DEFAULT_ANSWER, priority };
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
export function isDone(text) {
|
|
373
|
+
return String(text ?? "").includes(DONE_SENTINEL);
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/** Text content of an assistant message, with thinking blocks dropped. */
|
|
377
|
+
export function assistantText(message) {
|
|
378
|
+
const content = message?.content;
|
|
379
|
+
if (typeof content === "string") return content;
|
|
380
|
+
if (!Array.isArray(content)) return "";
|
|
381
|
+
return content
|
|
382
|
+
.filter((block) => block?.type === "text" && typeof block.text === "string")
|
|
383
|
+
.map((block) => block.text)
|
|
384
|
+
.join("\n");
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
export class Supervisor {
|
|
388
|
+
constructor(options) {
|
|
389
|
+
const defaults = {
|
|
390
|
+
cap: 3,
|
|
391
|
+
tickMs: 250,
|
|
392
|
+
askTtlMs: 600_000,
|
|
393
|
+
turnTimeoutMs: 300_000,
|
|
394
|
+
maxAttempts: 3,
|
|
395
|
+
idleExitMs: 5_000,
|
|
396
|
+
// The maintenance budget: how long a pass may spend meeting another
|
|
397
|
+
// process's write lock before it is recorded as deferred and left to a
|
|
398
|
+
// later tick. Two windows at ten seconds plus one backoff is the most a
|
|
399
|
+
// tick's maintenance can spend waiting. Nothing here bounds a lease: a
|
|
400
|
+
// lease is exclusive for as long as its owner answers (journal.mjs).
|
|
401
|
+
drainAttempts: 2,
|
|
402
|
+
drainBusyTimeoutMs: BUSY_TIMEOUT_MS,
|
|
403
|
+
drainBackoffMs: 250,
|
|
404
|
+
// How long a single drain may block before the fact is journalled, even
|
|
405
|
+
// though it completed: one attempt that waited 7s for the lock is a fact
|
|
406
|
+
// an operator wants, not a silent success.
|
|
407
|
+
drainWaitReportMs: 1_000,
|
|
408
|
+
driver: "rpc",
|
|
409
|
+
model: "opencode-go/deepseek-v4.1-flash",
|
|
410
|
+
piBin: "pi",
|
|
411
|
+
naive: false,
|
|
412
|
+
control: true,
|
|
413
|
+
// What a prompt appends for the messages it carries. The only caller that
|
|
414
|
+
// turns it off is the queue arm, which needs a prompt whose body appears
|
|
415
|
+
// somewhere else in the text; no flag maps to it, and it can only make a
|
|
416
|
+
// turn carry less.
|
|
417
|
+
queueSection: true,
|
|
418
|
+
extensions: [],
|
|
419
|
+
extraArgs: [],
|
|
420
|
+
};
|
|
421
|
+
// An explicit `undefined` option means "not given", never "override with
|
|
422
|
+
// nothing": the command line produces undefined for every flag it omits.
|
|
423
|
+
this.options = { ...defaults };
|
|
424
|
+
for (const [key, value] of Object.entries(options ?? {})) {
|
|
425
|
+
if (value !== undefined) this.options[key] = value;
|
|
426
|
+
}
|
|
427
|
+
this.journal = openJournal(this.options.root, {
|
|
428
|
+
projectRoot: this.options.projectRoot ?? this.options.root,
|
|
429
|
+
driver: this.options.driver,
|
|
430
|
+
childArgs: this.options.extraArgs ?? [],
|
|
431
|
+
piVersion: this.options.piVersion ?? "unset",
|
|
432
|
+
});
|
|
433
|
+
this.root = this.journal.root;
|
|
434
|
+
this.epoch = null;
|
|
435
|
+
this.active = new Map();
|
|
436
|
+
this.timer = null;
|
|
437
|
+
this.server = null;
|
|
438
|
+
// The file this process bound, as the filesystem names it. A name is not an
|
|
439
|
+
// identity: another process can rebind the same path, so the stop below
|
|
440
|
+
// compares this against the path rather than trusting the name.
|
|
441
|
+
this.boundSocket = null;
|
|
442
|
+
this.stopping = false;
|
|
443
|
+
this.fences = [];
|
|
444
|
+
this.idleSince = null;
|
|
445
|
+
// A maintenance pass that could not write: the store was locked when the
|
|
446
|
+
// fact was made. It is written beside the store at once and journalled at
|
|
447
|
+
// the first pass that gets through, so it outlives this process - the
|
|
448
|
+
// measured case is a tick that threw `database is locked` and left an
|
|
449
|
+
// overdue ask open with nothing saying why.
|
|
450
|
+
this.pendingStoreDeferral = null;
|
|
451
|
+
// Resolved when this process has stopped itself because the fleet had
|
|
452
|
+
// nothing left to do; the entry point exits on it rather than on a guess.
|
|
453
|
+
this.done = new Promise((resolveDone) => {
|
|
454
|
+
this.doneResolve = resolveDone;
|
|
455
|
+
});
|
|
456
|
+
this.log = this.options.log ?? ((line) => console.log(line));
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
async start() {
|
|
460
|
+
// The journal records the driver and the version it ran under, so the
|
|
461
|
+
// choice between an RPC child and an in-process SDK stays reversible.
|
|
462
|
+
if (!this.options.piVersion && this.options.driver !== "scripted") {
|
|
463
|
+
try {
|
|
464
|
+
this.options.piVersion = execFileSync(this.options.piBin, ["--version"], { encoding: "utf8" }).trim();
|
|
465
|
+
} catch (error) {
|
|
466
|
+
this.options.piVersion = `unavailable: ${error.code ?? error.message}`;
|
|
467
|
+
}
|
|
468
|
+
this.journal.setMeta("pi_version", this.options.piVersion);
|
|
469
|
+
}
|
|
470
|
+
this.epoch = this.journal.registerSupervisor({
|
|
471
|
+
pid: process.pid,
|
|
472
|
+
driver: this.options.driver,
|
|
473
|
+
childArgs: this.options.extraArgs ?? [],
|
|
474
|
+
piVersion: this.options.piVersion,
|
|
475
|
+
});
|
|
476
|
+
this.log(`fleet: supervisor pid ${process.pid} epoch ${this.epoch} store ${this.journal.path}`);
|
|
477
|
+
await this.reconcile();
|
|
478
|
+
if (this.options.control) await this._listen();
|
|
479
|
+
this._schedule();
|
|
480
|
+
return this;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// ── reconciliation ───────────────────────────────────────────────────────
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* Repair what a dead predecessor left: dead epoch rows, expired leases, and
|
|
487
|
+
* every turn still marked in-progress. The turn's process group is fenced
|
|
488
|
+
* before its effects are read, so the next drive cannot race a writer that
|
|
489
|
+
* is still alive.
|
|
490
|
+
*
|
|
491
|
+
* A lease and a turn left behind by the same owner are the same owner's, so
|
|
492
|
+
* both loops ask `ownerLiveness` whether that owner is still there. Fencing a
|
|
493
|
+
* live owner's turn kills a child that is still working and records a
|
|
494
|
+
* recovery that never happened.
|
|
495
|
+
*/
|
|
496
|
+
async reconcile() {
|
|
497
|
+
const supervisors = this.journal.supervisors();
|
|
498
|
+
for (const supervisor of supervisors) {
|
|
499
|
+
if (supervisor.pid === process.pid) continue;
|
|
500
|
+
if (supervisor.state === "live" && !pidIsAlive(supervisor.pid)) {
|
|
501
|
+
this.journal.setSupervisorState(supervisor.pid, "dead");
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
const owners = this.ownerLiveness();
|
|
505
|
+
for (const lease of this.journal.liveLeases()) {
|
|
506
|
+
if (lease.epoch === this.epoch) continue;
|
|
507
|
+
if (owners.get(lease.epoch)) continue;
|
|
508
|
+
this.journal.expireLease(lease.lease_id);
|
|
509
|
+
}
|
|
510
|
+
let skippedTurns = 0;
|
|
511
|
+
const openTurns = this.journal.allOpenTurns();
|
|
512
|
+
for (const turn of openTurns) {
|
|
513
|
+
if (turn.supervisor_epoch === this.epoch) continue;
|
|
514
|
+
if (owners.get(turn.supervisor_epoch)) {
|
|
515
|
+
skippedTurns += 1;
|
|
516
|
+
continue;
|
|
517
|
+
}
|
|
518
|
+
const fence = await this.fenceGroup(turn);
|
|
519
|
+
const unknown = this.journal.markUnknownEffects(turn.turn_id);
|
|
520
|
+
this.journal.endTurn(turn.turn_id, { state: "interrupted" });
|
|
521
|
+
// Messages the interrupted turn had already recorded as carried go back to
|
|
522
|
+
// the queue: the prompt went out and the turn did not finish, so the
|
|
523
|
+
// recovery turn below is the first place they can be acted on.
|
|
524
|
+
const abandoned = this.journal
|
|
525
|
+
.messages(turn.node_id)
|
|
526
|
+
.filter((message) => message.state === "delivered" && message.delivered_turn_id === turn.turn_id)
|
|
527
|
+
.map((message) => message.msg_id);
|
|
528
|
+
const requeued = this.journal.releaseMessages(abandoned);
|
|
529
|
+
this.journal.bumpAttempt(turn.node_id);
|
|
530
|
+
const node = this.journal.getNode(turn.node_id);
|
|
531
|
+
if (node && node.state !== "done") {
|
|
532
|
+
// The bound closes the crash loop: without it a node that kills its
|
|
533
|
+
// supervisor every time is interrupted, settled, resumed, and
|
|
534
|
+
// interrupted again, for ever.
|
|
535
|
+
if (node.attempts >= this.options.maxAttempts) {
|
|
536
|
+
this.journal.setNodeState(turn.node_id, "failed", {
|
|
537
|
+
done_reason: `attempt bound: ${node.attempts} failed or interrupted attempts reached maxAttempts ${this.options.maxAttempts}`,
|
|
538
|
+
});
|
|
539
|
+
this.log(
|
|
540
|
+
`fleet: node ${turn.node_id} reached the attempt bound (${node.attempts}/${this.options.maxAttempts}); not resuming it`,
|
|
541
|
+
);
|
|
542
|
+
} else {
|
|
543
|
+
this.journal.setNodeState(turn.node_id, "settled");
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
this.journal.setMeta("fences", JSON.stringify(this.fences));
|
|
547
|
+
this.log(
|
|
548
|
+
`fleet: reconciled turn ${turn.turn_id} (node ${turn.node_id}) fence=${fence.killed ? "killed" : fence.reason} unknown_effects=${unknown.length} requeued_messages=${requeued}`,
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
this.journal.setMeta("reconcile_skipped_turns", String(skippedTurns));
|
|
552
|
+
this.drainExpiredAsks();
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/**
|
|
556
|
+
* Which supervisor epochs are still alive, by the one predicate both repair
|
|
557
|
+
* loops need: a row that says `live` whose pid answers. The repair pass above
|
|
558
|
+
* has already demoted a live row whose pid is gone, so a `live` row here is a
|
|
559
|
+
* process the box still holds.
|
|
560
|
+
*/
|
|
561
|
+
ownerLiveness() {
|
|
562
|
+
const owners = new Map();
|
|
563
|
+
for (const row of this.journal.supervisors()) {
|
|
564
|
+
owners.set(row.epoch, row.state === "live" && pidIsAlive(row.pid));
|
|
565
|
+
}
|
|
566
|
+
return owners;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
// ── the ask deadline ─────────────────────────────────────────────────────
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* The one drain of expired asks. Resolving an ask to its default is only half
|
|
573
|
+
* the job: nothing tells the node to move unless the default is also queued
|
|
574
|
+
* as an ordinary message. Both callers - the tick, and the restart that finds
|
|
575
|
+
* a deadline passed while no supervisor was running - come through here, so
|
|
576
|
+
* the two paths cannot drift into one of them doing half the work.
|
|
577
|
+
*
|
|
578
|
+
* Only what `expireAsks()` claimed is queued. It returns the rows this call
|
|
579
|
+
* actually transitioned, not the rows it read, so a second drainer of the same
|
|
580
|
+
* ask claims nothing and queues nothing: the queue decision rests on the
|
|
581
|
+
* conditional update, not on a shared read.
|
|
582
|
+
*
|
|
583
|
+
* Contention is waited out, not thrown. A drain that meets the store's write
|
|
584
|
+
* lock waits out the journal's busy timeout, retries with a backoff, and
|
|
585
|
+
* reports a drain it could not finish as a fact (`store_deferred`) instead of
|
|
586
|
+
* as an exception from a tick. The earlier shape was measured: two drainers on
|
|
587
|
+
* one store, a full-suite run, one of them `database is locked` and gone, and
|
|
588
|
+
* the asks it was there to drain stayed held. A supervisor whose drain throws
|
|
589
|
+
* does not drain that ask, and its caller is a tick handler, so the failure
|
|
590
|
+
* was a log line and a skipped maintenance pass.
|
|
591
|
+
*/
|
|
592
|
+
drainExpiredAsks() {
|
|
593
|
+
const started = Date.now();
|
|
594
|
+
const attempts = Math.max(1, Math.floor(this.options.drainAttempts));
|
|
595
|
+
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
596
|
+
try {
|
|
597
|
+
const { expired, notes } = this.journal.transaction(() => this._claimExpiredAsks(), {
|
|
598
|
+
busyTimeoutMs: this.options.drainBusyTimeoutMs,
|
|
599
|
+
});
|
|
600
|
+
// After the commit, so a retried attempt cannot log the same claim
|
|
601
|
+
// twice, and so the log line describes a decision that survived.
|
|
602
|
+
for (const note of notes) this.log(note);
|
|
603
|
+
this._recordDrainContention({ attempt, waitedMs: Date.now() - started });
|
|
604
|
+
return expired;
|
|
605
|
+
} catch (error) {
|
|
606
|
+
// A locked store is the one error this drain answers itself. Anything
|
|
607
|
+
// else is a defect in the body, and swallowing it would hide a real
|
|
608
|
+
// failure behind a maintenance pass nobody reads.
|
|
609
|
+
if (!isBusyError(error)) throw error;
|
|
610
|
+
const why = error.errstr ?? error.message;
|
|
611
|
+
if (attempt >= attempts) {
|
|
612
|
+
this._recordStoreDeferral({ what: "drain", attempts: attempt, waitedMs: Date.now() - started, error: why });
|
|
613
|
+
return [];
|
|
614
|
+
}
|
|
615
|
+
const backoffMs = this.options.drainBackoffMs * attempt;
|
|
616
|
+
this.log(
|
|
617
|
+
`fleet: drain met ${why} on attempt ${attempt}/${attempts}; waiting ${backoffMs} ms and retrying`,
|
|
618
|
+
);
|
|
619
|
+
sleepSync(backoffMs);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
return [];
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/**
|
|
626
|
+
* The drain's body: claim the overdue asks and queue one message per claimed
|
|
627
|
+
* row. It runs inside the transaction, so it must stay synchronous and free
|
|
628
|
+
* of side effects that a rollback cannot undo - the log lines are returned to
|
|
629
|
+
* the caller rather than printed here.
|
|
630
|
+
*/
|
|
631
|
+
_claimExpiredAsks() {
|
|
632
|
+
// Claim and queue as one transaction. A supervisor killed between the two
|
|
633
|
+
// leaves an ask answered with no message queued for it, and a node parked
|
|
634
|
+
// on that ask then has no open ask, no pending message and no ready kind,
|
|
635
|
+
// so nothing ever drives it again: the parked-forever shape this drain
|
|
636
|
+
// exists to prevent, reached through the drain itself. BEGIN IMMEDIATE also
|
|
637
|
+
// serializes two drainers, so the read that decides a claim cannot land
|
|
638
|
+
// inside the other process's claims.
|
|
639
|
+
const expired = this.journal.expireAsks();
|
|
640
|
+
const notes = [];
|
|
641
|
+
for (const ask of expired) {
|
|
642
|
+
// The ask is claimed and answered by now; what is left is whether the
|
|
643
|
+
// node should be woken. A done node must not be resurrected: the fence
|
|
644
|
+
// loop guards this, and the drain queued for it anyway, so an overdue
|
|
645
|
+
// ask on a finished node flipped it back to `settled` and the default
|
|
646
|
+
// then drove it a second time.
|
|
647
|
+
const node = this.journal.getNode(ask.node_id);
|
|
648
|
+
if (!node || node.state === "done") {
|
|
649
|
+
notes.push(
|
|
650
|
+
`fleet: ask ${ask.token} expired for node ${ask.node_id}, which is ${node ? "done" : "missing"}; default not queued`,
|
|
651
|
+
);
|
|
652
|
+
continue;
|
|
653
|
+
}
|
|
654
|
+
this.journal.addMessage({
|
|
655
|
+
msgId: `m-${randomBytes(6).toString("hex")}`,
|
|
656
|
+
nodeId: ask.node_id,
|
|
657
|
+
kind: "answer",
|
|
658
|
+
body: ask.default_answer,
|
|
659
|
+
token: ask.token,
|
|
660
|
+
});
|
|
661
|
+
this.journal.setNodeState(ask.node_id, "settled");
|
|
662
|
+
notes.push(`fleet: ask ${ask.token} hit its deadline; default answer queued`);
|
|
663
|
+
}
|
|
664
|
+
return { expired, notes };
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
/**
|
|
668
|
+
* Record that a drain waited for the lock, and flush a deferral a previous
|
|
669
|
+
* drain could not write.
|
|
670
|
+
*
|
|
671
|
+
* Neither write is allowed to throw: this runs after a successful drain, and
|
|
672
|
+
* losing the note because another writer holds the lock again is not a reason
|
|
673
|
+
* to report the drain itself as failed. The deferral stays pending until a
|
|
674
|
+
* write lands, which is the earliest point the store can hold the fact at all.
|
|
675
|
+
*/
|
|
676
|
+
_recordDrainContention({ attempt, waitedMs }) {
|
|
677
|
+
this._flushStoreDeferral(Date.now());
|
|
678
|
+
if (attempt <= 1 && waitedMs < this.options.drainWaitReportMs) return;
|
|
679
|
+
try {
|
|
680
|
+
this.journal.transaction(
|
|
681
|
+
() =>
|
|
682
|
+
this.journal.setMeta("drain_contention", JSON.stringify({ at: Date.now(), attempts: attempt, waitedMs })),
|
|
683
|
+
{ busyTimeoutMs: Math.max(100, this.options.drainBackoffMs) },
|
|
684
|
+
);
|
|
685
|
+
} catch (error) {
|
|
686
|
+
// Best effort: the drain's decision is already committed and correct.
|
|
687
|
+
this.log(`fleet: the drain could not record its wait (${error.message}); the next drain retries the note`);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/** The path a fact goes to when the store will not take it. */
|
|
692
|
+
deferredPath() {
|
|
693
|
+
return join(this.root, DEFERRED_FILE);
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* The deferral as it stands: this process's, or one an earlier process left
|
|
698
|
+
* beside the store. Reading the file is what makes the fact survive a death,
|
|
699
|
+
* and it is also what lets the next supervisor finish the job.
|
|
700
|
+
*/
|
|
701
|
+
_pendingDeferral() {
|
|
702
|
+
if (this.pendingStoreDeferral) return this.pendingStoreDeferral;
|
|
703
|
+
const onDisk = deferralFromDisk(this.root);
|
|
704
|
+
if (onDisk) this.pendingStoreDeferral = onDisk;
|
|
705
|
+
return this.pendingStoreDeferral;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
/**
|
|
709
|
+
* Put the fact where it can be read with no lock: a file beside the journal.
|
|
710
|
+
*
|
|
711
|
+
* There is no second try if this fails - the store is unwritable and so is the
|
|
712
|
+
* root - so the log line is the last surface left, and it says so.
|
|
713
|
+
*/
|
|
714
|
+
_writeDeferralFile(fact) {
|
|
715
|
+
const path = this.deferredPath();
|
|
716
|
+
try {
|
|
717
|
+
writeFileSync(`${path}.tmp`, `${JSON.stringify(fact, null, 2)}\n`);
|
|
718
|
+
renameSync(`${path}.tmp`, path);
|
|
719
|
+
} catch (error) {
|
|
720
|
+
this.log(`fleet: the deferral could not be written to ${path} (${error.code ?? error.message})`);
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
/**
|
|
725
|
+
* Copy a pending deferral into the journal, if one is pending. `recoveredAt`
|
|
726
|
+
* says whether a later pass got the work done anyway, which is what tells a
|
|
727
|
+
* reader whether the store was merely busy or the work is still waiting.
|
|
728
|
+
*
|
|
729
|
+
* The file is removed only after the journal has the fact, so a crash between
|
|
730
|
+
* the two leaves it readable rather than losing it; the next pass writes it
|
|
731
|
+
* again.
|
|
732
|
+
*/
|
|
733
|
+
_flushStoreDeferral(recoveredAt) {
|
|
734
|
+
const pending = this._pendingDeferral();
|
|
735
|
+
if (!pending) return false;
|
|
736
|
+
const fact = recoveredAt == null ? pending : { ...pending, recoveredAt };
|
|
737
|
+
try {
|
|
738
|
+
this.journal.transaction(() => this.journal.setMeta("store_deferred", JSON.stringify(fact)), {
|
|
739
|
+
busyTimeoutMs: Math.max(100, this.options.drainBackoffMs),
|
|
740
|
+
});
|
|
741
|
+
} catch (error) {
|
|
742
|
+
this.log(`fleet: the deferral could not be journalled yet (${error.message}); the next pass retries it`);
|
|
743
|
+
return false;
|
|
744
|
+
}
|
|
745
|
+
this.pendingStoreDeferral = null;
|
|
746
|
+
rmSync(this.deferredPath(), { force: true });
|
|
747
|
+
if (recoveredAt != null) this.log("fleet: the earlier deferral was journalled; a later pass completed the work");
|
|
748
|
+
return true;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
/**
|
|
752
|
+
* A maintenance pass that gave up on a locked store. The store cannot hold
|
|
753
|
+
* this fact - that is the fact - so it goes to the file first, then into the
|
|
754
|
+
* journal at the first moment the store accepts a write. It is not an
|
|
755
|
+
* exception: the tick keeps ticking, and the pass is retried.
|
|
756
|
+
*/
|
|
757
|
+
_recordStoreDeferral({ what, attempts, waitedMs, error }) {
|
|
758
|
+
const fact = { at: Date.now(), what, attempts, waitedMs, error };
|
|
759
|
+
this.pendingStoreDeferral = fact;
|
|
760
|
+
this._writeDeferralFile(fact);
|
|
761
|
+
this.log(
|
|
762
|
+
`fleet: ${what} could not write after ${attempts} attempt(s) in ${waitedMs} ms (${error}); recorded in ${DEFERRED_FILE} until the store takes it`,
|
|
763
|
+
);
|
|
764
|
+
this._flushStoreDeferral(null);
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
/**
|
|
768
|
+
* Kill the process group a dead turn left behind, and wait until the group is
|
|
769
|
+
* gone before reading its effects. A signal that was sent is not a fence that
|
|
770
|
+
* held, so the record carries what was observed after the wait.
|
|
771
|
+
*/
|
|
772
|
+
async fenceGroup(turn) {
|
|
773
|
+
const pgid = turn.child_pgid;
|
|
774
|
+
const record = {
|
|
775
|
+
turnId: turn.turn_id,
|
|
776
|
+
nodeId: turn.node_id,
|
|
777
|
+
pgid,
|
|
778
|
+
killed: false,
|
|
779
|
+
reason: "",
|
|
780
|
+
at: Date.now(),
|
|
781
|
+
waitedMs: 0,
|
|
782
|
+
};
|
|
783
|
+
if (!pgid) {
|
|
784
|
+
record.reason = "no child recorded";
|
|
785
|
+
} else if (!groupIsAlive(pgid)) {
|
|
786
|
+
record.reason = "group already gone";
|
|
787
|
+
} else {
|
|
788
|
+
try {
|
|
789
|
+
process.kill(-pgid, "SIGKILL");
|
|
790
|
+
record.killed = true;
|
|
791
|
+
record.reason = "SIGKILL to process group";
|
|
792
|
+
} catch (error) {
|
|
793
|
+
record.reason = `kill failed: ${error.code ?? error.message}`;
|
|
794
|
+
}
|
|
795
|
+
const started = Date.now();
|
|
796
|
+
while (groupIsAlive(pgid) && Date.now() - started < 500) {
|
|
797
|
+
await new Promise((resolveWait) => setTimeout(resolveWait, 10));
|
|
798
|
+
}
|
|
799
|
+
record.waitedMs = Date.now() - started;
|
|
800
|
+
}
|
|
801
|
+
record.aliveAfter = groupIsAlive(pgid);
|
|
802
|
+
this.fences.push(record);
|
|
803
|
+
return record;
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
// ── scheduling ───────────────────────────────────────────────────────────
|
|
807
|
+
|
|
808
|
+
_schedule() {
|
|
809
|
+
if (this.stopping) return;
|
|
810
|
+
this.timer = setTimeout(() => {
|
|
811
|
+
this.tick()
|
|
812
|
+
.catch((error) => this.log(`fleet: tick failed: ${error.message}`))
|
|
813
|
+
.finally(() => this._schedule());
|
|
814
|
+
}, this.options.tickMs);
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
/**
|
|
818
|
+
* The tick's maintenance pass: the expired asks, then the queued messages no
|
|
819
|
+
* turn will ever carry.
|
|
820
|
+
*
|
|
821
|
+
* One named pass because it is the part of a tick another process's write lock
|
|
822
|
+
* can stall, and because both halves are retried by the next tick rather than
|
|
823
|
+
* failed for good - which is only true if the failure is recorded. Returns what
|
|
824
|
+
* the drain claimed, so a caller can say what the pass did.
|
|
825
|
+
*/
|
|
826
|
+
maintenance() {
|
|
827
|
+
const expired = this.drainExpiredAsks();
|
|
828
|
+
this._maintenanceWrite("undeliverable sweep", () => this.retireUndeliverableMessages());
|
|
829
|
+
return expired;
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* One maintenance write, with a busy store recorded instead of thrown.
|
|
834
|
+
*
|
|
835
|
+
* The drain answers its own contention; the pass after it meets the same lock
|
|
836
|
+
* and has no policy of its own, and it used to throw into the tick's catch: one
|
|
837
|
+
* log line, no fact, and the rest of the tick skipped. A store that will not
|
|
838
|
+
* take the write is a fact an operator can read, and the next tick retries the
|
|
839
|
+
* pass. Anything that is not contention is a defect and stays an exception.
|
|
840
|
+
*/
|
|
841
|
+
_maintenanceWrite(what, body) {
|
|
842
|
+
const started = Date.now();
|
|
843
|
+
try {
|
|
844
|
+
this.journal.withBusyTimeout(Math.max(100, this.options.drainBusyTimeoutMs), body);
|
|
845
|
+
} catch (error) {
|
|
846
|
+
if (!isBusyError(error)) throw error;
|
|
847
|
+
this._recordStoreDeferral({
|
|
848
|
+
what,
|
|
849
|
+
attempts: 1,
|
|
850
|
+
waitedMs: Date.now() - started,
|
|
851
|
+
error: error.errstr ?? error.message,
|
|
852
|
+
});
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
async tick() {
|
|
857
|
+
this.journal.heartbeatSupervisor(process.pid);
|
|
858
|
+
for (const [nodeId, ctx] of this.active) {
|
|
859
|
+
const renewed = this.journal.acquireLease({
|
|
860
|
+
nodeId,
|
|
861
|
+
ownerPid: process.pid,
|
|
862
|
+
epoch: this.epoch,
|
|
863
|
+
});
|
|
864
|
+
if (!renewed) this.log(`fleet: lease for ${nodeId} could not be renewed (ctx ${ctx.turnId})`);
|
|
865
|
+
}
|
|
866
|
+
this.maintenance();
|
|
867
|
+
if (this.active.size < this.options.cap) {
|
|
868
|
+
for (const node of this.journal.listNodes()) {
|
|
869
|
+
if (this.stopping || this.active.size >= this.options.cap) break;
|
|
870
|
+
if (this.active.has(node.node_id)) continue;
|
|
871
|
+
const kind = this.readyKind(node);
|
|
872
|
+
if (!kind) continue;
|
|
873
|
+
this.driveNode(node, kind).catch((error) => {
|
|
874
|
+
this.log(`fleet: drive ${node.node_id} threw: ${error.message}`);
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
await this.idleCheck();
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
// ── stopping when there is nothing to do ────────────────────────────────
|
|
882
|
+
|
|
883
|
+
/**
|
|
884
|
+
* Whether the fleet has anything left to do. Three things are work at any
|
|
885
|
+
* capacity: a turn this process is running, an open ask (a cap-0 supervisor
|
|
886
|
+
* still drains an expired one, because the drain runs before the cap gate),
|
|
887
|
+
* and a lease this epoch has not let go of.
|
|
888
|
+
*
|
|
889
|
+
* A runnable node is work only when this process could drive it. At cap 0 the
|
|
890
|
+
* drive loop never runs, so a node the scheduler would name is a node this
|
|
891
|
+
* supervisor is structurally unable to start; counting it as work is how a
|
|
892
|
+
* cap-0 supervisor outlives its usefulness holding a journal full of nodes it
|
|
893
|
+
* will never drive.
|
|
894
|
+
*/
|
|
895
|
+
hasWork() {
|
|
896
|
+
if (this.active.size > 0) return true;
|
|
897
|
+
if (this.journal.openAsks().length > 0) return true;
|
|
898
|
+
if (this.journal.liveLeases().some((lease) => lease.epoch === this.epoch)) return true;
|
|
899
|
+
if (this.options.cap <= 0) return false;
|
|
900
|
+
return this.journal.listNodes().some((node) => this.readyKind(node) !== null);
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
/**
|
|
904
|
+
* The idle exit: nothing to do for a sustained window means stop and let the
|
|
905
|
+
* process end. A single empty sample is not that window - a node is
|
|
906
|
+
* registered before the supervisor that drives it is started, so a fresh
|
|
907
|
+
* supervisor's first tick legitimately finds nothing ready - so the window is
|
|
908
|
+
* re-sampled on every tick and any work resets it. `idleExitMs: 0` disables
|
|
909
|
+
* the exit for a supervisor meant to wait.
|
|
910
|
+
*/
|
|
911
|
+
async idleCheck() {
|
|
912
|
+
const windowMs = this.options.idleExitMs;
|
|
913
|
+
if (!windowMs || this.stopping) return;
|
|
914
|
+
if (this.hasWork()) {
|
|
915
|
+
this.idleSince = null;
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
918
|
+
if (this.idleSince == null) {
|
|
919
|
+
this.idleSince = Date.now();
|
|
920
|
+
return;
|
|
921
|
+
}
|
|
922
|
+
const idleMs = Date.now() - this.idleSince;
|
|
923
|
+
if (idleMs < windowMs) return;
|
|
924
|
+
// One last look in the same synchronous stretch as the stop, so a node that
|
|
925
|
+
// arrived while the window elapsed is not exited from.
|
|
926
|
+
if (this.hasWork()) {
|
|
927
|
+
this.idleSince = null;
|
|
928
|
+
return;
|
|
929
|
+
}
|
|
930
|
+
this.log(`fleet: no work left for ${idleMs} ms; stopping`);
|
|
931
|
+
try {
|
|
932
|
+
await this.stop();
|
|
933
|
+
} catch (error) {
|
|
934
|
+
this.log(`fleet: stop during the idle exit failed: ${error.message}`);
|
|
935
|
+
this.doneResolve(false);
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
/**
|
|
940
|
+
* Why a node is ready to be driven right now, or null. The five states are
|
|
941
|
+
* the design's; `waiting_on_child` is repaired here rather than trusted,
|
|
942
|
+
* because a killed supervisor leaves it behind.
|
|
943
|
+
*
|
|
944
|
+
* Order matters: an interrupted turn outranks a queued message, because the
|
|
945
|
+
* recovery prompt carries the queued messages with it; a node that has never
|
|
946
|
+
* been driven takes its charter before anything else.
|
|
947
|
+
*/
|
|
948
|
+
readyKind(node) {
|
|
949
|
+
if (this.journal.openAsks(node.node_id).length > 0) return null;
|
|
950
|
+
if (undeliverableReason(node, this.options.maxAttempts)) return null;
|
|
951
|
+
const turns = this.journal.turnsFor(node.node_id);
|
|
952
|
+
const last = turns.at(-1) ?? null;
|
|
953
|
+
if (last && last.state === "in_progress") return null;
|
|
954
|
+
if (turns.length === 0) return "charter";
|
|
955
|
+
if (last && last.state === "interrupted") return this.options.naive ? "naive" : "recovery";
|
|
956
|
+
if (this.journal.pendingMessages(node.node_id).length > 0) return "message";
|
|
957
|
+
return null;
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
/**
|
|
961
|
+
* Queued messages on a node that will never run again take a terminal state
|
|
962
|
+
* of their own.
|
|
963
|
+
*
|
|
964
|
+
* `recorded` is a promise that some turn will carry the message. When the node
|
|
965
|
+
* it was routed to is done, or failed at its attempt bound, no turn will ever
|
|
966
|
+
* do that, and leaving the row in `recorded` is the same silent lie as
|
|
967
|
+
* retiring a message that never reached a prompt. The sweep runs on every
|
|
968
|
+
* tick, so a message is stamped as soon as a supervisor can see that it cannot
|
|
969
|
+
* be delivered - including one that was queued while the node was still
|
|
970
|
+
* running and whose later turn ended the node instead of carrying it.
|
|
971
|
+
*/
|
|
972
|
+
retireUndeliverableMessages() {
|
|
973
|
+
for (const node of this.journal.listNodes()) {
|
|
974
|
+
if (this.active.has(node.node_id)) continue;
|
|
975
|
+
const why = undeliverableReason(node, this.options.maxAttempts);
|
|
976
|
+
if (!why) continue;
|
|
977
|
+
const retired = this.journal.markMessagesUndeliverable(node.node_id);
|
|
978
|
+
if (retired.length > 0) {
|
|
979
|
+
this.log(
|
|
980
|
+
`fleet: ${why}; ${retired.length} queued message(s) for node ${node.node_id} can never be delivered: ${retired
|
|
981
|
+
.map((message) => message.msg_id)
|
|
982
|
+
.join(", ")}`,
|
|
983
|
+
);
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
// ── driving one node ─────────────────────────────────────────────────────
|
|
989
|
+
|
|
990
|
+
createDriver(node, turnId) {
|
|
991
|
+
const config = JSON.parse(node.driver_config || "{}");
|
|
992
|
+
const sessionId = node.session_id ?? `fleet-${node.node_id}-${randomBytes(3).toString("hex")}`;
|
|
993
|
+
if (node.driver === "scripted") {
|
|
994
|
+
return createScriptedDriver({
|
|
995
|
+
steps: config.steps ?? [],
|
|
996
|
+
worldFile: config.worldFile ?? join(this.root, "world.log"),
|
|
997
|
+
sessionId,
|
|
998
|
+
sessionFile: node.session_file ?? join(this.root, "sessions", `${sessionId}.jsonl`),
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
return createRpcDriver({
|
|
1002
|
+
piBin: this.options.piBin,
|
|
1003
|
+
cwd: node.project_root,
|
|
1004
|
+
sessionId,
|
|
1005
|
+
sessionDir: join(this.root, "sessions"),
|
|
1006
|
+
model: this.options.model,
|
|
1007
|
+
extensions: this.options.extensions,
|
|
1008
|
+
guardExtension: GUARD_EXTENSION,
|
|
1009
|
+
env: guardEnvironment({ root: this.root, nodeId: node.node_id, turnId }),
|
|
1010
|
+
extraArgs: this.options.extraArgs,
|
|
1011
|
+
fullDiscovery: this.options.fullDiscovery,
|
|
1012
|
+
thinking: this.options.thinking,
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
async driveNode(node, kind) {
|
|
1017
|
+
const lease = this.journal.acquireLease({
|
|
1018
|
+
nodeId: node.node_id,
|
|
1019
|
+
ownerPid: process.pid,
|
|
1020
|
+
epoch: this.epoch,
|
|
1021
|
+
});
|
|
1022
|
+
if (!lease) return;
|
|
1023
|
+
const ctx = {
|
|
1024
|
+
nodeId: node.node_id,
|
|
1025
|
+
turnId: null,
|
|
1026
|
+
driver: null,
|
|
1027
|
+
kind,
|
|
1028
|
+
toolEnds: 0,
|
|
1029
|
+
requestSeq: 0,
|
|
1030
|
+
effectSeq: 0,
|
|
1031
|
+
usage: { input: 0, output: 0 },
|
|
1032
|
+
settled: false,
|
|
1033
|
+
settleResolve: null,
|
|
1034
|
+
finalText: "",
|
|
1035
|
+
lastAssistantText: "",
|
|
1036
|
+
error: null,
|
|
1037
|
+
startedAt: Date.now(),
|
|
1038
|
+
// What this turn's prompt carries, and what was steered into it. Both are
|
|
1039
|
+
// decided by the turn itself; the settle gate retires exactly these.
|
|
1040
|
+
carried: [],
|
|
1041
|
+
steeredIds: [],
|
|
1042
|
+
};
|
|
1043
|
+
ctx.settlePromise = new Promise((resolveSettle) => {
|
|
1044
|
+
ctx.settleResolve = resolveSettle;
|
|
1045
|
+
});
|
|
1046
|
+
this.active.set(node.node_id, ctx);
|
|
1047
|
+
try {
|
|
1048
|
+
const refusal = this.journal.budgetRefusal(node.project_root, node.node_id);
|
|
1049
|
+
if (refusal) {
|
|
1050
|
+
this.journal.setNodeState(node.node_id, "failed", { done_reason: `budget: ${refusal}` });
|
|
1051
|
+
return;
|
|
1052
|
+
}
|
|
1053
|
+
const pending = this.journal.pendingMessages(node.node_id);
|
|
1054
|
+
// What the prompt is built with. The queue arm is the only caller that
|
|
1055
|
+
// passes none while messages are waiting, and it does so to build the
|
|
1056
|
+
// prompt the old carrier could not tell from a delivery: the body present
|
|
1057
|
+
// in the text, and no section that put it there.
|
|
1058
|
+
const queued = this.options.queueSection === false ? [] : pending;
|
|
1059
|
+
const turns = this.journal.turnsFor(node.node_id);
|
|
1060
|
+
const interrupted = turns.at(-1)?.state === "interrupted" ? turns.at(-1) : null;
|
|
1061
|
+
let built;
|
|
1062
|
+
let promptKind = kind;
|
|
1063
|
+
if (kind === "recovery" && interrupted) {
|
|
1064
|
+
built = recoveryPrompt({
|
|
1065
|
+
node,
|
|
1066
|
+
interruptedTurn: interrupted,
|
|
1067
|
+
completed: this.journal.settledEffects(interrupted.turn_id),
|
|
1068
|
+
unknown: this.journal
|
|
1069
|
+
.effects(interrupted.turn_id)
|
|
1070
|
+
.filter((effect) => effect.state === "unknown" || effect.state === "attempted"),
|
|
1071
|
+
pending: queued,
|
|
1072
|
+
// In-process only, and no command-line flag maps to it: the
|
|
1073
|
+
// note-removal arm is the single caller that turns the note off.
|
|
1074
|
+
note: this.options.recoveryNote !== false,
|
|
1075
|
+
});
|
|
1076
|
+
} else if (kind === "message") {
|
|
1077
|
+
built = messagePrompt(queued);
|
|
1078
|
+
} else {
|
|
1079
|
+
built = charterPrompt(node, this._charter(node), queued);
|
|
1080
|
+
if (kind === "naive") promptKind = "naive-replay";
|
|
1081
|
+
}
|
|
1082
|
+
const prompt = built.prompt;
|
|
1083
|
+
// The messages this turn carried, from the section the builder appended
|
|
1084
|
+
// and the text that holds it - never from a search of the finished prompt
|
|
1085
|
+
// for a body. A charter that repeats an operator instruction is not a
|
|
1086
|
+
// delivery of the message that happens to say the same thing: the row
|
|
1087
|
+
// keeps waiting, and the next turn that carries it retires it.
|
|
1088
|
+
ctx.carried = carriedBy(prompt, built.section);
|
|
1089
|
+
this.journal.setNodeState(node.node_id, "waiting_on_child");
|
|
1090
|
+
const turnId = this.journal.beginTurn(node.node_id, {
|
|
1091
|
+
kind: promptKind,
|
|
1092
|
+
promptText: prompt,
|
|
1093
|
+
sessionId: node.session_id,
|
|
1094
|
+
supervisorEpoch: this.epoch,
|
|
1095
|
+
});
|
|
1096
|
+
ctx.turnId = turnId;
|
|
1097
|
+
for (const msgId of ctx.carried) {
|
|
1098
|
+
this.journal.markMessageDelivered(msgId, {
|
|
1099
|
+
tier: "B",
|
|
1100
|
+
turnId,
|
|
1101
|
+
afterEffectSeq: 0,
|
|
1102
|
+
beforeRequestSeq: 0,
|
|
1103
|
+
});
|
|
1104
|
+
}
|
|
1105
|
+
const driver = this.createDriver(node, turnId);
|
|
1106
|
+
ctx.driver = driver;
|
|
1107
|
+
driver.onEvent((event) => this._onDriverEvent(ctx, event));
|
|
1108
|
+
const { sessionId, sessionFile } = await driver.start();
|
|
1109
|
+
ctx.sessionId = sessionId;
|
|
1110
|
+
this.journal.setNodeSession(node.node_id, { sessionId, sessionFile });
|
|
1111
|
+
if (driver.pid) this.journal.setTurnChild(turnId, { pid: driver.pid, pgid: driver.pgid });
|
|
1112
|
+
await driver.prompt(prompt);
|
|
1113
|
+
await this._waitForSettle(ctx);
|
|
1114
|
+
await this._settleTurn(node, ctx);
|
|
1115
|
+
} catch (error) {
|
|
1116
|
+
ctx.error = error.message;
|
|
1117
|
+
if (ctx.turnId) {
|
|
1118
|
+
this.journal.markUnknownEffects(ctx.turnId);
|
|
1119
|
+
this.journal.endTurn(ctx.turnId, { state: "failed", ...this._usage(ctx) });
|
|
1120
|
+
// The prompt went out and the turn did not finish it, so its messages
|
|
1121
|
+
// are queued again rather than left in a `delivered` row nothing will
|
|
1122
|
+
// ever read.
|
|
1123
|
+
this.journal.releaseMessages([...ctx.carried, ...ctx.steeredIds]);
|
|
1124
|
+
this.journal.bumpAttempt(node.node_id);
|
|
1125
|
+
}
|
|
1126
|
+
this.journal.setNodeState(node.node_id, "failed", { done_reason: `turn failed: ${error.message}` });
|
|
1127
|
+
this.log(`fleet: node ${node.node_id} turn failed: ${error.message}`);
|
|
1128
|
+
} finally {
|
|
1129
|
+
if (ctx.driver) await ctx.driver.stop().catch(() => {});
|
|
1130
|
+
// After the child is gone, so stderr written on the way out is included.
|
|
1131
|
+
const report = childStderrReport(ctx.driver?.stderr);
|
|
1132
|
+
if (report) {
|
|
1133
|
+
this.log(
|
|
1134
|
+
`fleet: node ${node.node_id} turn ${ctx.turnId ?? "-"} child stderr ${report.chars} chars: ${report.shown}`,
|
|
1135
|
+
);
|
|
1136
|
+
}
|
|
1137
|
+
this.journal.releaseLease(node.node_id, this.epoch);
|
|
1138
|
+
this.active.delete(node.node_id);
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
_charter(node) {
|
|
1143
|
+
try {
|
|
1144
|
+
return readFileSync(node.charter_path, "utf8");
|
|
1145
|
+
} catch (error) {
|
|
1146
|
+
return `(charter unreadable at ${node.charter_path}: ${error.code ?? error.message})`;
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
_usage(ctx) {
|
|
1151
|
+
return {
|
|
1152
|
+
tokensIn: ctx.usage.input,
|
|
1153
|
+
tokensOut: ctx.usage.output,
|
|
1154
|
+
spendUsd: 0,
|
|
1155
|
+
};
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
_onDriverEvent(ctx, event) {
|
|
1159
|
+
if (!ctx.turnId) return;
|
|
1160
|
+
switch (event.type) {
|
|
1161
|
+
case "tool_start":
|
|
1162
|
+
ctx.effectSeq += 1;
|
|
1163
|
+
this.journal.effectAttempted({
|
|
1164
|
+
turnId: ctx.turnId,
|
|
1165
|
+
nodeId: ctx.nodeId,
|
|
1166
|
+
toolCallId: event.toolCallId,
|
|
1167
|
+
toolName: event.toolName,
|
|
1168
|
+
args: event.args,
|
|
1169
|
+
replay: replayPolicy(event.toolName),
|
|
1170
|
+
seq: ctx.effectSeq,
|
|
1171
|
+
});
|
|
1172
|
+
break;
|
|
1173
|
+
case "tool_end":
|
|
1174
|
+
ctx.toolEnds += 1;
|
|
1175
|
+
this.journal.effectSettled({
|
|
1176
|
+
turnId: ctx.turnId,
|
|
1177
|
+
toolCallId: event.toolCallId,
|
|
1178
|
+
result: event.result,
|
|
1179
|
+
isError: event.isError,
|
|
1180
|
+
});
|
|
1181
|
+
break;
|
|
1182
|
+
case "model_request":
|
|
1183
|
+
ctx.requestSeq = this.journal.recordModelRequest(ctx.turnId, {
|
|
1184
|
+
messageCount: event.messageCount,
|
|
1185
|
+
toolCount: event.toolCount,
|
|
1186
|
+
});
|
|
1187
|
+
break;
|
|
1188
|
+
case "message_end":
|
|
1189
|
+
if (event.role === "assistant") {
|
|
1190
|
+
ctx.lastAssistantText = assistantText(event.message);
|
|
1191
|
+
const usage = event.message?.usage;
|
|
1192
|
+
if (usage) {
|
|
1193
|
+
ctx.usage.input += Number(usage.input ?? 0);
|
|
1194
|
+
ctx.usage.output += Number(usage.output ?? 0);
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
break;
|
|
1198
|
+
case "final_text":
|
|
1199
|
+
ctx.finalText = event.text;
|
|
1200
|
+
ctx.settled = true;
|
|
1201
|
+
ctx.settleResolve?.();
|
|
1202
|
+
break;
|
|
1203
|
+
case "steer_delivered": {
|
|
1204
|
+
// The only delivery this fleet can testify to: the child ran the text it
|
|
1205
|
+
// was given, in this turn. Rows are matched by the body the child
|
|
1206
|
+
// carried, so a delivery cannot be charged to a message that was never
|
|
1207
|
+
// run. Every message is recorded before the steer is requested, which
|
|
1208
|
+
// is what makes an event that arrives before the ack findable.
|
|
1209
|
+
const waiting = this.journal.pendingMessages(ctx.nodeId);
|
|
1210
|
+
if (event.text == null) {
|
|
1211
|
+
// A driver that cannot name the body leaves the oldest waiting row as
|
|
1212
|
+
// the only honest candidate; saying so is better than a silent guess.
|
|
1213
|
+
this.log(`fleet: node ${ctx.nodeId} reported a steer with no body; the oldest waiting message is charged with it`);
|
|
1214
|
+
}
|
|
1215
|
+
const matched = event.text == null ? waiting.slice(0, 1) : waiting.filter((message) => message.body === event.text);
|
|
1216
|
+
if (event.text != null && matched.length === 0) {
|
|
1217
|
+
this.log(`fleet: node ${ctx.nodeId} ran a steered message no queued row carried, so no row is marked delivered`);
|
|
1218
|
+
}
|
|
1219
|
+
for (const message of matched) {
|
|
1220
|
+
this.journal.markMessageDelivered(message.msg_id, {
|
|
1221
|
+
tier: "A",
|
|
1222
|
+
turnId: ctx.turnId,
|
|
1223
|
+
afterEffectSeq: event.afterEffectSeq,
|
|
1224
|
+
beforeRequestSeq: ctx.requestSeq,
|
|
1225
|
+
});
|
|
1226
|
+
if (!ctx.steeredIds.includes(message.msg_id)) ctx.steeredIds.push(message.msg_id);
|
|
1227
|
+
}
|
|
1228
|
+
break;
|
|
1229
|
+
}
|
|
1230
|
+
case "settled":
|
|
1231
|
+
ctx.finalText = ctx.finalText || ctx.lastAssistantText;
|
|
1232
|
+
ctx.settled = true;
|
|
1233
|
+
ctx.settleResolve?.();
|
|
1234
|
+
break;
|
|
1235
|
+
default:
|
|
1236
|
+
break;
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
async _waitForSettle(ctx) {
|
|
1241
|
+
const deadline = Date.now() + this.options.turnTimeoutMs;
|
|
1242
|
+
while (!ctx.settled) {
|
|
1243
|
+
if (Date.now() > deadline) throw new Error(`turn exceeded ${this.options.turnTimeoutMs} ms`);
|
|
1244
|
+
if (ctx.driver?.exit) {
|
|
1245
|
+
const exited = await Promise.race([ctx.driver.exit.then(() => true), new Promise((resolveWait) => setTimeout(() => resolveWait(false), 50))]);
|
|
1246
|
+
if (exited && !ctx.settled) throw new Error("driver exited before settling");
|
|
1247
|
+
}
|
|
1248
|
+
await new Promise((resolveWait) => setTimeout(resolveWait, 50));
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
async _settleTurn(node, ctx) {
|
|
1253
|
+
const text = ctx.finalText || ctx.lastAssistantText;
|
|
1254
|
+
const ask = parseAsk(text);
|
|
1255
|
+
let turnState = "settled";
|
|
1256
|
+
let nodeState = "settled";
|
|
1257
|
+
let doneReason = null;
|
|
1258
|
+
if (ask) {
|
|
1259
|
+
const token = `ask-${randomBytes(6).toString("hex")}`;
|
|
1260
|
+
this.journal.recordAsk({
|
|
1261
|
+
token,
|
|
1262
|
+
nodeId: node.node_id,
|
|
1263
|
+
turnId: ctx.turnId,
|
|
1264
|
+
priority: ask.priority,
|
|
1265
|
+
question: ask.question,
|
|
1266
|
+
defaultAnswer: ask.defaultAnswer,
|
|
1267
|
+
deadlineAt: Date.now() + this.options.askTtlMs,
|
|
1268
|
+
});
|
|
1269
|
+
turnState = "parked";
|
|
1270
|
+
nodeState = "waiting_on_human";
|
|
1271
|
+
doneReason = `parked on ${token}`;
|
|
1272
|
+
this.log(`fleet: node ${node.node_id} parked on ${token}: ${ask.question}`);
|
|
1273
|
+
} else if (isDone(text)) {
|
|
1274
|
+
turnState = "done";
|
|
1275
|
+
nodeState = "done";
|
|
1276
|
+
doneReason = "charter reported done";
|
|
1277
|
+
}
|
|
1278
|
+
this.journal.endTurn(ctx.turnId, { state: turnState, ...this._usage(ctx), sessionId: ctx.sessionId });
|
|
1279
|
+
this.journal.addNodeUsage(node.node_id, {
|
|
1280
|
+
tokensIn: this._usage(ctx).tokensIn,
|
|
1281
|
+
tokensOut: this._usage(ctx).tokensOut,
|
|
1282
|
+
projectRoot: node.project_root,
|
|
1283
|
+
});
|
|
1284
|
+
// Only what this turn's prompt carried is retired. A message whose body was
|
|
1285
|
+
// not in the prompt keeps the state that makes the next turn pick it up,
|
|
1286
|
+
// which is what makes `consumed` a record of what a model read.
|
|
1287
|
+
this.journal.consumeMessages([...ctx.carried, ...ctx.steeredIds]);
|
|
1288
|
+
this.journal.setNodeState(node.node_id, nodeState, doneReason ? { done_reason: doneReason } : {});
|
|
1289
|
+
this.log(
|
|
1290
|
+
`fleet: node ${node.node_id} turn ${ctx.turnId} ${turnState} effects=${this.journal.effects(ctx.turnId).length} requests=${ctx.requestSeq}`,
|
|
1291
|
+
);
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
// ── control channel ──────────────────────────────────────────────────────
|
|
1295
|
+
|
|
1296
|
+
socketPath() {
|
|
1297
|
+
return controlSocketPath(this.root);
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
/**
|
|
1301
|
+
* The live responder on `path`, or null when nothing answered: no file, no
|
|
1302
|
+
* listener behind the file, or a reply this process wrote itself.
|
|
1303
|
+
*/
|
|
1304
|
+
async _responderOn(path) {
|
|
1305
|
+
if (!existsSync(path)) return null;
|
|
1306
|
+
const reply = await requestOnControlSocket(path, { op: "ping" }, { timeoutMs: CONTROL_PROBE_MS });
|
|
1307
|
+
const data = reply?.ok ? reply.data : null;
|
|
1308
|
+
if (!data || data.pid == null || data.pid === process.pid) return null;
|
|
1309
|
+
return data;
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
async _listen() {
|
|
1313
|
+
const path = this.socketPath();
|
|
1314
|
+
const incumbent = await this._responderOn(path);
|
|
1315
|
+
if (incumbent) {
|
|
1316
|
+
// A live responder with another pid is the owner of this root's channel.
|
|
1317
|
+
// Binding over it would leave a supervisor that runs and works while
|
|
1318
|
+
// nothing can reach it, and nothing would say so for the rest of its
|
|
1319
|
+
// life, so this process refuses instead of clobbering. The journal row
|
|
1320
|
+
// follows: `live` for a process that never served would be the same lie.
|
|
1321
|
+
this.journal.setSupervisorState(process.pid, "refused");
|
|
1322
|
+
throw new Error(
|
|
1323
|
+
`control channel ${path} is served by supervisor pid ${incumbent.pid} (epoch ${incumbent.epoch}); refusing to bind over it`,
|
|
1324
|
+
);
|
|
1325
|
+
}
|
|
1326
|
+
if (existsSync(path)) {
|
|
1327
|
+
// A socket file outlives the process that bound it, so a path nobody
|
|
1328
|
+
// answers is a departed binder's leavings. Reclaiming it is what lets the
|
|
1329
|
+
// channel come back after a crash; refusing to would leave a root that no
|
|
1330
|
+
// supervisor can ever serve again.
|
|
1331
|
+
this.log(`fleet: no answer on ${path}; reclaiming the path`);
|
|
1332
|
+
rmSync(path, { force: true });
|
|
1333
|
+
}
|
|
1334
|
+
this.server = createServer((socket) => {
|
|
1335
|
+
let buffer = "";
|
|
1336
|
+
socket.on("data", (chunk) => {
|
|
1337
|
+
buffer += chunk.toString("utf8");
|
|
1338
|
+
let index;
|
|
1339
|
+
while ((index = buffer.indexOf("\n")) >= 0) {
|
|
1340
|
+
const line = buffer.slice(0, index);
|
|
1341
|
+
buffer = buffer.slice(index + 1);
|
|
1342
|
+
if (!line.trim()) continue;
|
|
1343
|
+
this._handleControl(line)
|
|
1344
|
+
.then((reply) => socket.write(JSON.stringify(reply) + "\n"))
|
|
1345
|
+
.catch((error) => socket.write(JSON.stringify({ ok: false, error: error.message }) + "\n"));
|
|
1346
|
+
}
|
|
1347
|
+
});
|
|
1348
|
+
socket.on("error", () => socket.destroy());
|
|
1349
|
+
});
|
|
1350
|
+
await new Promise((resolveListen, rejectListen) => {
|
|
1351
|
+
this.server.once("error", rejectListen);
|
|
1352
|
+
this.server.listen(path, () => {
|
|
1353
|
+
this.server.removeListener("error", rejectListen);
|
|
1354
|
+
chmodSync(path, 0o600);
|
|
1355
|
+
resolveListen();
|
|
1356
|
+
});
|
|
1357
|
+
});
|
|
1358
|
+
const identity = fileIdentity(path);
|
|
1359
|
+
this.boundSocket = identity ? { path, dev: identity.dev, ino: identity.ino } : null;
|
|
1360
|
+
this.journal.setMeta("control_socket", path);
|
|
1361
|
+
return path;
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
async _handleControl(line) {
|
|
1365
|
+
const request = JSON.parse(line);
|
|
1366
|
+
switch (request.op) {
|
|
1367
|
+
case "ping":
|
|
1368
|
+
return { ok: true, data: { pid: process.pid, epoch: this.epoch, active: [...this.active.keys()] } };
|
|
1369
|
+
case "status":
|
|
1370
|
+
return { ok: true, data: buildStatus(this.journal, this) };
|
|
1371
|
+
case "inject": {
|
|
1372
|
+
const node = this.journal.getNode(request.node);
|
|
1373
|
+
if (!node) return { ok: false, error: `no such node: ${request.node}` };
|
|
1374
|
+
// A message for a node no future turn will drive cannot be delivered,
|
|
1375
|
+
// and recording it would be the command claiming it can. Refusing here
|
|
1376
|
+
// is what turns that into an answer the operator sees now.
|
|
1377
|
+
const retired = undeliverableReason(node, this.options.maxAttempts);
|
|
1378
|
+
if (retired) return { ok: false, error: `cannot queue a message for ${node.node_id}: ${retired}; no turn will ever carry it` };
|
|
1379
|
+
const msgId = request.msgId ?? `m-${randomBytes(6).toString("hex")}`;
|
|
1380
|
+
let kind = "user";
|
|
1381
|
+
if (request.token) {
|
|
1382
|
+
const ask = this.journal.getAsk(request.token);
|
|
1383
|
+
if (!ask || ask.state !== "open") return { ok: false, error: `no open ask: ${request.token}` };
|
|
1384
|
+
this.journal.resolveAsk(request.token, { answerText: request.text, answerSource: "fleet-inject" });
|
|
1385
|
+
kind = "answer";
|
|
1386
|
+
if (node.state === "waiting_on_human") this.journal.setNodeState(node.node_id, "settled");
|
|
1387
|
+
}
|
|
1388
|
+
const message = this.journal.addMessage({
|
|
1389
|
+
msgId,
|
|
1390
|
+
nodeId: node.node_id,
|
|
1391
|
+
kind,
|
|
1392
|
+
body: request.text,
|
|
1393
|
+
token: request.token,
|
|
1394
|
+
});
|
|
1395
|
+
const ctx = this.active.get(node.node_id);
|
|
1396
|
+
if (ctx?.driver && !ctx.settled && ctx.turnId) {
|
|
1397
|
+
try {
|
|
1398
|
+
const ack = await ctx.driver.steer(request.text);
|
|
1399
|
+
const disposition = ack?.data?.disposition ?? ack?.disposition ?? "queued";
|
|
1400
|
+
// The ack is the child's queue talking, not a delivery. pi answers
|
|
1401
|
+
// `steer` once the text is queued and hands it to the model later,
|
|
1402
|
+
// or never, so the row stays `recorded` until a `steer_delivered`
|
|
1403
|
+
// event matches its body. That is what keeps a steer the turn never
|
|
1404
|
+
// ran: the next turn carries it, instead of the row being retired
|
|
1405
|
+
// as read by a model that never saw it.
|
|
1406
|
+
return {
|
|
1407
|
+
ok: true,
|
|
1408
|
+
data: { msgId, steered: true, disposition, turnId: ctx.turnId, effectSeq: ctx.toolEnds, message },
|
|
1409
|
+
};
|
|
1410
|
+
} catch (error) {
|
|
1411
|
+
this.log(`fleet: steer for ${node.node_id} refused: ${error.message}`);
|
|
1412
|
+
return { ok: true, data: { msgId, tier: "B", reason: `steer refused: ${error.message}`, message } };
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
// Tier B is journalled and prepended to the next drive's prompt. It is
|
|
1416
|
+
// deliberately not pi's `nextTurn` delivery: that waits for the next
|
|
1417
|
+
// user prompt and can land after the topic has moved, which is drift.
|
|
1418
|
+
return { ok: true, data: { msgId, tier: "B", reason: "no turn in flight", message } };
|
|
1419
|
+
}
|
|
1420
|
+
case "stop":
|
|
1421
|
+
setTimeout(() => {
|
|
1422
|
+
this.stop().catch((error) => this.log(`fleet: stop failed: ${error.message}`));
|
|
1423
|
+
}, 10);
|
|
1424
|
+
return { ok: true, data: { stopping: true } };
|
|
1425
|
+
default:
|
|
1426
|
+
return { ok: false, error: `unknown op: ${request.op}` };
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
/**
|
|
1431
|
+
* Whether the path still resolves to the file this process bound, and why
|
|
1432
|
+
* not when it does not. An unreadable path is not an ownership claim either:
|
|
1433
|
+
* a claim this process cannot check is a claim it does not have.
|
|
1434
|
+
*/
|
|
1435
|
+
_socketPathState(path) {
|
|
1436
|
+
const mine = this.boundSocket;
|
|
1437
|
+
if (!mine || mine.path !== path || mine.ino == null) return { owned: false, why: "this process never bound it" };
|
|
1438
|
+
let current;
|
|
1439
|
+
try {
|
|
1440
|
+
current = fileIdentity(path);
|
|
1441
|
+
} catch (error) {
|
|
1442
|
+
return { owned: false, why: `it cannot be read (${error.code ?? error.message})` };
|
|
1443
|
+
}
|
|
1444
|
+
if (!current) return { owned: false, why: "the file is gone" };
|
|
1445
|
+
if (current.dev !== mine.dev || current.ino !== mine.ino) return { owned: false, why: "another process bound it" };
|
|
1446
|
+
return { owned: true, why: "this process bound it and nothing has replaced it" };
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
/**
|
|
1450
|
+
* Give up the control path, if it is still this process's to give up.
|
|
1451
|
+
*
|
|
1452
|
+
* Closing a unix-socket server removes the path it bound, whatever is there
|
|
1453
|
+
* now: `net.Server.close()` unlinks by name, so a server closed after another
|
|
1454
|
+
* process rebound the path deletes that process's socket and leaves a live
|
|
1455
|
+
* supervisor unreachable. The close is therefore the deletion that has to be
|
|
1456
|
+
* gated, not only the unlink after it. When the path is no longer this
|
|
1457
|
+
* process's, the listener is abandoned instead: it is unreachable either way,
|
|
1458
|
+
* and it keeps the event loop referenced, so the only way out of this process
|
|
1459
|
+
* is an explicit `process.exit()` - which does not run the close that would
|
|
1460
|
+
* delete the file that replaced ours.
|
|
1461
|
+
*/
|
|
1462
|
+
async _releaseSocketPath() {
|
|
1463
|
+
const path = this.socketPath();
|
|
1464
|
+
const state = this._socketPathState(path);
|
|
1465
|
+
if (!state.owned) {
|
|
1466
|
+
this.log(`fleet: leaving control socket ${path} alone: ${state.why}`);
|
|
1467
|
+
return;
|
|
1468
|
+
}
|
|
1469
|
+
await new Promise((resolveClose) => this.server.close(resolveClose));
|
|
1470
|
+
this.server = null;
|
|
1471
|
+
// The close above removed the path on this platform; the unlink is for a
|
|
1472
|
+
// runtime whose close leaves it behind, and it is safe only because the
|
|
1473
|
+
// identity was confirmed first.
|
|
1474
|
+
rmSync(path, { force: true });
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
async stop() {
|
|
1478
|
+
if (this.stopping) return;
|
|
1479
|
+
this.stopping = true;
|
|
1480
|
+
if (this.timer) clearTimeout(this.timer);
|
|
1481
|
+
if (this.server) await this._releaseSocketPath();
|
|
1482
|
+
for (const [nodeId, ctx] of [...this.active]) {
|
|
1483
|
+
ctx.driver?.killGroup?.("SIGKILL");
|
|
1484
|
+
if (ctx.turnId) {
|
|
1485
|
+
this.journal.markUnknownEffects(ctx.turnId);
|
|
1486
|
+
this.journal.endTurn(ctx.turnId, { state: "interrupted", ...this._usage(ctx) });
|
|
1487
|
+
// The same rule as a failed turn: the prompt went out and the turn did
|
|
1488
|
+
// not settle, so its messages are queued for the next drive rather than
|
|
1489
|
+
// left in a `delivered` row nothing will read again.
|
|
1490
|
+
this.journal.releaseMessages([...ctx.carried, ...ctx.steeredIds]);
|
|
1491
|
+
}
|
|
1492
|
+
this.journal.setNodeState(nodeId, "settled");
|
|
1493
|
+
this.journal.releaseLease(nodeId, this.epoch);
|
|
1494
|
+
this.active.delete(nodeId);
|
|
1495
|
+
}
|
|
1496
|
+
this.journal.setSupervisorState(process.pid, "stopped");
|
|
1497
|
+
this.journal.setMeta("fences", JSON.stringify(this.fences));
|
|
1498
|
+
this.journal.close();
|
|
1499
|
+
// Every stop ends the process the entry point is waiting on: the idle exit,
|
|
1500
|
+
// the control channel's `stop`, and a signal all leave the same way. Stop
|
|
1501
|
+
// is the exit, so nothing has to guess whether the process should still be
|
|
1502
|
+
// up.
|
|
1503
|
+
this.doneResolve(true);
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
/**
|
|
1508
|
+
* One readable snapshot of the fleet, assembled from the journal plus `ps`.
|
|
1509
|
+
* Shared by `fleet status` and the control channel so both print the same
|
|
1510
|
+
* numbers instead of two nearly-equal views.
|
|
1511
|
+
*/
|
|
1512
|
+
export function buildStatus(journal, supervisor = null) {
|
|
1513
|
+
const nodes = journal.listNodes().map((node) => {
|
|
1514
|
+
const turns = journal.turnsFor(node.node_id);
|
|
1515
|
+
const last = turns.at(-1) ?? null;
|
|
1516
|
+
const effects = last ? journal.effects(last.turn_id) : [];
|
|
1517
|
+
const counts = effects.reduce((acc, effect) => {
|
|
1518
|
+
acc[effect.state] = (acc[effect.state] ?? 0) + 1;
|
|
1519
|
+
return acc;
|
|
1520
|
+
}, {});
|
|
1521
|
+
return {
|
|
1522
|
+
nodeId: node.node_id,
|
|
1523
|
+
parentId: node.parent_id,
|
|
1524
|
+
state: node.state,
|
|
1525
|
+
attempts: node.attempts,
|
|
1526
|
+
turns: node.turn_count,
|
|
1527
|
+
sessionId: node.session_id,
|
|
1528
|
+
sessionFile: node.session_file,
|
|
1529
|
+
doneReason: node.done_reason,
|
|
1530
|
+
lastTurn: last ? { turnId: last.turn_id, kind: last.kind, state: last.state, effects: counts } : null,
|
|
1531
|
+
openAsks: journal.openAsks(node.node_id).map((ask) => ask.token),
|
|
1532
|
+
lease: journal.liveLease(node.node_id)?.epoch ?? null,
|
|
1533
|
+
pending: journal.pendingMessages(node.node_id).length,
|
|
1534
|
+
// Queued messages that no turn will ever carry: a count an operator can
|
|
1535
|
+
// read without opening SQLite.
|
|
1536
|
+
undeliverable: journal.messages(node.node_id).filter((message) => message.state === "undeliverable").length,
|
|
1537
|
+
};
|
|
1538
|
+
});
|
|
1539
|
+
const liveSupervisors = journal.supervisors().filter((row) => row.state === "live" && pidIsAlive(row.pid));
|
|
1540
|
+
const childPids = journal
|
|
1541
|
+
.allOpenTurns()
|
|
1542
|
+
.map((turn) => turn.child_pid)
|
|
1543
|
+
.filter(Boolean);
|
|
1544
|
+
const resident = residentMb([...liveSupervisors.map((row) => row.pid), ...childPids]);
|
|
1545
|
+
const meta = journal.meta();
|
|
1546
|
+
return {
|
|
1547
|
+
store: journal.path,
|
|
1548
|
+
schemaVersion: SCHEMA_VERSION,
|
|
1549
|
+
meta,
|
|
1550
|
+
epoch: supervisor?.epoch ?? null,
|
|
1551
|
+
cap: supervisor?.options.cap ?? null,
|
|
1552
|
+
active: supervisor ? [...supervisor.active.keys()] : [],
|
|
1553
|
+
nodes,
|
|
1554
|
+
leases: journal.liveLeases().map((lease) => ({ nodeId: lease.node_id, epoch: lease.epoch, pid: lease.owner_pid })),
|
|
1555
|
+
childPids,
|
|
1556
|
+
supervisors: journal.supervisors().map((row) => ({ pid: row.pid, epoch: row.epoch, state: row.state, driver: row.driver })),
|
|
1557
|
+
asks: journal.openAsks().map((ask) => ({
|
|
1558
|
+
token: ask.token,
|
|
1559
|
+
nodeId: ask.node_id,
|
|
1560
|
+
priority: ask.priority,
|
|
1561
|
+
question: ask.question,
|
|
1562
|
+
defaultAnswer: ask.default_answer,
|
|
1563
|
+
deadlineAt: ask.deadline_at,
|
|
1564
|
+
})),
|
|
1565
|
+
fences: JSON.parse(meta.fences ?? "[]"),
|
|
1566
|
+
// The store's own account of a pass it would not take: a maintenance pass
|
|
1567
|
+
// that could not finish, and one that had to wait for the lock. The journal
|
|
1568
|
+
// is the first place to look and the file beside it the second, because the
|
|
1569
|
+
// deferral is written while the journal cannot be - a reader that stops at
|
|
1570
|
+
// the journal would see nothing exactly when the store is wedged.
|
|
1571
|
+
drain: {
|
|
1572
|
+
deferred:
|
|
1573
|
+
jsonMeta(meta, "store_deferred") ?? jsonMeta(meta, "drain_deferred") ?? deferralFromDisk(journal.root),
|
|
1574
|
+
waited: jsonMeta(meta, "drain_contention"),
|
|
1575
|
+
},
|
|
1576
|
+
resident,
|
|
1577
|
+
};
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
// ── entry point ──────────────────────────────────────────────────────────────
|
|
1581
|
+
|
|
1582
|
+
function parseArgs(argv) {
|
|
1583
|
+
const args = { _: [] };
|
|
1584
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
1585
|
+
const token = argv[i];
|
|
1586
|
+
if (!token.startsWith("--")) {
|
|
1587
|
+
args._.push(token);
|
|
1588
|
+
continue;
|
|
1589
|
+
}
|
|
1590
|
+
const [key, inline] = token.slice(2).split("=");
|
|
1591
|
+
if (inline != null) {
|
|
1592
|
+
args[key] = inline;
|
|
1593
|
+
continue;
|
|
1594
|
+
}
|
|
1595
|
+
const next = argv[i + 1];
|
|
1596
|
+
if (next == null || next.startsWith("--")) {
|
|
1597
|
+
args[key] = true;
|
|
1598
|
+
continue;
|
|
1599
|
+
}
|
|
1600
|
+
args[key] = next;
|
|
1601
|
+
i += 1;
|
|
1602
|
+
}
|
|
1603
|
+
return args;
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
1607
|
+
const args = parseArgs(process.argv.slice(2));
|
|
1608
|
+
if (!args.root) {
|
|
1609
|
+
console.error(
|
|
1610
|
+
"usage: supervisor.mjs --root <fleet-root> [--cap N] [--driver rpc|scripted] [--naive] [--idle-exit-ms N]\n" +
|
|
1611
|
+
"\n" +
|
|
1612
|
+
"Each child loads the duplication guard. FLEET_GUARD=off in this process's environment disables it for every child it spawns.",
|
|
1613
|
+
);
|
|
1614
|
+
process.exit(2);
|
|
1615
|
+
}
|
|
1616
|
+
const supervisor = new Supervisor({
|
|
1617
|
+
root: String(args.root),
|
|
1618
|
+
cap: args.cap ? Number(args.cap) : undefined,
|
|
1619
|
+
driver: args.driver ? String(args.driver) : undefined,
|
|
1620
|
+
model: args.model ? String(args.model) : undefined,
|
|
1621
|
+
naive: Boolean(args.naive),
|
|
1622
|
+
turnTimeoutMs: args["turn-timeout-ms"] ? Number(args["turn-timeout-ms"]) : undefined,
|
|
1623
|
+
askTtlMs: args["ask-ttl-ms"] ? Number(args["ask-ttl-ms"]) : undefined,
|
|
1624
|
+
idleExitMs: args["idle-exit-ms"] ? Number(args["idle-exit-ms"]) : undefined,
|
|
1625
|
+
extensions: args.extension ? [].concat(args.extension) : [],
|
|
1626
|
+
fullDiscovery: Boolean(args["full-discovery"]),
|
|
1627
|
+
piVersion: args["pi-version"] ? String(args["pi-version"]) : undefined,
|
|
1628
|
+
});
|
|
1629
|
+
try {
|
|
1630
|
+
await supervisor.start();
|
|
1631
|
+
} catch (error) {
|
|
1632
|
+
// A start that cannot happen is an operator's answer, not a stack trace:
|
|
1633
|
+
// the loudest thing this process can do with a refusal is print it and go.
|
|
1634
|
+
console.error(`fleet: ${error.message}`);
|
|
1635
|
+
process.exit(1);
|
|
1636
|
+
}
|
|
1637
|
+
const shutdown = async (signal) => {
|
|
1638
|
+
console.log(`fleet: ${signal} received, stopping`);
|
|
1639
|
+
await supervisor.stop();
|
|
1640
|
+
process.exit(0);
|
|
1641
|
+
};
|
|
1642
|
+
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
1643
|
+
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
1644
|
+
// The tick timer is what held the process up; it stops rescheduling itself
|
|
1645
|
+
// when the fleet has nothing left to do. `done` resolves after that stop has
|
|
1646
|
+
// finished, so this is an exit and not an intention to exit.
|
|
1647
|
+
const cleanExit = await supervisor.done;
|
|
1648
|
+
process.exit(cleanExit ? 0 : 1);
|
|
1649
|
+
}
|