pi-braid 0.41.0 → 0.41.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build-info.json +4 -4
- package/docs/FLEET.md +59 -3
- package/docs/LOCAL_DEV.md +1 -1
- package/docs/SETUP.md +1 -1
- package/package.json +2 -2
- package/scripts/fleet/cli.mjs +58 -17
- package/scripts/fleet/journal.mjs +198 -14
- package/scripts/fleet/supervisor.mjs +224 -69
- package/scripts/fleet/tool-guard.mjs +1 -2
package/build-info.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-braid",
|
|
3
|
-
"version": "0.41.
|
|
4
|
-
"sha": "
|
|
5
|
-
"shaShort": "
|
|
3
|
+
"version": "0.41.2",
|
|
4
|
+
"sha": "b46aa4f474e84f5394ddcbc4feb549181b0c6c8e",
|
|
5
|
+
"shaShort": "b46aa4f",
|
|
6
6
|
"branch": "detached",
|
|
7
7
|
"dirty": false,
|
|
8
|
-
"packedAt": "2026-10-
|
|
8
|
+
"packedAt": "2026-10-25T10:56:25.509Z"
|
|
9
9
|
}
|
package/docs/FLEET.md
CHANGED
|
@@ -53,10 +53,33 @@ used, and the inbox row records it:
|
|
|
53
53
|
- **C** - no supervisor was running; the message waits in the journal for the
|
|
54
54
|
next supervisor that drives the node.
|
|
55
55
|
|
|
56
|
+
A turn retires a message only when the section the turn appends carries it: the
|
|
57
|
+
supervisor takes the ids from that section, and checks them against the prompt
|
|
58
|
+
before it writes anything. A charter that repeats the operator's instruction
|
|
59
|
+
word for word is therefore not read as a delivery - the coincidence is the one
|
|
60
|
+
thing this cannot be - and the message keeps waiting for a turn that really
|
|
61
|
+
carries it.
|
|
62
|
+
|
|
56
63
|
When a turn cannot carry a message at all (the node is done, or failed at its
|
|
57
64
|
attempt bound), the supervisor stamps the row `undeliverable` and `status`
|
|
58
65
|
counts it, rather than leaving a promise nobody keeps.
|
|
59
66
|
|
|
67
|
+
## Leases
|
|
68
|
+
|
|
69
|
+
A node is leased to one supervisor epoch at a time, so two supervisors never
|
|
70
|
+
drive the same node. The lease is held for as long as its owner answers for it:
|
|
71
|
+
exclusivity is the owner's liveness, not a clock, and reconciliation ends a
|
|
72
|
+
predecessor's lease only once that owner is gone. A supervisor whose turn runs
|
|
73
|
+
long - a tick under write contention can - keeps its node, which is the property
|
|
74
|
+
a wall-clock expiry would have broken.
|
|
75
|
+
|
|
76
|
+
The lease row therefore carries an `expires_at` column that means nothing. It is
|
|
77
|
+
kept because the build that shipped schema version 1 writes it by name, so a
|
|
78
|
+
store without it is a store that build cannot open; this build neither reads nor
|
|
79
|
+
writes it, and a store from the previous version has the column relaxed to
|
|
80
|
+
nullable with its stale values cleared the first time it is opened here. Read no
|
|
81
|
+
expiry into it.
|
|
82
|
+
|
|
60
83
|
## Asks and deadlines
|
|
61
84
|
|
|
62
85
|
An ask is a question a human can answer, with a deadline and a default. When the
|
|
@@ -67,9 +90,13 @@ neither, and the drain is what a restart runs to recover.
|
|
|
67
90
|
|
|
68
91
|
Two supervisors may drain the same root: the claim is a conditional update, so
|
|
69
92
|
each overdue ask is answered exactly once and queued exactly once. If the store's
|
|
70
|
-
write lock is held by someone else, the drain waits and retries; a
|
|
71
|
-
cannot finish is
|
|
72
|
-
the error) and only reported, never thrown at the
|
|
93
|
+
write lock is held by someone else, the drain waits and retries; a pass that
|
|
94
|
+
cannot finish is recorded as `store_deferred` (with the pass's name, the
|
|
95
|
+
attempts, the wait and the error) and only reported, never thrown at the
|
|
96
|
+
process. Because the store is exactly what cannot take that record, it is also
|
|
97
|
+
written to `<fleet-root>/store-deferred.json` while the lock is held and moved
|
|
98
|
+
into the journal by the first pass that gets through, so a supervisor killed with
|
|
99
|
+
the fact in hand still leaves it where `status` reads it. `status` prints both
|
|
73
100
|
facts when they exist.
|
|
74
101
|
|
|
75
102
|
## The journal
|
|
@@ -80,6 +107,35 @@ failure each column prevents are documented where they are defined -
|
|
|
80
107
|
[`../scripts/fleet/journal.mjs`](../scripts/fleet/journal.mjs) - and this page
|
|
81
108
|
does not restate them. Read them there before writing to a journal by hand.
|
|
82
109
|
|
|
110
|
+
The store records the schema version that wrote it. Opening a store that records
|
|
111
|
+
a version above the running build's is refused with one sentence naming both
|
|
112
|
+
numbers, rather than opened and left to fail inside a statement: that store's
|
|
113
|
+
shape belongs to a newer build, and the way forward is that build or a new root.
|
|
114
|
+
Opening an older store migrates it and records the new version, and the previous
|
|
115
|
+
build keeps its access: a rollback can open and write a store this build created,
|
|
116
|
+
because the column it needs was kept rather than dropped.
|
|
117
|
+
|
|
118
|
+
## Node versions
|
|
119
|
+
|
|
120
|
+
The journal is reached through one module,
|
|
121
|
+
[`../scripts/fleet/journal.mjs`](../scripts/fleet/journal.mjs), which is also
|
|
122
|
+
where the fleet answers what importing SQLite costs. `node:sqlite` says it is
|
|
123
|
+
experimental on stderr, once per process, at the import: true of every 24 release
|
|
124
|
+
before 24.15, and no longer true from 24.15, where the module became a release
|
|
125
|
+
candidate and stopped saying it. The floor `engines` declares is 24.0.0, so the
|
|
126
|
+
answer is installed at that import - the warning withheld by its own type and
|
|
127
|
+
text, for the length of the import only, with the emitter restored in a `finally`
|
|
128
|
+
so a warning from any other module still reaches stderr. It is not done with
|
|
129
|
+
`--disable-warning` or `NODE_OPTIONS`, which remove the whole
|
|
130
|
+
`ExperimentalWarning` class for the process and hide those along with it, and it
|
|
131
|
+
cannot be done with a `warning` listener at all - Node's default printer is
|
|
132
|
+
registered by the runtime and prints whether or not another listener exists.
|
|
133
|
+
|
|
134
|
+
Both workflows run Node 24, the line `engines` declares, so the gate that reports
|
|
135
|
+
red and the gate that decides a release see the same runtime. The answer costs
|
|
136
|
+
nothing on 24.15 and above, and goes with its test once the whole supported line
|
|
137
|
+
is past it.
|
|
138
|
+
|
|
83
139
|
## Where the code is
|
|
84
140
|
|
|
85
141
|
| File | What it owns |
|
package/docs/LOCAL_DEV.md
CHANGED
|
@@ -11,7 +11,7 @@ For one-time setup, the full install-path catalog, and module toggling see
|
|
|
11
11
|
|
|
12
12
|
| Tool | Version | Verify |
|
|
13
13
|
| ---- | ------------- | -------------------------- |
|
|
14
|
-
| Node | ≥
|
|
14
|
+
| Node | ≥ 24.0 | `node --version` |
|
|
15
15
|
| npm | bundled | `npm --version` |
|
|
16
16
|
| git | any | `git --version` |
|
|
17
17
|
| pi | peer-resolved | `which pi && pi --version` |
|
package/docs/SETUP.md
CHANGED
|
@@ -26,7 +26,7 @@ Complete reference for installing braid, configuring pi to load it, enabling com
|
|
|
26
26
|
|
|
27
27
|
| Tool | Version | Verify |
|
|
28
28
|
| --------------- | ------------- | -------------------------- |
|
|
29
|
-
| Node.js | ≥
|
|
29
|
+
| Node.js | ≥ 24.0 | `node --version` |
|
|
30
30
|
| pi-coding-agent | peer-resolved | `which pi && pi --version` |
|
|
31
31
|
| npm | bundled | `npm --version` |
|
|
32
32
|
| git | any recent | `git --version` |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-braid",
|
|
3
|
-
"version": "0.41.
|
|
3
|
+
"version": "0.41.2",
|
|
4
4
|
"description": "Metacognitive extension suite for pi-coding-agent: enforcement, memory, reasoning, self-improvement, and structured cognitive signals.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -102,6 +102,6 @@
|
|
|
102
102
|
"author": "kifirkin",
|
|
103
103
|
"license": "MIT",
|
|
104
104
|
"engines": {
|
|
105
|
-
"node": ">=
|
|
105
|
+
"node": ">=24.0.0"
|
|
106
106
|
}
|
|
107
107
|
}
|
package/scripts/fleet/cli.mjs
CHANGED
|
@@ -40,8 +40,15 @@ import { closeSync, existsSync, mkdirSync, openSync, realpathSync } from "node:f
|
|
|
40
40
|
import { basename, dirname, join, resolve } from "node:path";
|
|
41
41
|
import { fileURLToPath } from "node:url";
|
|
42
42
|
|
|
43
|
-
import { openJournal, resolveFleetRoot } from "./journal.mjs";
|
|
44
|
-
import {
|
|
43
|
+
import { journalPath, openJournal, resolveFleetRoot } from "./journal.mjs";
|
|
44
|
+
import {
|
|
45
|
+
buildStatus,
|
|
46
|
+
controlSocketPath,
|
|
47
|
+
DEFAULT_ANSWER,
|
|
48
|
+
deferralFromDisk,
|
|
49
|
+
requestOnControlSocket,
|
|
50
|
+
undeliverableReason,
|
|
51
|
+
} from "./supervisor.mjs";
|
|
45
52
|
import { pidIsAlive } from "./procs.mjs";
|
|
46
53
|
|
|
47
54
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
@@ -229,42 +236,75 @@ async function cmdStatus(args) {
|
|
|
229
236
|
const live = await controlRequest(root, { op: "status" }, { timeoutMs: 1500 });
|
|
230
237
|
let status = live?.ok ? live.data : null;
|
|
231
238
|
let supervisor = "up";
|
|
239
|
+
let unavailable = null;
|
|
232
240
|
if (!status) {
|
|
233
241
|
supervisor = "down";
|
|
234
|
-
const journal = openJournal(root, {});
|
|
235
242
|
try {
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
243
|
+
const journal = openJournal(root, {});
|
|
244
|
+
try {
|
|
245
|
+
status = buildStatus(journal);
|
|
246
|
+
} finally {
|
|
247
|
+
journal.close();
|
|
248
|
+
}
|
|
249
|
+
} catch (error) {
|
|
250
|
+
// A store another process holds the write lock on for longer than this
|
|
251
|
+
// connection may wait cannot be read, and that is exactly when this
|
|
252
|
+
// command is run. The deferral beside the store needs no lock, so the
|
|
253
|
+
// reason the fleet stopped working survives the store being unreadable -
|
|
254
|
+
// which is the whole reason that fact is written there. The exit stays
|
|
255
|
+
// non-zero: what was printed is partial and says so.
|
|
256
|
+
const deferred = deferralFromDisk(root);
|
|
257
|
+
if (!deferred) throw error;
|
|
258
|
+
unavailable = error.message;
|
|
259
|
+
status = {
|
|
260
|
+
store: journalPath(root),
|
|
261
|
+
unavailable,
|
|
262
|
+
drain: { deferred },
|
|
263
|
+
nodes: [],
|
|
264
|
+
asks: [],
|
|
265
|
+
supervisors: [],
|
|
266
|
+
leases: [],
|
|
267
|
+
active: [],
|
|
268
|
+
fences: [],
|
|
269
|
+
};
|
|
239
270
|
}
|
|
240
271
|
}
|
|
241
272
|
if (args.json) {
|
|
242
273
|
console.log(JSON.stringify(status, null, 2));
|
|
274
|
+
if (unavailable) process.exitCode = 1;
|
|
243
275
|
return;
|
|
244
276
|
}
|
|
245
277
|
printStatus(status, { supervisor });
|
|
278
|
+
if (unavailable) process.exitCode = 1;
|
|
246
279
|
}
|
|
247
280
|
|
|
248
281
|
function printStatus(status, { supervisor }) {
|
|
249
282
|
const resident = status.resident ?? { pids: [], totalMb: 0 };
|
|
250
283
|
console.log(`fleet store ${status.store}`);
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
284
|
+
if (status.unavailable) {
|
|
285
|
+
// The rows below the store line were unreadable; say so, then print what a
|
|
286
|
+
// lock-free read could still say.
|
|
287
|
+
console.log(`store unreadable: ${status.unavailable}`);
|
|
288
|
+
} else {
|
|
289
|
+
console.log(
|
|
290
|
+
`schema ${status.schemaVersion} supervisor ${supervisor} epoch ${status.epoch ?? "-"} cap ${status.cap ?? "-"} active ${status.active.length}`,
|
|
291
|
+
);
|
|
292
|
+
console.log(
|
|
293
|
+
`resident ${resident.totalMb} MB over ${resident.pids.length} pids ${resident.pids.map((row) => `${row.pid}:${row.mb}`).join(" ") || "-"}`,
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
// The store's own account of a pass it would not take. A deferral is the
|
|
297
|
+
// record of a pass that could not finish, and a completed one says so rather
|
|
298
|
+
// than reading as an open failure; a wait is a supervisor that is contending
|
|
299
|
+
// for the write lock, which is worth seeing before a pass ever gives up. The
|
|
300
|
+
// deferral names the pass, because the store can refuse more than one.
|
|
261
301
|
if (status.drain?.deferred) {
|
|
262
302
|
const deferred = status.drain.deferred;
|
|
263
303
|
const outcome = deferred.recoveredAt
|
|
264
304
|
? `completed at ${new Date(deferred.recoveredAt).toISOString()}`
|
|
265
305
|
: "still outstanding";
|
|
266
306
|
console.log(
|
|
267
|
-
|
|
307
|
+
`${deferred.what ?? "drain"} deferred ${new Date(deferred.at).toISOString()}: ${deferred.attempts} attempt(s) in ${deferred.waitedMs} ms (${deferred.error}); ${outcome}`,
|
|
268
308
|
);
|
|
269
309
|
}
|
|
270
310
|
if (status.drain?.waited) {
|
|
@@ -273,6 +313,7 @@ function printStatus(status, { supervisor }) {
|
|
|
273
313
|
`drain waited ${waited.waitedMs} ms over ${waited.attempts} attempt(s) for the store lock, last at ${new Date(waited.at).toISOString()}`,
|
|
274
314
|
);
|
|
275
315
|
}
|
|
316
|
+
if (status.unavailable) return;
|
|
276
317
|
console.log(`nodes ${status.nodes.length}:`);
|
|
277
318
|
for (const node of status.nodes) {
|
|
278
319
|
const effects = node.lastTurn ? JSON.stringify(node.lastTurn.effects) : "{}";
|
|
@@ -25,9 +25,76 @@
|
|
|
25
25
|
import { createHash } from "node:crypto";
|
|
26
26
|
import { mkdirSync } from "node:fs";
|
|
27
27
|
import { dirname, isAbsolute, join, resolve } from "node:path";
|
|
28
|
-
import { DatabaseSync } from "node:sqlite";
|
|
29
28
|
|
|
30
|
-
|
|
29
|
+
/**
|
|
30
|
+
* The fleet's only way in to SQLite, and the answer to what its import costs.
|
|
31
|
+
*
|
|
32
|
+
* `node:sqlite` says it is experimental on stderr, once per process, at the moment
|
|
33
|
+
* it is imported: true of every 24 release up to 24.14, and no longer true from
|
|
34
|
+
* 24.15, where the module became a release candidate and the warning stopped
|
|
35
|
+
* (measured: 24.0.0 and 24.14.0 warn, 24.15.0 and 24.22.1 are silent). The floor
|
|
36
|
+
* `engines` declares is 24.0.0, so without this block a user on the floor meets a
|
|
37
|
+
* warning nobody documented, and every command that asserts an empty stderr - the
|
|
38
|
+
* fleet's own arms among them - fails on a healthy run. Once Node's whole supported
|
|
39
|
+
* line is past 24.15 this block answers nothing and can be deleted with its test.
|
|
40
|
+
*
|
|
41
|
+
* Every process that opens a journal pays it: the four commands, the supervisor
|
|
42
|
+
* they start, the drivers, and the guard extension. One import, answered once, is
|
|
43
|
+
* why they do not each need their own handling - `DatabaseSync` is re-exported for
|
|
44
|
+
* the guard rather than imported there a second time.
|
|
45
|
+
*
|
|
46
|
+
* The answer is matched on the warning's own type and text, and it lasts for the
|
|
47
|
+
* length of that import only: the original emitter is restored in `finally`, so a
|
|
48
|
+
* warning from any other module - another experimental API, a deprecation -
|
|
49
|
+
* reaches stderr exactly as it would without this file. The flags that would also
|
|
50
|
+
* remove this line, `--disable-warning` and `NODE_OPTIONS`, silence the whole
|
|
51
|
+
* ExperimentalWarning class for the entire process, which would hide those.
|
|
52
|
+
*
|
|
53
|
+
* A `warning` listener is not an alternative: it cannot suppress anything. Node's
|
|
54
|
+
* default printer is registered by the runtime, and it prints whether or not
|
|
55
|
+
* another listener exists (measured on 22.19.0, 22.23.4 and 24.0.0 - the listener
|
|
56
|
+
* sees the warning and stderr gets it too), so the emitter call itself is answered
|
|
57
|
+
* here, inside an import nobody else can route around.
|
|
58
|
+
*/
|
|
59
|
+
const SQLITE_WARNING_TYPE = "ExperimentalWarning";
|
|
60
|
+
const SQLITE_WARNING_TEXT = "SQLite is an experimental feature and might change at any time";
|
|
61
|
+
|
|
62
|
+
/** The runtime's emitter, with the SQLite import's one warning withheld. */
|
|
63
|
+
function answerSqliteWarning(emit) {
|
|
64
|
+
return function (warning, type, ...rest) {
|
|
65
|
+
const text = typeof warning === "string" ? warning : warning?.message;
|
|
66
|
+
const name = type ?? warning?.name;
|
|
67
|
+
if (text === SQLITE_WARNING_TEXT && name === SQLITE_WARNING_TYPE) return undefined;
|
|
68
|
+
return emit.call(process, warning, type, ...rest);
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function importSqlite() {
|
|
73
|
+
const emit = process.emitWarning;
|
|
74
|
+
process.emitWarning = answerSqliteWarning(emit);
|
|
75
|
+
try {
|
|
76
|
+
return await import("node:sqlite");
|
|
77
|
+
} finally {
|
|
78
|
+
process.emitWarning = emit;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export const { DatabaseSync } = await importSqlite();
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* The shape a store has once `openJournal` returns.
|
|
86
|
+
*
|
|
87
|
+
* Version 2 is version 1 with the lease's expiry column nullable and unset. The
|
|
88
|
+
* column stays because the build that shipped version 1 names it in its lease
|
|
89
|
+
* write, so a store without it is a store that build cannot open: a rollback
|
|
90
|
+
* would die on "no such column" instead of on something an operator can act on.
|
|
91
|
+
* Nothing here reads or writes it (see the leases DDL for why).
|
|
92
|
+
*
|
|
93
|
+
* A store recording a version above this one is refused rather than opened. Its
|
|
94
|
+
* shape is one this code does not know, and `CREATE TABLE IF NOT EXISTS` would
|
|
95
|
+
* leave it looking fine and then fail inside a statement far from the reason.
|
|
96
|
+
*/
|
|
97
|
+
export const SCHEMA_VERSION = 2;
|
|
31
98
|
export const JOURNAL_FILE = "fleet.sqlite";
|
|
32
99
|
|
|
33
100
|
/**
|
|
@@ -39,11 +106,12 @@ export const JOURNAL_FILE = "fleet.sqlite";
|
|
|
39
106
|
* asks holds the lock for as long as that takes on a loaded box. A drainer that
|
|
40
107
|
* met contention therefore threw `database is locked` at five seconds - a
|
|
41
108
|
* legitimate drain under load, reported as a crash. Ten seconds is the wait
|
|
42
|
-
* that turns that class of contention into waiting
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
109
|
+
* that turns that class of contention into waiting.
|
|
110
|
+
*
|
|
111
|
+
* It bounds one statement. Not a pass, and not a lease: a pass with its own
|
|
112
|
+
* budget (the drain) or its own record (the tick's maintenance) sets the wait
|
|
113
|
+
* per call with `withBusyTimeout`, and a node lease is not time-bounded at all
|
|
114
|
+
* - exclusivity is its owner answering for it, as `acquireLease` explains.
|
|
47
115
|
*/
|
|
48
116
|
export const BUSY_TIMEOUT_MS = 10_000;
|
|
49
117
|
|
|
@@ -164,7 +232,7 @@ export const FLEET_JOURNAL_FIELDS = [
|
|
|
164
232
|
},
|
|
165
233
|
{
|
|
166
234
|
n: 11,
|
|
167
|
-
what: "the node lease: owner, epoch
|
|
235
|
+
what: "the node lease: its owner, epoch, heartbeat and state",
|
|
168
236
|
columns: ["leases.node_id", "leases.owner_pid", "leases.epoch", "leases.heartbeat_at", "leases.state"],
|
|
169
237
|
failure: "two supervisors both believe they own one node",
|
|
170
238
|
},
|
|
@@ -311,6 +379,13 @@ CREATE TABLE IF NOT EXISTS routes (
|
|
|
311
379
|
delivery_state TEXT NOT NULL
|
|
312
380
|
);
|
|
313
381
|
|
|
382
|
+
-- expires_at is kept here, and left unset. The build that shipped schema version 1
|
|
383
|
+
-- writes it by name in its lease INSERT, so a store without the column is a store
|
|
384
|
+
-- that build cannot open. Nothing here reads it or writes it: a lease is
|
|
385
|
+
-- exclusive for as long as its owner answers for the node (see acquireLease), so
|
|
386
|
+
-- the value bounded nothing, and an expiry sitting in it is a number an operator
|
|
387
|
+
-- could read and be wrong about. A version-1 store is relaxed to this shape on
|
|
388
|
+
-- open and its stale values are cleared, so the column says "unset" afterwards.
|
|
314
389
|
CREATE TABLE IF NOT EXISTS leases (
|
|
315
390
|
lease_id TEXT PRIMARY KEY,
|
|
316
391
|
node_id TEXT NOT NULL,
|
|
@@ -318,7 +393,7 @@ CREATE TABLE IF NOT EXISTS leases (
|
|
|
318
393
|
epoch INTEGER NOT NULL,
|
|
319
394
|
acquired_at INTEGER NOT NULL,
|
|
320
395
|
heartbeat_at INTEGER NOT NULL,
|
|
321
|
-
expires_at INTEGER
|
|
396
|
+
expires_at INTEGER,
|
|
322
397
|
released_at INTEGER,
|
|
323
398
|
state TEXT NOT NULL
|
|
324
399
|
);
|
|
@@ -362,6 +437,58 @@ export function journalPath(root) {
|
|
|
362
437
|
* recorded inside the file. A copy that travels without its root still says
|
|
363
438
|
* which root it came from; two roots are two fleets and never one store.
|
|
364
439
|
*/
|
|
440
|
+
|
|
441
|
+
/** The schema version a store records, or null when it has never been written. */
|
|
442
|
+
function readStoredSchemaVersion(db) {
|
|
443
|
+
const table = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'fleet_meta'").get();
|
|
444
|
+
if (!table) return null;
|
|
445
|
+
const row = db.prepare("SELECT value FROM fleet_meta WHERE key = 'schema_version'").get();
|
|
446
|
+
const version = Number(row?.value);
|
|
447
|
+
return Number.isFinite(version) ? version : null;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Relax `leases.expires_at` from NOT NULL to nullable, clearing every value, so
|
|
452
|
+
* that a store written by the version-1 build opens here and that build's own
|
|
453
|
+
* lease write keeps working after a rollback.
|
|
454
|
+
*
|
|
455
|
+
* SQLite cannot change a column in place, so the table is rebuilt once: every
|
|
456
|
+
* row and the index come with it. The values do not - a stale expiry is the one
|
|
457
|
+
* thing this column must not carry, because the column says nothing about the
|
|
458
|
+
* lease and a number in it reads like something that does.
|
|
459
|
+
*/
|
|
460
|
+
function relaxLeaseExpiry(db) {
|
|
461
|
+
const column = db
|
|
462
|
+
.prepare("PRAGMA table_info(leases)")
|
|
463
|
+
.all()
|
|
464
|
+
.find((entry) => entry.name === "expires_at");
|
|
465
|
+
if (!column || column.notnull === 0) return false;
|
|
466
|
+
db.exec("BEGIN IMMEDIATE");
|
|
467
|
+
try {
|
|
468
|
+
db.exec(`CREATE TABLE leases_relaxed (
|
|
469
|
+
lease_id TEXT PRIMARY KEY,
|
|
470
|
+
node_id TEXT NOT NULL,
|
|
471
|
+
owner_pid INTEGER NOT NULL,
|
|
472
|
+
epoch INTEGER NOT NULL,
|
|
473
|
+
acquired_at INTEGER NOT NULL,
|
|
474
|
+
heartbeat_at INTEGER NOT NULL,
|
|
475
|
+
expires_at INTEGER,
|
|
476
|
+
released_at INTEGER,
|
|
477
|
+
state TEXT NOT NULL
|
|
478
|
+
)`);
|
|
479
|
+
db.exec(`INSERT INTO leases_relaxed (lease_id, node_id, owner_pid, epoch, acquired_at, heartbeat_at, expires_at, released_at, state)
|
|
480
|
+
SELECT lease_id, node_id, owner_pid, epoch, acquired_at, heartbeat_at, NULL, released_at, state FROM leases`);
|
|
481
|
+
db.exec("DROP TABLE leases");
|
|
482
|
+
db.exec("ALTER TABLE leases_relaxed RENAME TO leases");
|
|
483
|
+
db.exec("CREATE INDEX IF NOT EXISTS leases_node ON leases(node_id, state)");
|
|
484
|
+
db.exec("COMMIT");
|
|
485
|
+
} catch (error) {
|
|
486
|
+
db.exec("ROLLBACK");
|
|
487
|
+
throw error;
|
|
488
|
+
}
|
|
489
|
+
return true;
|
|
490
|
+
}
|
|
491
|
+
|
|
365
492
|
export function openJournal(root, options = {}) {
|
|
366
493
|
const fleetRoot = resolveFleetRoot(root);
|
|
367
494
|
mkdirSync(dirname(journalPath(fleetRoot)), { recursive: true });
|
|
@@ -398,7 +525,16 @@ export function openJournal(root, options = {}) {
|
|
|
398
525
|
}
|
|
399
526
|
}
|
|
400
527
|
db.exec("PRAGMA synchronous = FULL");
|
|
528
|
+
// Before the DDL, because the DDL is written for this version's shape and a
|
|
529
|
+
// store from a newer build has one this code does not know.
|
|
530
|
+
const storedVersion = readStoredSchemaVersion(db);
|
|
531
|
+
if (storedVersion != null && storedVersion > SCHEMA_VERSION) {
|
|
532
|
+
throw new Error(
|
|
533
|
+
`fleet store ${journalPath(fleetRoot)} was written by a newer build (schema ${storedVersion}; this build speaks ${SCHEMA_VERSION}): open it with that build, or move it aside and start a new root`,
|
|
534
|
+
);
|
|
535
|
+
}
|
|
401
536
|
db.exec(DDL);
|
|
537
|
+
relaxLeaseExpiry(db);
|
|
402
538
|
const projectRoot = options.projectRoot ? resolve(options.projectRoot) : fleetRoot;
|
|
403
539
|
const meta = {
|
|
404
540
|
store_path: journalPath(fleetRoot),
|
|
@@ -411,6 +547,12 @@ export function openJournal(root, options = {}) {
|
|
|
411
547
|
};
|
|
412
548
|
const insertMeta = db.prepare("INSERT OR IGNORE INTO fleet_meta(key, value) VALUES (?, ?)");
|
|
413
549
|
for (const [key, value] of Object.entries(meta)) insertMeta.run(key, value);
|
|
550
|
+
// A store opened here records the shape it now has. The insert above ignores a
|
|
551
|
+
// row that already exists, so a store migrated from an older version needs the
|
|
552
|
+
// value written rather than inserted.
|
|
553
|
+
if (storedVersion !== SCHEMA_VERSION) {
|
|
554
|
+
db.prepare("UPDATE fleet_meta SET value = ? WHERE key = 'schema_version'").run(String(SCHEMA_VERSION));
|
|
555
|
+
}
|
|
414
556
|
db.prepare("INSERT OR IGNORE INTO projects(project_root, created_at) VALUES (?, ?)").run(projectRoot, now());
|
|
415
557
|
return new Journal(db, fleetRoot);
|
|
416
558
|
}
|
|
@@ -466,6 +608,31 @@ export class Journal {
|
|
|
466
608
|
}
|
|
467
609
|
}
|
|
468
610
|
|
|
611
|
+
/**
|
|
612
|
+
* Run `body` with a statement wait of `ms`, restoring the previous one after.
|
|
613
|
+
*
|
|
614
|
+
* The wait is per-connection state, and a caller with a bounded budget of its
|
|
615
|
+
* own (the tick's maintenance passes, which must not block on a lock another
|
|
616
|
+
* process may hold for as long as it likes) sets it here rather than leaving
|
|
617
|
+
* every later statement in the process with it.
|
|
618
|
+
*/
|
|
619
|
+
withBusyTimeout(ms, body) {
|
|
620
|
+
const previous = this.busyTimeoutMs;
|
|
621
|
+
const bounded = Math.max(0, Math.floor(ms));
|
|
622
|
+
if (bounded !== previous) {
|
|
623
|
+
this.db.exec(`PRAGMA busy_timeout = ${bounded}`);
|
|
624
|
+
this.busyTimeoutMs = bounded;
|
|
625
|
+
}
|
|
626
|
+
try {
|
|
627
|
+
return body();
|
|
628
|
+
} finally {
|
|
629
|
+
if (bounded !== previous) {
|
|
630
|
+
this.db.exec(`PRAGMA busy_timeout = ${previous}`);
|
|
631
|
+
this.busyTimeoutMs = previous;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
469
636
|
_stmt(sql) {
|
|
470
637
|
let stmt = this._cache.get(sql);
|
|
471
638
|
if (!stmt) {
|
|
@@ -921,15 +1088,32 @@ export class Journal {
|
|
|
921
1088
|
return this._stmt("SELECT * FROM supervisors ORDER BY epoch").all();
|
|
922
1089
|
}
|
|
923
1090
|
|
|
924
|
-
|
|
1091
|
+
/**
|
|
1092
|
+
* Take or renew this epoch's lease on one node.
|
|
1093
|
+
*
|
|
1094
|
+
* Exclusivity is the owner answering for the node, not a clock. A lease lives
|
|
1095
|
+
* as long as its owner keeps renewing it, and a predecessor's lease ends in
|
|
1096
|
+
* reconciliation, which asks whether that owner is still there before it
|
|
1097
|
+
* touches anything.
|
|
1098
|
+
*
|
|
1099
|
+
* There is no expiry here, and the stored column of that name is deliberately
|
|
1100
|
+
* neither read nor written: a wall-clock bound was measured into disrepute (one
|
|
1101
|
+
* legitimate tick ran 32.7s under write contention, so any TTL short enough to
|
|
1102
|
+
* name lapsed mid-turn, and a lapsed lease a second supervisor may claim puts
|
|
1103
|
+
* one node under two drivers). Reconciliation refuses to fence a live owner's
|
|
1104
|
+
* turn for the same reason: a clock cannot see whether a child is still
|
|
1105
|
+
* working. The column survives only so the build that shipped it can still
|
|
1106
|
+
* open the store - see the leases DDL.
|
|
1107
|
+
*/
|
|
1108
|
+
acquireLease({ nodeId, ownerPid, epoch }) {
|
|
925
1109
|
const existing = this.liveLease(nodeId);
|
|
926
1110
|
if (existing && existing.epoch !== epoch) return null;
|
|
927
1111
|
const leaseId = `${nodeId}:${epoch}`;
|
|
928
1112
|
this._stmt(
|
|
929
|
-
`INSERT INTO leases (lease_id, node_id, owner_pid, epoch, acquired_at, heartbeat_at,
|
|
930
|
-
VALUES (?, ?, ?, ?, ?, ?,
|
|
931
|
-
ON CONFLICT(lease_id) DO UPDATE SET heartbeat_at = excluded.heartbeat_at,
|
|
932
|
-
).run(leaseId, nodeId, ownerPid, epoch, now(), now()
|
|
1113
|
+
`INSERT INTO leases (lease_id, node_id, owner_pid, epoch, acquired_at, heartbeat_at, state)
|
|
1114
|
+
VALUES (?, ?, ?, ?, ?, ?, 'live')
|
|
1115
|
+
ON CONFLICT(lease_id) DO UPDATE SET heartbeat_at = excluded.heartbeat_at, state = 'live'`,
|
|
1116
|
+
).run(leaseId, nodeId, ownerPid, epoch, now(), now());
|
|
933
1117
|
return this._stmt("SELECT * FROM leases WHERE lease_id = ?").get(leaseId);
|
|
934
1118
|
}
|
|
935
1119
|
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
|
|
30
30
|
import { execFileSync } from "node:child_process";
|
|
31
31
|
import { createHash, randomBytes } from "node:crypto";
|
|
32
|
-
import { chmodSync, existsSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
32
|
+
import { chmodSync, existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
33
33
|
import { createServer, connect } from "node:net";
|
|
34
34
|
import { tmpdir } from "node:os";
|
|
35
35
|
import { join } from "node:path";
|
|
@@ -44,6 +44,14 @@ export { ASK_SENTINEL, DONE_SENTINEL };
|
|
|
44
44
|
|
|
45
45
|
export const NODE_STATES = ["settled", "waiting_on_child", "waiting_on_human", "failed", "done"];
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Where a maintenance fact goes when the store will not take the write: beside
|
|
49
|
+
* the journal, because a fact about a wedged store is the one fact the store
|
|
50
|
+
* cannot hold. `status` reads it, and the first pass that gets through copies it
|
|
51
|
+
* into the journal and removes it.
|
|
52
|
+
*/
|
|
53
|
+
export const DEFERRED_FILE = "store-deferred.json";
|
|
54
|
+
|
|
47
55
|
/**
|
|
48
56
|
* Why no future turn can carry a message for this node, or null.
|
|
49
57
|
*
|
|
@@ -102,6 +110,20 @@ function jsonMeta(meta, key) {
|
|
|
102
110
|
}
|
|
103
111
|
}
|
|
104
112
|
|
|
113
|
+
/**
|
|
114
|
+
* The deferral a wedged or dead supervisor left beside the store, or null.
|
|
115
|
+
*
|
|
116
|
+
* Exported because `status` reads it too: when the store cannot be opened at all,
|
|
117
|
+
* this is the one fact about why the fleet is idle that is still readable.
|
|
118
|
+
*/
|
|
119
|
+
export function deferralFromDisk(root) {
|
|
120
|
+
try {
|
|
121
|
+
return JSON.parse(readFileSync(join(root, DEFERRED_FILE), "utf8"));
|
|
122
|
+
} catch {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
105
127
|
/**
|
|
106
128
|
* Where the control socket lives.
|
|
107
129
|
*
|
|
@@ -188,6 +210,46 @@ export const CHILD_FLAGS = [
|
|
|
188
210
|
"off",
|
|
189
211
|
];
|
|
190
212
|
|
|
213
|
+
/**
|
|
214
|
+
* The queue section of one prompt: the lines it appends for the messages the
|
|
215
|
+
* turn carries, and the ids those lines are.
|
|
216
|
+
*
|
|
217
|
+
* Both halves come out of one array here, and no other code builds the section
|
|
218
|
+
* by hand, so the answer to "what did this turn carry?" is the section that went
|
|
219
|
+
* into the prompt rather than a second reading of the finished text. That is the
|
|
220
|
+
* difference between a delivery and a coincidence: a charter written beside a
|
|
221
|
+
* queued message very often repeats the operator's instruction word for word,
|
|
222
|
+
* and a body found by search is not a message the model was given.
|
|
223
|
+
*/
|
|
224
|
+
export function queueSection(header, pending) {
|
|
225
|
+
const entries = pending.map((message) => ({ msgId: message.msg_id, line: `- ${message.body}` }));
|
|
226
|
+
return { header, lines: entries.map((entry) => entry.line), entries };
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Which of the section's messages the prompt really carries.
|
|
231
|
+
*
|
|
232
|
+
* The builder already says what it appended, and the text has to agree: the
|
|
233
|
+
* section's own sentence must be there, and so must the message's own line. A
|
|
234
|
+
* turn kind that appends no section carries nothing, and a prompt that repeats a
|
|
235
|
+
* body in another section - above all a charter - is not read as one.
|
|
236
|
+
*/
|
|
237
|
+
export function carriedBy(prompt, section) {
|
|
238
|
+
if (section.entries.length === 0) return [];
|
|
239
|
+
if (!prompt.includes(section.header)) return [];
|
|
240
|
+
return section.entries.filter((entry) => prompt.includes(entry.line)).map((entry) => entry.msgId);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** A prompt assembled around its queue section: the head, the section, the tail. */
|
|
244
|
+
function withSection(head, section, tail = []) {
|
|
245
|
+
const lines = [...head];
|
|
246
|
+
if (section.entries.length > 0) {
|
|
247
|
+
if (lines.length > 0) lines.push("");
|
|
248
|
+
lines.push(section.header, ...section.lines);
|
|
249
|
+
}
|
|
250
|
+
return { prompt: [...lines, ...tail].join("\n"), section };
|
|
251
|
+
}
|
|
252
|
+
|
|
191
253
|
/** The fleet preamble a node's first turn carries, before its charter body. */
|
|
192
254
|
export function charterPrompt(node, body, pending = []) {
|
|
193
255
|
const lines = [
|
|
@@ -203,11 +265,15 @@ export function charterPrompt(node, body, pending = []) {
|
|
|
203
265
|
// A queued message is an operator's instruction for this node, and the first
|
|
204
266
|
// turn is a turn: it rides along instead of waiting for a later one. The
|
|
205
267
|
// charter keeps its shape, and the queue is a section of its own after it.
|
|
206
|
-
|
|
207
|
-
lines
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
268
|
+
return withSection(
|
|
269
|
+
lines,
|
|
270
|
+
queueSection("The operator sent this before your first turn. It applies to this turn as well:", pending),
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/** The between-turns turn: the queued messages, and nothing else to read. */
|
|
275
|
+
export function messagePrompt(pending) {
|
|
276
|
+
return withSection([], queueSection("FLEET MESSAGE.", pending), ["", "Act on it, then end your turn."]);
|
|
211
277
|
}
|
|
212
278
|
|
|
213
279
|
/**
|
|
@@ -246,12 +312,10 @@ export function recoveryPrompt({ node, interruptedTurn, completed, unknown, pend
|
|
|
246
312
|
if (note && completed.length === 0 && unknown.length === 0) {
|
|
247
313
|
lines.push("", "No effect had been recorded when the turn was interrupted.");
|
|
248
314
|
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
lines.push("", "Continue the charter from its first unfinished step.");
|
|
254
|
-
return lines.join("\n");
|
|
315
|
+
return withSection(lines, queueSection("Messages waiting for you:", pending), [
|
|
316
|
+
"",
|
|
317
|
+
"Continue the charter from its first unfinished step.",
|
|
318
|
+
]);
|
|
255
319
|
}
|
|
256
320
|
|
|
257
321
|
/**
|
|
@@ -325,14 +389,15 @@ export class Supervisor {
|
|
|
325
389
|
const defaults = {
|
|
326
390
|
cap: 3,
|
|
327
391
|
tickMs: 250,
|
|
328
|
-
leaseTtlMs: 30_000,
|
|
329
392
|
askTtlMs: 600_000,
|
|
330
393
|
turnTimeoutMs: 300_000,
|
|
331
394
|
maxAttempts: 3,
|
|
332
395
|
idleExitMs: 5_000,
|
|
333
|
-
// The
|
|
334
|
-
//
|
|
335
|
-
//
|
|
396
|
+
// The maintenance budget: how long a pass may spend meeting another
|
|
397
|
+
// process's write lock before it is recorded as deferred and left to a
|
|
398
|
+
// later tick. Two windows at ten seconds plus one backoff is the most a
|
|
399
|
+
// tick's maintenance can spend waiting. Nothing here bounds a lease: a
|
|
400
|
+
// lease is exclusive for as long as its owner answers (journal.mjs).
|
|
336
401
|
drainAttempts: 2,
|
|
337
402
|
drainBusyTimeoutMs: BUSY_TIMEOUT_MS,
|
|
338
403
|
drainBackoffMs: 250,
|
|
@@ -345,6 +410,11 @@ export class Supervisor {
|
|
|
345
410
|
piBin: "pi",
|
|
346
411
|
naive: false,
|
|
347
412
|
control: true,
|
|
413
|
+
// What a prompt appends for the messages it carries. The only caller that
|
|
414
|
+
// turns it off is the queue arm, which needs a prompt whose body appears
|
|
415
|
+
// somewhere else in the text; no flag maps to it, and it can only make a
|
|
416
|
+
// turn carry less.
|
|
417
|
+
queueSection: true,
|
|
348
418
|
extensions: [],
|
|
349
419
|
extraArgs: [],
|
|
350
420
|
};
|
|
@@ -372,9 +442,12 @@ export class Supervisor {
|
|
|
372
442
|
this.stopping = false;
|
|
373
443
|
this.fences = [];
|
|
374
444
|
this.idleSince = null;
|
|
375
|
-
// A
|
|
376
|
-
//
|
|
377
|
-
this
|
|
445
|
+
// A maintenance pass that could not write: the store was locked when the
|
|
446
|
+
// fact was made. It is written beside the store at once and journalled at
|
|
447
|
+
// the first pass that gets through, so it outlives this process - the
|
|
448
|
+
// measured case is a tick that threw `database is locked` and left an
|
|
449
|
+
// overdue ask open with nothing saying why.
|
|
450
|
+
this.pendingStoreDeferral = null;
|
|
378
451
|
// Resolved when this process has stopped itself because the fleet had
|
|
379
452
|
// nothing left to do; the entry point exits on it rather than on a guess.
|
|
380
453
|
this.done = new Promise((resolveDone) => {
|
|
@@ -509,13 +582,12 @@ export class Supervisor {
|
|
|
509
582
|
*
|
|
510
583
|
* Contention is waited out, not thrown. A drain that meets the store's write
|
|
511
584
|
* lock waits out the journal's busy timeout, retries with a backoff, and
|
|
512
|
-
* reports a drain it could not finish as a
|
|
513
|
-
*
|
|
514
|
-
*
|
|
515
|
-
*
|
|
516
|
-
*
|
|
517
|
-
*
|
|
518
|
-
* maintenance pass.
|
|
585
|
+
* reports a drain it could not finish as a fact (`store_deferred`) instead of
|
|
586
|
+
* as an exception from a tick. The earlier shape was measured: two drainers on
|
|
587
|
+
* one store, a full-suite run, one of them `database is locked` and gone, and
|
|
588
|
+
* the asks it was there to drain stayed held. A supervisor whose drain throws
|
|
589
|
+
* does not drain that ask, and its caller is a tick handler, so the failure
|
|
590
|
+
* was a log line and a skipped maintenance pass.
|
|
519
591
|
*/
|
|
520
592
|
drainExpiredAsks() {
|
|
521
593
|
const started = Date.now();
|
|
@@ -537,7 +609,7 @@ export class Supervisor {
|
|
|
537
609
|
if (!isBusyError(error)) throw error;
|
|
538
610
|
const why = error.errstr ?? error.message;
|
|
539
611
|
if (attempt >= attempts) {
|
|
540
|
-
this.
|
|
612
|
+
this._recordStoreDeferral({ what: "drain", attempts: attempt, waitedMs: Date.now() - started, error: why });
|
|
541
613
|
return [];
|
|
542
614
|
}
|
|
543
615
|
const backoffMs = this.options.drainBackoffMs * attempt;
|
|
@@ -602,7 +674,7 @@ export class Supervisor {
|
|
|
602
674
|
* write lands, which is the earliest point the store can hold the fact at all.
|
|
603
675
|
*/
|
|
604
676
|
_recordDrainContention({ attempt, waitedMs }) {
|
|
605
|
-
this.
|
|
677
|
+
this._flushStoreDeferral(Date.now());
|
|
606
678
|
if (attempt <= 1 && waitedMs < this.options.drainWaitReportMs) return;
|
|
607
679
|
try {
|
|
608
680
|
this.journal.transaction(
|
|
@@ -616,39 +688,80 @@ export class Supervisor {
|
|
|
616
688
|
}
|
|
617
689
|
}
|
|
618
690
|
|
|
691
|
+
/** The path a fact goes to when the store will not take it. */
|
|
692
|
+
deferredPath() {
|
|
693
|
+
return join(this.root, DEFERRED_FILE);
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* The deferral as it stands: this process's, or one an earlier process left
|
|
698
|
+
* beside the store. Reading the file is what makes the fact survive a death,
|
|
699
|
+
* and it is also what lets the next supervisor finish the job.
|
|
700
|
+
*/
|
|
701
|
+
_pendingDeferral() {
|
|
702
|
+
if (this.pendingStoreDeferral) return this.pendingStoreDeferral;
|
|
703
|
+
const onDisk = deferralFromDisk(this.root);
|
|
704
|
+
if (onDisk) this.pendingStoreDeferral = onDisk;
|
|
705
|
+
return this.pendingStoreDeferral;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
/**
|
|
709
|
+
* Put the fact where it can be read with no lock: a file beside the journal.
|
|
710
|
+
*
|
|
711
|
+
* There is no second try if this fails - the store is unwritable and so is the
|
|
712
|
+
* root - so the log line is the last surface left, and it says so.
|
|
713
|
+
*/
|
|
714
|
+
_writeDeferralFile(fact) {
|
|
715
|
+
const path = this.deferredPath();
|
|
716
|
+
try {
|
|
717
|
+
writeFileSync(`${path}.tmp`, `${JSON.stringify(fact, null, 2)}\n`);
|
|
718
|
+
renameSync(`${path}.tmp`, path);
|
|
719
|
+
} catch (error) {
|
|
720
|
+
this.log(`fleet: the deferral could not be written to ${path} (${error.code ?? error.message})`);
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
619
724
|
/**
|
|
620
|
-
*
|
|
621
|
-
* later
|
|
622
|
-
* the
|
|
725
|
+
* Copy a pending deferral into the journal, if one is pending. `recoveredAt`
|
|
726
|
+
* says whether a later pass got the work done anyway, which is what tells a
|
|
727
|
+
* reader whether the store was merely busy or the work is still waiting.
|
|
728
|
+
*
|
|
729
|
+
* The file is removed only after the journal has the fact, so a crash between
|
|
730
|
+
* the two leaves it readable rather than losing it; the next pass writes it
|
|
731
|
+
* again.
|
|
623
732
|
*/
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
733
|
+
_flushStoreDeferral(recoveredAt) {
|
|
734
|
+
const pending = this._pendingDeferral();
|
|
735
|
+
if (!pending) return false;
|
|
736
|
+
const fact = recoveredAt == null ? pending : { ...pending, recoveredAt };
|
|
627
737
|
try {
|
|
628
|
-
this.journal.transaction(() => this.journal.setMeta("
|
|
738
|
+
this.journal.transaction(() => this.journal.setMeta("store_deferred", JSON.stringify(fact)), {
|
|
629
739
|
busyTimeoutMs: Math.max(100, this.options.drainBackoffMs),
|
|
630
740
|
});
|
|
631
|
-
this.pendingDrainDeferral = null;
|
|
632
|
-
if (recoveredAt != null) this.log("fleet: the earlier drain deferral was journalled; a later drain completed the work");
|
|
633
|
-
return true;
|
|
634
741
|
} catch (error) {
|
|
635
|
-
this.log(`fleet: the
|
|
742
|
+
this.log(`fleet: the deferral could not be journalled yet (${error.message}); the next pass retries it`);
|
|
636
743
|
return false;
|
|
637
744
|
}
|
|
745
|
+
this.pendingStoreDeferral = null;
|
|
746
|
+
rmSync(this.deferredPath(), { force: true });
|
|
747
|
+
if (recoveredAt != null) this.log("fleet: the earlier deferral was journalled; a later pass completed the work");
|
|
748
|
+
return true;
|
|
638
749
|
}
|
|
639
750
|
|
|
640
751
|
/**
|
|
641
|
-
* A
|
|
642
|
-
*
|
|
643
|
-
*
|
|
644
|
-
*
|
|
752
|
+
* A maintenance pass that gave up on a locked store. The store cannot hold
|
|
753
|
+
* this fact - that is the fact - so it goes to the file first, then into the
|
|
754
|
+
* journal at the first moment the store accepts a write. It is not an
|
|
755
|
+
* exception: the tick keeps ticking, and the pass is retried.
|
|
645
756
|
*/
|
|
646
|
-
|
|
647
|
-
|
|
757
|
+
_recordStoreDeferral({ what, attempts, waitedMs, error }) {
|
|
758
|
+
const fact = { at: Date.now(), what, attempts, waitedMs, error };
|
|
759
|
+
this.pendingStoreDeferral = fact;
|
|
760
|
+
this._writeDeferralFile(fact);
|
|
648
761
|
this.log(
|
|
649
|
-
`fleet:
|
|
762
|
+
`fleet: ${what} could not write after ${attempts} attempt(s) in ${waitedMs} ms (${error}); recorded in ${DEFERRED_FILE} until the store takes it`,
|
|
650
763
|
);
|
|
651
|
-
this.
|
|
764
|
+
this._flushStoreDeferral(null);
|
|
652
765
|
}
|
|
653
766
|
|
|
654
767
|
/**
|
|
@@ -701,6 +814,45 @@ export class Supervisor {
|
|
|
701
814
|
}, this.options.tickMs);
|
|
702
815
|
}
|
|
703
816
|
|
|
817
|
+
/**
|
|
818
|
+
* The tick's maintenance pass: the expired asks, then the queued messages no
|
|
819
|
+
* turn will ever carry.
|
|
820
|
+
*
|
|
821
|
+
* One named pass because it is the part of a tick another process's write lock
|
|
822
|
+
* can stall, and because both halves are retried by the next tick rather than
|
|
823
|
+
* failed for good - which is only true if the failure is recorded. Returns what
|
|
824
|
+
* the drain claimed, so a caller can say what the pass did.
|
|
825
|
+
*/
|
|
826
|
+
maintenance() {
|
|
827
|
+
const expired = this.drainExpiredAsks();
|
|
828
|
+
this._maintenanceWrite("undeliverable sweep", () => this.retireUndeliverableMessages());
|
|
829
|
+
return expired;
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* One maintenance write, with a busy store recorded instead of thrown.
|
|
834
|
+
*
|
|
835
|
+
* The drain answers its own contention; the pass after it meets the same lock
|
|
836
|
+
* and has no policy of its own, and it used to throw into the tick's catch: one
|
|
837
|
+
* log line, no fact, and the rest of the tick skipped. A store that will not
|
|
838
|
+
* take the write is a fact an operator can read, and the next tick retries the
|
|
839
|
+
* pass. Anything that is not contention is a defect and stays an exception.
|
|
840
|
+
*/
|
|
841
|
+
_maintenanceWrite(what, body) {
|
|
842
|
+
const started = Date.now();
|
|
843
|
+
try {
|
|
844
|
+
this.journal.withBusyTimeout(Math.max(100, this.options.drainBusyTimeoutMs), body);
|
|
845
|
+
} catch (error) {
|
|
846
|
+
if (!isBusyError(error)) throw error;
|
|
847
|
+
this._recordStoreDeferral({
|
|
848
|
+
what,
|
|
849
|
+
attempts: 1,
|
|
850
|
+
waitedMs: Date.now() - started,
|
|
851
|
+
error: error.errstr ?? error.message,
|
|
852
|
+
});
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
|
|
704
856
|
async tick() {
|
|
705
857
|
this.journal.heartbeatSupervisor(process.pid);
|
|
706
858
|
for (const [nodeId, ctx] of this.active) {
|
|
@@ -708,12 +860,10 @@ export class Supervisor {
|
|
|
708
860
|
nodeId,
|
|
709
861
|
ownerPid: process.pid,
|
|
710
862
|
epoch: this.epoch,
|
|
711
|
-
ttlMs: this.options.leaseTtlMs,
|
|
712
863
|
});
|
|
713
864
|
if (!renewed) this.log(`fleet: lease for ${nodeId} could not be renewed (ctx ${ctx.turnId})`);
|
|
714
865
|
}
|
|
715
|
-
this.
|
|
716
|
-
this.retireUndeliverableMessages();
|
|
866
|
+
this.maintenance();
|
|
717
867
|
if (this.active.size < this.options.cap) {
|
|
718
868
|
for (const node of this.journal.listNodes()) {
|
|
719
869
|
if (this.stopping || this.active.size >= this.options.cap) break;
|
|
@@ -868,7 +1018,6 @@ export class Supervisor {
|
|
|
868
1018
|
nodeId: node.node_id,
|
|
869
1019
|
ownerPid: process.pid,
|
|
870
1020
|
epoch: this.epoch,
|
|
871
|
-
ttlMs: this.options.leaseTtlMs,
|
|
872
1021
|
});
|
|
873
1022
|
if (!lease) return;
|
|
874
1023
|
const ctx = {
|
|
@@ -902,37 +1051,41 @@ export class Supervisor {
|
|
|
902
1051
|
return;
|
|
903
1052
|
}
|
|
904
1053
|
const pending = this.journal.pendingMessages(node.node_id);
|
|
1054
|
+
// What the prompt is built with. The queue arm is the only caller that
|
|
1055
|
+
// passes none while messages are waiting, and it does so to build the
|
|
1056
|
+
// prompt the old carrier could not tell from a delivery: the body present
|
|
1057
|
+
// in the text, and no section that put it there.
|
|
1058
|
+
const queued = this.options.queueSection === false ? [] : pending;
|
|
905
1059
|
const turns = this.journal.turnsFor(node.node_id);
|
|
906
1060
|
const interrupted = turns.at(-1)?.state === "interrupted" ? turns.at(-1) : null;
|
|
907
|
-
let
|
|
1061
|
+
let built;
|
|
908
1062
|
let promptKind = kind;
|
|
909
1063
|
if (kind === "recovery" && interrupted) {
|
|
910
|
-
|
|
1064
|
+
built = recoveryPrompt({
|
|
911
1065
|
node,
|
|
912
1066
|
interruptedTurn: interrupted,
|
|
913
1067
|
completed: this.journal.settledEffects(interrupted.turn_id),
|
|
914
1068
|
unknown: this.journal
|
|
915
1069
|
.effects(interrupted.turn_id)
|
|
916
1070
|
.filter((effect) => effect.state === "unknown" || effect.state === "attempted"),
|
|
917
|
-
pending,
|
|
1071
|
+
pending: queued,
|
|
918
1072
|
// In-process only, and no command-line flag maps to it: the
|
|
919
1073
|
// note-removal arm is the single caller that turns the note off.
|
|
920
1074
|
note: this.options.recoveryNote !== false,
|
|
921
1075
|
});
|
|
922
1076
|
} else if (kind === "message") {
|
|
923
|
-
|
|
924
|
-
"\n",
|
|
925
|
-
);
|
|
1077
|
+
built = messagePrompt(queued);
|
|
926
1078
|
} else {
|
|
927
|
-
|
|
1079
|
+
built = charterPrompt(node, this._charter(node), queued);
|
|
928
1080
|
if (kind === "naive") promptKind = "naive-replay";
|
|
929
1081
|
}
|
|
930
|
-
|
|
931
|
-
//
|
|
932
|
-
//
|
|
933
|
-
//
|
|
934
|
-
//
|
|
935
|
-
|
|
1082
|
+
const prompt = built.prompt;
|
|
1083
|
+
// The messages this turn carried, from the section the builder appended
|
|
1084
|
+
// and the text that holds it - never from a search of the finished prompt
|
|
1085
|
+
// for a body. A charter that repeats an operator instruction is not a
|
|
1086
|
+
// delivery of the message that happens to say the same thing: the row
|
|
1087
|
+
// keeps waiting, and the next turn that carries it retires it.
|
|
1088
|
+
ctx.carried = carriedBy(prompt, built.section);
|
|
936
1089
|
this.journal.setNodeState(node.node_id, "waiting_on_child");
|
|
937
1090
|
const turnId = this.journal.beginTurn(node.node_id, {
|
|
938
1091
|
kind: promptKind,
|
|
@@ -1410,12 +1563,14 @@ export function buildStatus(journal, supervisor = null) {
|
|
|
1410
1563
|
deadlineAt: ask.deadline_at,
|
|
1411
1564
|
})),
|
|
1412
1565
|
fences: JSON.parse(meta.fences ?? "[]"),
|
|
1413
|
-
// The
|
|
1414
|
-
// to wait for the
|
|
1415
|
-
//
|
|
1416
|
-
// deferral
|
|
1566
|
+
// The store's own account of a pass it would not take: a maintenance pass
|
|
1567
|
+
// that could not finish, and one that had to wait for the lock. The journal
|
|
1568
|
+
// is the first place to look and the file beside it the second, because the
|
|
1569
|
+
// deferral is written while the journal cannot be - a reader that stops at
|
|
1570
|
+
// the journal would see nothing exactly when the store is wedged.
|
|
1417
1571
|
drain: {
|
|
1418
|
-
deferred:
|
|
1572
|
+
deferred:
|
|
1573
|
+
jsonMeta(meta, "store_deferred") ?? jsonMeta(meta, "drain_deferred") ?? deferralFromDisk(journal.root),
|
|
1419
1574
|
waited: jsonMeta(meta, "drain_contention"),
|
|
1420
1575
|
},
|
|
1421
1576
|
resident,
|
|
@@ -97,9 +97,8 @@
|
|
|
97
97
|
|
|
98
98
|
import { appendFileSync, existsSync } from "node:fs";
|
|
99
99
|
import { join, resolve } from "node:path";
|
|
100
|
-
import { DatabaseSync } from "node:sqlite";
|
|
101
100
|
|
|
102
|
-
import { effectKey, journalPath, replayPolicy } from "./journal.mjs";
|
|
101
|
+
import { DatabaseSync, effectKey, journalPath, replayPolicy } from "./journal.mjs";
|
|
103
102
|
|
|
104
103
|
/** The names the child reads. `off` is the switch, not an identifier. */
|
|
105
104
|
export const GUARD_ENV = {
|