pi-braid 0.40.6 → 0.41.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1124 @@
1
+ /**
2
+ * fleet journal: the single SQLite file a fleet supervisor and its nodes
3
+ * resume from, and the discovery record that says which file is the fleet's.
4
+ *
5
+ * The design constrains what a crash must survive; each of those constraints
6
+ * is a field here, and `FLEET_JOURNAL_FIELDS` names the field-to-column
7
+ * mapping next to the failure it prevents. Tests read that export and derive
8
+ * the schema census from it, so a column dropped from the DDL fails a test
9
+ * without anyone editing a second list.
10
+ *
11
+ * Two invariants the DDL enforces rather than documents:
12
+ *
13
+ * - An effect is recorded at `tool_execution_start`, before the tool runs,
14
+ * and settled at `tool_execution_end`. `attempted` without an end is the
15
+ * only legal state a crash can leave, and it is what makes an unknown
16
+ * effect visible to the next turn.
17
+ * - Effect identity is `(turn_id, tool_call_id)`, so a resumed turn can
18
+ * never write a second row for a call it already recorded.
19
+ *
20
+ * Every write is synchronous and durable (`journal_mode = WAL`,
21
+ * `synchronous = FULL`): a journal that can lose its last record across a
22
+ * power cut is not the durable half of anything.
23
+ */
24
+
25
+ import { createHash } from "node:crypto";
26
+ import { mkdirSync } from "node:fs";
27
+ import { dirname, isAbsolute, join, resolve } from "node:path";
28
+ import { DatabaseSync } from "node:sqlite";
29
+
30
+ /**
31
+ * The shape a store has once `openJournal` returns.
32
+ *
33
+ * Version 2 is version 1 with the lease's expiry column nullable and unset. The
34
+ * column stays because the build that shipped version 1 names it in its lease
35
+ * write, so a store without it is a store that build cannot open: a rollback
36
+ * would die on "no such column" instead of on something an operator can act on.
37
+ * Nothing here reads or writes it (see the leases DDL for why).
38
+ *
39
+ * A store recording a version above this one is refused rather than opened. Its
40
+ * shape is one this code does not know, and `CREATE TABLE IF NOT EXISTS` would
41
+ * leave it looking fine and then fail inside a statement far from the reason.
42
+ */
43
+ export const SCHEMA_VERSION = 2;
44
+ export const JOURNAL_FILE = "fleet.sqlite";
45
+
46
+ /**
47
+ * How long one statement waits for the write lock before SQLite reports it
48
+ * busy, in milliseconds.
49
+ *
50
+ * 5000 was too small. Every write here commits with `synchronous = FULL`, so a
51
+ * single commit is an fsync, and a supervisor draining a few hundred overdue
52
+ * asks holds the lock for as long as that takes on a loaded box. A drainer that
53
+ * met contention therefore threw `database is locked` at five seconds - a
54
+ * legitimate drain under load, reported as a crash. Ten seconds is the wait
55
+ * that turns that class of contention into waiting.
56
+ *
57
+ * It bounds one statement. Not a pass, and not a lease: a pass with its own
58
+ * budget (the drain) or its own record (the tick's maintenance) sets the wait
59
+ * per call with `withBusyTimeout`, and a node lease is not time-bounded at all
60
+ * - exclusivity is its owner answering for it, as `acquireLease` explains.
61
+ */
62
+ export const BUSY_TIMEOUT_MS = 10_000;
63
+
64
+ /**
65
+ * The twelve things the journal must hold, in the design's order. `columns`
66
+ * are `table.column`; `failure` is what breaks without the field, quoted so
67
+ * the reason survives independently of the design document.
68
+ */
69
+ export const FLEET_JOURNAL_FIELDS = [
70
+ {
71
+ n: 1,
72
+ what: "node id, parent id, project root, charter path",
73
+ columns: [
74
+ "nodes.node_id",
75
+ "nodes.parent_id",
76
+ "nodes.project_root",
77
+ "nodes.charter_path",
78
+ ],
79
+ failure: "a resumed fleet cannot say which nodes existed or where they belong",
80
+ },
81
+ {
82
+ n: 2,
83
+ what: "pi session id and session file path",
84
+ columns: ["nodes.session_id", "nodes.session_file", "turns.session_id"],
85
+ failure: "resume cannot reopen the same conversation, so it restarts the node",
86
+ },
87
+ {
88
+ n: 3,
89
+ what: "node state and attempt counter",
90
+ columns: ["nodes.state", "nodes.attempts", "nodes.turn_count"],
91
+ failure: "a crashed turn is indistinguishable from a finished one, and a lost node is silent",
92
+ },
93
+ {
94
+ n: 4,
95
+ what: "turn id, prompt ref, per-turn message history reference",
96
+ columns: [
97
+ "turns.turn_id",
98
+ "turns.node_id",
99
+ "turns.prompt_text",
100
+ "model_requests.turn_id",
101
+ "model_requests.seq",
102
+ "model_requests.message_count",
103
+ "model_requests.tool_count",
104
+ ],
105
+ failure: "the parent cannot tell in-flight work from queued work, or see what it sent",
106
+ },
107
+ {
108
+ n: 5,
109
+ what: "each completed effect, its idempotency key, its replay policy",
110
+ columns: [
111
+ "effects.tool_call_id",
112
+ "effects.tool_name",
113
+ "effects.idempotency_key",
114
+ "effects.replay",
115
+ "effects.state",
116
+ "effects.result_json",
117
+ ],
118
+ failure: "the resume repeats the effect",
119
+ },
120
+ {
121
+ n: 6,
122
+ what: "parked ask: token, priority, question, proposed default, deadline, answer, answer source",
123
+ columns: [
124
+ "asks.token",
125
+ "asks.priority",
126
+ "asks.question",
127
+ "asks.default_answer",
128
+ "asks.deadline_at",
129
+ "asks.state",
130
+ "asks.answer_text",
131
+ "asks.answer_source",
132
+ ],
133
+ failure: "a parked node's deadline passes with an answer no one can route",
134
+ },
135
+ {
136
+ n: 7,
137
+ what: "token and spend budget per node and per project",
138
+ columns: [
139
+ "nodes.token_cap",
140
+ "nodes.spend_cap_usd",
141
+ "nodes.tokens_in",
142
+ "nodes.tokens_out",
143
+ "nodes.spend_usd",
144
+ "projects.token_cap",
145
+ "projects.spend_cap_usd",
146
+ "projects.tokens_in",
147
+ "projects.tokens_out",
148
+ "projects.spend_usd",
149
+ ],
150
+ failure: "a queued fleet is a bill with no ceiling",
151
+ },
152
+ {
153
+ n: 8,
154
+ what: "the lock each node holds on its workspace paths",
155
+ columns: ["locks.path", "locks.owner_node", "locks.owner_epoch", "locks.acquired_at"],
156
+ failure: "two nodes write the same file after a restart",
157
+ },
158
+ {
159
+ n: 9,
160
+ what: "the child process id per turn, with its start time",
161
+ columns: ["turns.child_pid", "turns.child_pgid", "turns.started_at", "turns.supervisor_epoch"],
162
+ failure: "an orphaned node keeps writing after the supervisor died",
163
+ },
164
+ {
165
+ n: 10,
166
+ what: "the message id, the node it was routed to, its delivery state",
167
+ columns: [
168
+ "inbox.msg_id",
169
+ "inbox.node_id",
170
+ "inbox.state",
171
+ "inbox.delivered_tier",
172
+ "inbox.delivered_turn_id",
173
+ "routes.msg_id",
174
+ "routes.node_id",
175
+ "routes.delivery_state",
176
+ ],
177
+ failure: "a steered message is delivered twice or lost",
178
+ },
179
+ {
180
+ n: 11,
181
+ what: "the node lease: its owner, epoch, heartbeat and state",
182
+ columns: ["leases.node_id", "leases.owner_pid", "leases.epoch", "leases.heartbeat_at", "leases.state"],
183
+ failure: "two supervisors both believe they own one node",
184
+ },
185
+ {
186
+ n: 12,
187
+ what: "the supervisor's own epoch, so a restart can tell itself from a stale predecessor",
188
+ columns: ["supervisors.pid", "supervisors.epoch", "supervisors.state", "supervisors.heartbeat_at"],
189
+ failure: "a restarted supervisor waits on work its own dead predecessor still appears to hold",
190
+ },
191
+ ];
192
+
193
+ const DDL = `
194
+ CREATE TABLE IF NOT EXISTS fleet_meta (
195
+ key TEXT PRIMARY KEY,
196
+ value TEXT NOT NULL
197
+ );
198
+
199
+ CREATE TABLE IF NOT EXISTS projects (
200
+ project_root TEXT PRIMARY KEY,
201
+ created_at INTEGER NOT NULL,
202
+ tokens_in INTEGER NOT NULL DEFAULT 0,
203
+ tokens_out INTEGER NOT NULL DEFAULT 0,
204
+ spend_usd REAL NOT NULL DEFAULT 0,
205
+ token_cap INTEGER,
206
+ spend_cap_usd REAL
207
+ );
208
+
209
+ CREATE TABLE IF NOT EXISTS nodes (
210
+ node_id TEXT PRIMARY KEY,
211
+ parent_id TEXT,
212
+ project_root TEXT NOT NULL,
213
+ charter_path TEXT NOT NULL,
214
+ driver TEXT NOT NULL,
215
+ driver_config TEXT NOT NULL DEFAULT '{}',
216
+ state TEXT NOT NULL,
217
+ attempts INTEGER NOT NULL DEFAULT 0,
218
+ turn_count INTEGER NOT NULL DEFAULT 0,
219
+ session_id TEXT,
220
+ session_file TEXT,
221
+ tokens_in INTEGER NOT NULL DEFAULT 0,
222
+ tokens_out INTEGER NOT NULL DEFAULT 0,
223
+ spend_usd REAL NOT NULL DEFAULT 0,
224
+ token_cap INTEGER,
225
+ spend_cap_usd REAL,
226
+ done_reason TEXT,
227
+ state_changed_at INTEGER NOT NULL,
228
+ created_at INTEGER NOT NULL
229
+ );
230
+
231
+ CREATE TABLE IF NOT EXISTS turns (
232
+ turn_id INTEGER PRIMARY KEY AUTOINCREMENT,
233
+ node_id TEXT NOT NULL,
234
+ attempt INTEGER NOT NULL,
235
+ kind TEXT NOT NULL,
236
+ state TEXT NOT NULL,
237
+ prompt_text TEXT,
238
+ session_id TEXT,
239
+ child_pid INTEGER,
240
+ child_pgid INTEGER,
241
+ supervisor_epoch INTEGER,
242
+ model_requests INTEGER NOT NULL DEFAULT 0,
243
+ tokens_in INTEGER NOT NULL DEFAULT 0,
244
+ tokens_out INTEGER NOT NULL DEFAULT 0,
245
+ spend_usd REAL NOT NULL DEFAULT 0,
246
+ started_at INTEGER NOT NULL,
247
+ ended_at INTEGER
248
+ );
249
+
250
+ CREATE TABLE IF NOT EXISTS model_requests (
251
+ turn_id INTEGER NOT NULL,
252
+ seq INTEGER NOT NULL,
253
+ message_count INTEGER NOT NULL DEFAULT 0,
254
+ tool_count INTEGER NOT NULL DEFAULT 0,
255
+ tokens_in INTEGER NOT NULL DEFAULT 0,
256
+ tokens_out INTEGER NOT NULL DEFAULT 0,
257
+ at INTEGER NOT NULL,
258
+ PRIMARY KEY (turn_id, seq)
259
+ );
260
+
261
+ CREATE TABLE IF NOT EXISTS effects (
262
+ turn_id INTEGER NOT NULL,
263
+ node_id TEXT NOT NULL,
264
+ tool_call_id TEXT NOT NULL,
265
+ seq INTEGER NOT NULL,
266
+ tool_name TEXT NOT NULL,
267
+ args_json TEXT NOT NULL,
268
+ replay TEXT NOT NULL,
269
+ idempotency_key TEXT NOT NULL,
270
+ state TEXT NOT NULL,
271
+ result_json TEXT,
272
+ is_error INTEGER,
273
+ started_at INTEGER NOT NULL,
274
+ ended_at INTEGER,
275
+ PRIMARY KEY (turn_id, tool_call_id)
276
+ );
277
+ CREATE INDEX IF NOT EXISTS effects_idempotency_key ON effects(idempotency_key);
278
+ -- One node's calls to one tool, oldest turn first: the read the duplication
279
+ -- guard makes before every refusable call. Without it SQLite walks every effect
280
+ -- in the store and sorts the matches, so the refusal path costs more as the
281
+ -- project's history grows. The columns are the guard's own filters in its own
282
+ -- order - the node and tool are equalities, the turn is the range that bounds
283
+ -- its recovery window - so the ORDER BY ends on seq and is the index walk
284
+ -- rather than a temp b-tree. What is left (the state disjunction and is_error)
285
+ -- filters the handful of rows the window holds, which is why it is not in the
286
+ -- key: an equality-shaped index would leave the rows unordered by turn.
287
+ CREATE INDEX IF NOT EXISTS effects_call_history ON effects(node_id, tool_name, turn_id, seq);
288
+
289
+ CREATE TABLE IF NOT EXISTS asks (
290
+ token TEXT PRIMARY KEY,
291
+ node_id TEXT NOT NULL,
292
+ turn_id INTEGER,
293
+ priority INTEGER NOT NULL DEFAULT 0,
294
+ question TEXT NOT NULL,
295
+ default_answer TEXT NOT NULL,
296
+ deadline_at INTEGER NOT NULL,
297
+ state TEXT NOT NULL,
298
+ answer_text TEXT,
299
+ answer_source TEXT,
300
+ asked_at INTEGER NOT NULL,
301
+ answered_at INTEGER
302
+ );
303
+
304
+ CREATE TABLE IF NOT EXISTS inbox (
305
+ msg_id TEXT PRIMARY KEY,
306
+ node_id TEXT NOT NULL,
307
+ kind TEXT NOT NULL,
308
+ body TEXT NOT NULL,
309
+ state TEXT NOT NULL,
310
+ token TEXT,
311
+ recorded_at INTEGER NOT NULL,
312
+ delivered_at INTEGER,
313
+ delivered_tier TEXT,
314
+ delivered_turn_id INTEGER,
315
+ delivered_after_effect_seq INTEGER,
316
+ delivered_before_request_seq INTEGER
317
+ );
318
+ CREATE INDEX IF NOT EXISTS inbox_pending ON inbox(node_id, state);
319
+
320
+ CREATE TABLE IF NOT EXISTS routes (
321
+ msg_id TEXT PRIMARY KEY,
322
+ node_id TEXT NOT NULL,
323
+ token TEXT,
324
+ decided_at INTEGER NOT NULL,
325
+ delivery_state TEXT NOT NULL
326
+ );
327
+
328
+ -- expires_at is kept here, and left unset. The build that shipped schema version 1
329
+ -- writes it by name in its lease INSERT, so a store without the column is a store
330
+ -- that build cannot open. Nothing here reads it or writes it: a lease is
331
+ -- exclusive for as long as its owner answers for the node (see acquireLease), so
332
+ -- the value bounded nothing, and an expiry sitting in it is a number an operator
333
+ -- could read and be wrong about. A version-1 store is relaxed to this shape on
334
+ -- open and its stale values are cleared, so the column says "unset" afterwards.
335
+ CREATE TABLE IF NOT EXISTS leases (
336
+ lease_id TEXT PRIMARY KEY,
337
+ node_id TEXT NOT NULL,
338
+ owner_pid INTEGER NOT NULL,
339
+ epoch INTEGER NOT NULL,
340
+ acquired_at INTEGER NOT NULL,
341
+ heartbeat_at INTEGER NOT NULL,
342
+ expires_at INTEGER,
343
+ released_at INTEGER,
344
+ state TEXT NOT NULL
345
+ );
346
+ CREATE INDEX IF NOT EXISTS leases_node ON leases(node_id, state);
347
+
348
+ CREATE TABLE IF NOT EXISTS supervisors (
349
+ pid INTEGER PRIMARY KEY,
350
+ epoch INTEGER NOT NULL,
351
+ started_at INTEGER NOT NULL,
352
+ heartbeat_at INTEGER NOT NULL,
353
+ state TEXT NOT NULL,
354
+ driver TEXT NOT NULL,
355
+ child_args TEXT NOT NULL,
356
+ pi_version TEXT
357
+ );
358
+
359
+ CREATE TABLE IF NOT EXISTS locks (
360
+ path TEXT PRIMARY KEY,
361
+ owner_node TEXT NOT NULL,
362
+ owner_epoch INTEGER NOT NULL,
363
+ acquired_at INTEGER NOT NULL,
364
+ released_at INTEGER
365
+ );
366
+ `;
367
+
368
+ const now = () => Date.now();
369
+
370
+ /** A fleet root is an absolute directory; a relative one resolves against cwd. */
371
+ export function resolveFleetRoot(root) {
372
+ const absolute = resolve(root);
373
+ if (!isAbsolute(absolute)) throw new Error(`fleet root must be absolute: ${root}`);
374
+ return absolute;
375
+ }
376
+
377
+ export function journalPath(root) {
378
+ return join(resolveFleetRoot(root), JOURNAL_FILE);
379
+ }
380
+
381
+ /**
382
+ * Discovery: `<root>/fleet.sqlite` is the fleet's journal, and that path is
383
+ * recorded inside the file. A copy that travels without its root still says
384
+ * which root it came from; two roots are two fleets and never one store.
385
+ */
386
+
387
+ /** The schema version a store records, or null when it has never been written. */
388
+ function readStoredSchemaVersion(db) {
389
+ const table = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'fleet_meta'").get();
390
+ if (!table) return null;
391
+ const row = db.prepare("SELECT value FROM fleet_meta WHERE key = 'schema_version'").get();
392
+ const version = Number(row?.value);
393
+ return Number.isFinite(version) ? version : null;
394
+ }
395
+
396
+ /**
397
+ * Relax `leases.expires_at` from NOT NULL to nullable, clearing every value, so
398
+ * that a store written by the version-1 build opens here and that build's own
399
+ * lease write keeps working after a rollback.
400
+ *
401
+ * SQLite cannot change a column in place, so the table is rebuilt once: every
402
+ * row and the index come with it. The values do not - a stale expiry is the one
403
+ * thing this column must not carry, because the column says nothing about the
404
+ * lease and a number in it reads like something that does.
405
+ */
406
+ function relaxLeaseExpiry(db) {
407
+ const column = db
408
+ .prepare("PRAGMA table_info(leases)")
409
+ .all()
410
+ .find((entry) => entry.name === "expires_at");
411
+ if (!column || column.notnull === 0) return false;
412
+ db.exec("BEGIN IMMEDIATE");
413
+ try {
414
+ db.exec(`CREATE TABLE leases_relaxed (
415
+ lease_id TEXT PRIMARY KEY,
416
+ node_id TEXT NOT NULL,
417
+ owner_pid INTEGER NOT NULL,
418
+ epoch INTEGER NOT NULL,
419
+ acquired_at INTEGER NOT NULL,
420
+ heartbeat_at INTEGER NOT NULL,
421
+ expires_at INTEGER,
422
+ released_at INTEGER,
423
+ state TEXT NOT NULL
424
+ )`);
425
+ db.exec(`INSERT INTO leases_relaxed (lease_id, node_id, owner_pid, epoch, acquired_at, heartbeat_at, expires_at, released_at, state)
426
+ SELECT lease_id, node_id, owner_pid, epoch, acquired_at, heartbeat_at, NULL, released_at, state FROM leases`);
427
+ db.exec("DROP TABLE leases");
428
+ db.exec("ALTER TABLE leases_relaxed RENAME TO leases");
429
+ db.exec("CREATE INDEX IF NOT EXISTS leases_node ON leases(node_id, state)");
430
+ db.exec("COMMIT");
431
+ } catch (error) {
432
+ db.exec("ROLLBACK");
433
+ throw error;
434
+ }
435
+ return true;
436
+ }
437
+
438
+ export function openJournal(root, options = {}) {
439
+ const fleetRoot = resolveFleetRoot(root);
440
+ mkdirSync(dirname(journalPath(fleetRoot)), { recursive: true });
441
+ const db = new DatabaseSync(journalPath(fleetRoot));
442
+ // The wait goes first. Every statement below can block on another process's
443
+ // write, and a busy handler installed after them never applies to them: setting
444
+ // the journal mode on a store another process is writing raises
445
+ // "database is locked" immediately, because at that point this connection has
446
+ // no handler and SQLite cannot wait. Measured: a two-drainer race failed about
447
+ // one run in twenty at this line, before the timeout was ever in force.
448
+ db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`);
449
+ // The mode is a property of the file, not of this connection. Asking first
450
+ // means a reader never has to take the exclusive lock that changing the mode
451
+ // needs, and on a store that is already WAL the answer costs one pragma read.
452
+ // The mode is a property of the file, not of this connection, so ask before
453
+ // setting it: a reader never needs the exclusive lock that changing it takes.
454
+ const isWal = () => String(Object.values(db.prepare("PRAGMA journal_mode").get() ?? {})[0] ?? "").toLowerCase() === "wal";
455
+ if (!isWal()) {
456
+ // Setting the mode needs an exclusive lock, and SQLite answers SQLITE_BUSY
457
+ // for that statement without consulting the busy handler, so waiting on it is
458
+ // not available. Two processes initialising one fresh store therefore race
459
+ // and one of them throws. Retry a bounded number of times, and treat "someone
460
+ // else already made it WAL" as success.
461
+ const deadline = Date.now() + BUSY_TIMEOUT_MS;
462
+ for (let attempt = 0; ; attempt += 1) {
463
+ try {
464
+ db.exec("PRAGMA journal_mode = WAL");
465
+ break;
466
+ } catch (error) {
467
+ if (isWal()) break;
468
+ if (Date.now() >= deadline) throw error;
469
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.min(50 * (attempt + 1), 250));
470
+ }
471
+ }
472
+ }
473
+ db.exec("PRAGMA synchronous = FULL");
474
+ // Before the DDL, because the DDL is written for this version's shape and a
475
+ // store from a newer build has one this code does not know.
476
+ const storedVersion = readStoredSchemaVersion(db);
477
+ if (storedVersion != null && storedVersion > SCHEMA_VERSION) {
478
+ throw new Error(
479
+ `fleet store ${journalPath(fleetRoot)} was written by a newer build (schema ${storedVersion}; this build speaks ${SCHEMA_VERSION}): open it with that build, or move it aside and start a new root`,
480
+ );
481
+ }
482
+ db.exec(DDL);
483
+ relaxLeaseExpiry(db);
484
+ const projectRoot = options.projectRoot ? resolve(options.projectRoot) : fleetRoot;
485
+ const meta = {
486
+ store_path: journalPath(fleetRoot),
487
+ schema_version: String(SCHEMA_VERSION),
488
+ project_root: projectRoot,
489
+ driver: options.driver ?? "unset",
490
+ child_args: JSON.stringify(options.childArgs ?? []),
491
+ pi_version: options.piVersion ?? "unset",
492
+ created_at: String(now()),
493
+ };
494
+ const insertMeta = db.prepare("INSERT OR IGNORE INTO fleet_meta(key, value) VALUES (?, ?)");
495
+ for (const [key, value] of Object.entries(meta)) insertMeta.run(key, value);
496
+ // A store opened here records the shape it now has. The insert above ignores a
497
+ // row that already exists, so a store migrated from an older version needs the
498
+ // value written rather than inserted.
499
+ if (storedVersion !== SCHEMA_VERSION) {
500
+ db.prepare("UPDATE fleet_meta SET value = ? WHERE key = 'schema_version'").run(String(SCHEMA_VERSION));
501
+ }
502
+ db.prepare("INSERT OR IGNORE INTO projects(project_root, created_at) VALUES (?, ?)").run(projectRoot, now());
503
+ return new Journal(db, fleetRoot);
504
+ }
505
+
506
+ export class Journal {
507
+ constructor(db, root) {
508
+ this.db = db;
509
+ this.root = root;
510
+ this.path = journalPath(root);
511
+ this._cache = new Map();
512
+ this.busyTimeoutMs = BUSY_TIMEOUT_MS;
513
+ }
514
+
515
+ close() {
516
+ this.db.close();
517
+ }
518
+
519
+ /**
520
+ * Run `body` as one write transaction.
521
+ *
522
+ * `BEGIN IMMEDIATE` takes the write lock before the body reads, so two
523
+ * processes that both intend to act on the same rows serialize: the second
524
+ * one's read happens after the first one's commit, rather than against the
525
+ * state the first one was about to change. A body that throws is rolled back
526
+ * whole, which is what lets a caller treat several statements as one edit.
527
+ *
528
+ * `busyTimeoutMs` is the wait this transaction may spend acquiring that
529
+ * lock, and it is per-connection state, so it is restored afterwards: a
530
+ * caller with a bounded window of its own (the drain, whose retry loop is the
531
+ * policy) does not leave every later statement in the process with it.
532
+ */
533
+ transaction(body, { busyTimeoutMs } = {}) {
534
+ const previous = this.busyTimeoutMs;
535
+ if (busyTimeoutMs != null && busyTimeoutMs !== previous) {
536
+ this.db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.floor(busyTimeoutMs))}`);
537
+ this.busyTimeoutMs = Math.max(0, Math.floor(busyTimeoutMs));
538
+ }
539
+ try {
540
+ this.db.exec("BEGIN IMMEDIATE");
541
+ try {
542
+ const result = body();
543
+ this.db.exec("COMMIT");
544
+ return result;
545
+ } catch (error) {
546
+ this.db.exec("ROLLBACK");
547
+ throw error;
548
+ }
549
+ } finally {
550
+ if (this.busyTimeoutMs !== previous) {
551
+ this.db.exec(`PRAGMA busy_timeout = ${previous}`);
552
+ this.busyTimeoutMs = previous;
553
+ }
554
+ }
555
+ }
556
+
557
+ /**
558
+ * Run `body` with a statement wait of `ms`, restoring the previous one after.
559
+ *
560
+ * The wait is per-connection state, and a caller with a bounded budget of its
561
+ * own (the tick's maintenance passes, which must not block on a lock another
562
+ * process may hold for as long as it likes) sets it here rather than leaving
563
+ * every later statement in the process with it.
564
+ */
565
+ withBusyTimeout(ms, body) {
566
+ const previous = this.busyTimeoutMs;
567
+ const bounded = Math.max(0, Math.floor(ms));
568
+ if (bounded !== previous) {
569
+ this.db.exec(`PRAGMA busy_timeout = ${bounded}`);
570
+ this.busyTimeoutMs = bounded;
571
+ }
572
+ try {
573
+ return body();
574
+ } finally {
575
+ if (bounded !== previous) {
576
+ this.db.exec(`PRAGMA busy_timeout = ${previous}`);
577
+ this.busyTimeoutMs = previous;
578
+ }
579
+ }
580
+ }
581
+
582
+ _stmt(sql) {
583
+ let stmt = this._cache.get(sql);
584
+ if (!stmt) {
585
+ stmt = this.db.prepare(sql);
586
+ this._cache.set(sql, stmt);
587
+ }
588
+ return stmt;
589
+ }
590
+
591
+ meta() {
592
+ const rows = this._stmt("SELECT key, value FROM fleet_meta").all();
593
+ return Object.fromEntries(rows.map((row) => [row.key, row.value]));
594
+ }
595
+
596
+ setMeta(key, value) {
597
+ this._stmt("INSERT INTO fleet_meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(
598
+ key,
599
+ String(value),
600
+ );
601
+ }
602
+
603
+ /** Drop a meta key, for a fact that stops being true (a deferral that recovered). */
604
+ clearMeta(key) {
605
+ this._stmt("DELETE FROM fleet_meta WHERE key = ?").run(key);
606
+ }
607
+
608
+ // ── nodes ────────────────────────────────────────────────────────────────
609
+
610
+ addNode(node) {
611
+ this._stmt(
612
+ `INSERT INTO nodes (node_id, parent_id, project_root, charter_path, driver, driver_config,
613
+ state, attempts, turn_count, token_cap, spend_cap_usd, state_changed_at, created_at)
614
+ VALUES (?, ?, ?, ?, ?, ?, ?, 0, 0, ?, ?, ?, ?)`,
615
+ ).run(
616
+ node.nodeId,
617
+ node.parentId ?? null,
618
+ resolve(node.projectRoot),
619
+ node.charterPath,
620
+ node.driver ?? "rpc",
621
+ JSON.stringify(node.driverConfig ?? {}),
622
+ node.state ?? "settled",
623
+ node.tokenCap ?? null,
624
+ node.spendCapUsd ?? null,
625
+ now(),
626
+ now(),
627
+ );
628
+ return this.getNode(node.nodeId);
629
+ }
630
+
631
+ getNode(nodeId) {
632
+ return this._stmt("SELECT * FROM nodes WHERE node_id = ?").get(nodeId) ?? null;
633
+ }
634
+
635
+ listNodes() {
636
+ return this._stmt("SELECT * FROM nodes ORDER BY created_at, node_id").all();
637
+ }
638
+
639
+ setNodeState(nodeId, state, extra = {}) {
640
+ const fields = ["state = ?", "state_changed_at = ?"];
641
+ const values = [state, now()];
642
+ for (const [key, value] of Object.entries(extra)) {
643
+ fields.push(`${key} = ?`);
644
+ values.push(value);
645
+ }
646
+ values.push(nodeId);
647
+ this._stmt(`UPDATE nodes SET ${fields.join(", ")} WHERE node_id = ?`).run(...values);
648
+ }
649
+
650
+ setNodeSession(nodeId, { sessionId, sessionFile }) {
651
+ this._stmt("UPDATE nodes SET session_id = ?, session_file = ? WHERE node_id = ?").run(
652
+ sessionId,
653
+ sessionFile,
654
+ nodeId,
655
+ );
656
+ }
657
+
658
+ addNodeUsage(nodeId, { tokensIn = 0, tokensOut = 0, spendUsd = 0, projectRoot }) {
659
+ this._stmt(
660
+ "UPDATE nodes SET tokens_in = tokens_in + ?, tokens_out = tokens_out + ?, spend_usd = spend_usd + ? WHERE node_id = ?",
661
+ ).run(tokensIn, tokensOut, spendUsd, nodeId);
662
+ if (projectRoot) {
663
+ this._stmt(
664
+ "UPDATE projects SET tokens_in = tokens_in + ?, tokens_out = tokens_out + ?, spend_usd = spend_usd + ? WHERE project_root = ?",
665
+ ).run(tokensIn, tokensOut, spendUsd, projectRoot);
666
+ }
667
+ }
668
+
669
+ project(projectRoot) {
670
+ return (
671
+ this._stmt("SELECT * FROM projects WHERE project_root = ?").get(projectRoot) ?? {
672
+ project_root: projectRoot,
673
+ tokens_in: 0,
674
+ tokens_out: 0,
675
+ spend_usd: 0,
676
+ token_cap: null,
677
+ spend_cap_usd: null,
678
+ }
679
+ );
680
+ }
681
+
682
+ /** A budget refusal is a decision, so it reads both caps and returns a reason. */
683
+ budgetRefusal(projectRoot, nodeId) {
684
+ const node = this.getNode(nodeId);
685
+ const project = this.project(projectRoot);
686
+ const over = (used, cap, label) =>
687
+ cap != null && used >= cap ? `${label} spent ${Math.round(used)} of cap ${Math.round(cap)}` : null;
688
+ return (
689
+ over(node.tokens_in + node.tokens_out, node.token_cap, "node tokens") ??
690
+ over(node.spend_usd, node.spend_cap_usd, "node spend") ??
691
+ over(project.tokens_in + project.tokens_out, project.token_cap, "project tokens") ??
692
+ over(project.spend_usd, project.spend_cap_usd, "project spend")
693
+ );
694
+ }
695
+
696
+ // ── turns ────────────────────────────────────────────────────────────────
697
+
698
+ /**
699
+ * A turn is not an attempt. `attempts` counts recoveries and failures, so a
700
+ * node that works steadily never accumulates one; the turn carries the
701
+ * attempt number it was started under for the reader.
702
+ */
703
+ beginTurn(nodeId, { kind, promptText, sessionId, supervisorEpoch }) {
704
+ const node = this.getNode(nodeId);
705
+ const attempt = node?.attempts ?? 0;
706
+ this._stmt("UPDATE nodes SET turn_count = turn_count + 1 WHERE node_id = ?").run(nodeId);
707
+ const info = this._stmt(
708
+ `INSERT INTO turns (node_id, attempt, kind, state, prompt_text, session_id, supervisor_epoch, started_at)
709
+ VALUES (?, ?, ?, 'in_progress', ?, ?, ?, ?)`,
710
+ ).run(nodeId, attempt, kind, promptText, sessionId ?? null, supervisorEpoch ?? null, now());
711
+ return Number(info.lastInsertRowid);
712
+ }
713
+
714
+ turn(turnId) {
715
+ return this._stmt("SELECT * FROM turns WHERE turn_id = ?").get(turnId) ?? null;
716
+ }
717
+
718
+ turnsFor(nodeId) {
719
+ return this._stmt("SELECT * FROM turns WHERE node_id = ? ORDER BY turn_id").all(nodeId);
720
+ }
721
+
722
+ bumpAttempt(nodeId) {
723
+ this._stmt("UPDATE nodes SET attempts = attempts + 1 WHERE node_id = ?").run(nodeId);
724
+ }
725
+
726
+ openTurns(nodeId) {
727
+ return this._stmt("SELECT * FROM turns WHERE node_id = ? AND state = 'in_progress'").all(nodeId);
728
+ }
729
+
730
+ allOpenTurns() {
731
+ return this._stmt("SELECT * FROM turns WHERE state = 'in_progress'").all();
732
+ }
733
+
734
+ setTurnChild(turnId, { pid, pgid }) {
735
+ this._stmt("UPDATE turns SET child_pid = ?, child_pgid = ? WHERE turn_id = ?").run(pid, pgid, turnId);
736
+ }
737
+
738
+ endTurn(turnId, { state, tokensIn = 0, tokensOut = 0, spendUsd = 0, sessionId }) {
739
+ this._stmt(
740
+ `UPDATE turns SET state = ?, ended_at = ?, tokens_in = ?, tokens_out = ?, spend_usd = ?,
741
+ session_id = COALESCE(?, session_id) WHERE turn_id = ?`,
742
+ ).run(state, now(), tokensIn, tokensOut, spendUsd, sessionId ?? null, turnId);
743
+ }
744
+
745
+ recordModelRequest(turnId, { messageCount, toolCount, tokensIn = 0, tokensOut = 0 }) {
746
+ const row = this._stmt("SELECT model_requests AS n FROM turns WHERE turn_id = ?").get(turnId);
747
+ const seq = (row?.n ?? 0) + 1;
748
+ this._stmt(
749
+ "INSERT INTO model_requests (turn_id, seq, message_count, tool_count, tokens_in, tokens_out, at) VALUES (?, ?, ?, ?, ?, ?, ?)",
750
+ ).run(turnId, seq, messageCount, toolCount, tokensIn, tokensOut, now());
751
+ this._stmt("UPDATE turns SET model_requests = ? WHERE turn_id = ?").run(seq, turnId);
752
+ return seq;
753
+ }
754
+
755
+ modelRequests(turnId) {
756
+ return this._stmt("SELECT * FROM model_requests WHERE turn_id = ? ORDER BY seq").all(turnId);
757
+ }
758
+
759
+ // ── effects ──────────────────────────────────────────────────────────────
760
+
761
+ /**
762
+ * Record the attempt before the tool runs. Two calls with the same identity
763
+ * collapse into the first row (`INSERT OR IGNORE`): a replayed event stream
764
+ * cannot invent a second attempt.
765
+ */
766
+ effectAttempted({ turnId, nodeId, toolCallId, toolName, args, replay, seq }) {
767
+ const idempotencyKey = effectKey({ nodeId, turnId, toolCallId });
768
+ this._stmt(
769
+ `INSERT OR IGNORE INTO effects (turn_id, node_id, tool_call_id, seq, tool_name, args_json,
770
+ replay, idempotency_key, state, started_at)
771
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'attempted', ?)`,
772
+ ).run(
773
+ turnId,
774
+ nodeId,
775
+ toolCallId,
776
+ seq ?? 0,
777
+ toolName,
778
+ JSON.stringify(args ?? null),
779
+ replay,
780
+ idempotencyKey,
781
+ now(),
782
+ );
783
+ return idempotencyKey;
784
+ }
785
+
786
+ effectSettled({ turnId, toolCallId, result, isError }) {
787
+ this._stmt(
788
+ "UPDATE effects SET state = 'completed', result_json = ?, is_error = ?, ended_at = ? WHERE turn_id = ? AND tool_call_id = ? AND state = 'attempted'",
789
+ ).run(JSON.stringify(result ?? null), isError ? 1 : 0, now(), turnId, toolCallId);
790
+ }
791
+
792
+ /**
793
+ * An attempt with no end is the crash residue: the effect may or may not
794
+ * have reached the world. It stays visible with `state = 'unknown'` so the
795
+ * next turn is told about it instead of quietly repeating it.
796
+ *
797
+ * This state is inside the duplication guard, and the note says so. An
798
+ * `unknown` row counts as history: a call it matches is refused before it
799
+ * runs, because an effect that may have reached the world is the one class
800
+ * where repeating is dangerous and the record cannot settle it. The guard's
801
+ * window is the run being resumed - its current turn back to the last turn
802
+ * that ended normally - so this is not "never again": a turn that finished its
803
+ * work retires the run, and an operator retires a single row.
804
+ *
805
+ * Two instruments cover this class and they must agree: the guard refuses the
806
+ * call, and `recoveryPrompt` in supervisor.mjs names every unknown effect with
807
+ * the idempotency key that identifies it and tells the node the guard will
808
+ * refuse a repeat. The interrupted turn this row belongs to is never a settled
809
+ * turn, so it is inside the window of the turn that resumes it by
810
+ * construction; a note that promised a refusal the guard does not make, or a
811
+ * guard that refused without the note naming why, would each be half a fact.
812
+ */
813
+ markUnknownEffects(turnId) {
814
+ const rows = this.attemptedEffects(turnId);
815
+ this._stmt("UPDATE effects SET state = 'unknown' WHERE turn_id = ? AND state = 'attempted'").run(turnId);
816
+ return rows;
817
+ }
818
+
819
+ attemptedEffects(turnId) {
820
+ return this._stmt("SELECT * FROM effects WHERE turn_id = ? AND state = 'attempted' ORDER BY seq").all(turnId);
821
+ }
822
+
823
+ effects(turnId) {
824
+ return this._stmt("SELECT * FROM effects WHERE turn_id = ? ORDER BY seq").all(turnId);
825
+ }
826
+
827
+ /** Effects a resumed turn must know about, in journal order. */
828
+ settledEffects(turnId) {
829
+ return this._stmt(
830
+ "SELECT * FROM effects WHERE turn_id = ? AND state = 'completed' ORDER BY seq",
831
+ ).all(turnId);
832
+ }
833
+
834
+ effectsByKey(idempotencyKey) {
835
+ return this._stmt("SELECT * FROM effects WHERE idempotency_key = ? ORDER BY turn_id").all(idempotencyKey);
836
+ }
837
+
838
+ // ── asks and inbox ───────────────────────────────────────────────────────
839
+
840
+ recordAsk(ask) {
841
+ this._stmt(
842
+ `INSERT INTO asks (token, node_id, turn_id, priority, question, default_answer, deadline_at, state, asked_at)
843
+ VALUES (?, ?, ?, ?, ?, ?, ?, 'open', ?)`,
844
+ ).run(
845
+ ask.token,
846
+ ask.nodeId,
847
+ ask.turnId ?? null,
848
+ Number(ask.priority ?? 0),
849
+ ask.question,
850
+ ask.defaultAnswer,
851
+ ask.deadlineAt,
852
+ now(),
853
+ );
854
+ return this.getAsk(ask.token);
855
+ }
856
+
857
+ getAsk(token) {
858
+ return this._stmt("SELECT * FROM asks WHERE token = ?").get(token) ?? null;
859
+ }
860
+
861
+ /**
862
+ * Open asks, most urgent first. Priority orders the head's attention; it
863
+ * never changes the deadline, because a high priority that postponed the
864
+ * default would be a park with extra steps.
865
+ */
866
+ openAsks(nodeId) {
867
+ return nodeId
868
+ ? this._stmt("SELECT * FROM asks WHERE node_id = ? AND state = 'open' ORDER BY priority DESC, asked_at").all(nodeId)
869
+ : this._stmt("SELECT * FROM asks WHERE state = 'open' ORDER BY priority DESC, asked_at").all();
870
+ }
871
+
872
+ resolveAsk(token, { answerText, answerSource }) {
873
+ this._stmt(
874
+ "UPDATE asks SET state = 'answered', answer_text = ?, answer_source = ?, answered_at = ? WHERE token = ? AND state = 'open'",
875
+ ).run(answerText, answerSource, now(), token);
876
+ return this.getAsk(token);
877
+ }
878
+
879
+ /**
880
+ * Parked work whose deadline passed takes its default. This is what keeps a
881
+ * deadline from being decoration: the answer source says nobody answered.
882
+ *
883
+ * Returns only the asks THIS call claimed, and the claim is the conditional
884
+ * UPDATE rather than the read. Two supervisors that both read the same
885
+ * overdue rows must not both report them: each caller queues one message per
886
+ * returned row, so a shared read set gives a parked node two copies of its
887
+ * default in one prompt. `changes` is what decides, so the loser of the race
888
+ * returns nothing and queues nothing.
889
+ */
890
+ expireAsks(at = now()) {
891
+ const claimed = [];
892
+ for (const ask of this.openAsks().filter((ask) => ask.deadline_at <= at)) {
893
+ const result = this._stmt(
894
+ "UPDATE asks SET state = 'answered', answer_text = ?, answer_source = ?, answered_at = ? WHERE token = ? AND state = 'open'",
895
+ ).run(ask.default_answer, "deadline-default", now(), ask.token);
896
+ if (result.changes > 0) {
897
+ claimed.push({
898
+ ...ask,
899
+ state: "answered",
900
+ answer_text: ask.default_answer,
901
+ answer_source: "deadline-default",
902
+ });
903
+ }
904
+ }
905
+ return claimed;
906
+ }
907
+
908
+ addMessage(msg) {
909
+ this._stmt(
910
+ `INSERT INTO inbox (msg_id, node_id, kind, body, state, token, recorded_at)
911
+ VALUES (?, ?, ?, ?, 'recorded', ?, ?)`,
912
+ ).run(msg.msgId, msg.nodeId, msg.kind, msg.body, msg.token ?? null, now());
913
+ this._stmt("INSERT OR REPLACE INTO routes (msg_id, node_id, token, decided_at, delivery_state) VALUES (?, ?, ?, ?, 'pending')").run(
914
+ msg.msgId,
915
+ msg.nodeId,
916
+ msg.token ?? null,
917
+ now(),
918
+ );
919
+ return this.getMessage(msg.msgId);
920
+ }
921
+
922
+ getMessage(msgId) {
923
+ return this._stmt("SELECT * FROM inbox WHERE msg_id = ?").get(msgId) ?? null;
924
+ }
925
+
926
+ pendingMessages(nodeId) {
927
+ return this._stmt(
928
+ "SELECT * FROM inbox WHERE node_id = ? AND state = 'recorded' ORDER BY recorded_at, msg_id",
929
+ ).all(nodeId);
930
+ }
931
+
932
+ markMessageDelivered(msgId, { tier, turnId, afterEffectSeq, beforeRequestSeq }) {
933
+ this._stmt(
934
+ `UPDATE inbox SET state = 'delivered', delivered_at = ?, delivered_tier = ?, delivered_turn_id = ?,
935
+ delivered_after_effect_seq = ?, delivered_before_request_seq = ? WHERE msg_id = ?`,
936
+ ).run(now(), tier, turnId ?? null, afterEffectSeq ?? null, beforeRequestSeq ?? null, msgId);
937
+ this._stmt("UPDATE routes SET delivery_state = ? WHERE msg_id = ?").run(tier === "A" ? "steered" : "delivered", msgId);
938
+ }
939
+
940
+ /**
941
+ * Retire every delivered row a node holds.
942
+ *
943
+ * Node-wide, so it cannot tell a message a prompt carried from one it did
944
+ * not: the settle gate uses `consumeMessages` with the turn's own list, and
945
+ * this remains the blunt instrument for callers that mean all of them.
946
+ */
947
+ markMessagesConsumed(nodeId) {
948
+ this._stmt("UPDATE inbox SET state = 'consumed' WHERE node_id = ? AND state = 'delivered'").run(nodeId);
949
+ }
950
+
951
+ /**
952
+ * Retire exactly the messages a turn carried.
953
+ *
954
+ * The ids are the turn's own account of what its prompt contained, so this
955
+ * consumes what a model read and nothing else: a row that reached `delivered`
956
+ * another way stays where it is, for the next turn that actually carries it.
957
+ */
958
+ consumeMessages(msgIds) {
959
+ let consumed = 0;
960
+ for (const msgId of msgIds) {
961
+ consumed += this._stmt("UPDATE inbox SET state = 'consumed' WHERE msg_id = ? AND state = 'delivered'").run(msgId).changes;
962
+ }
963
+ return consumed;
964
+ }
965
+
966
+ /**
967
+ * Put delivered messages back in the queue.
968
+ *
969
+ * A turn that never settled sent its prompt and did not finish: the body
970
+ * reached a model and nothing acted on it, so the delivery columns are
971
+ * cleared and the next turn carries it again. For an operator's instruction
972
+ * the safe side to err on is at-least-once, and the recovery prompt is the
973
+ * instrument that keeps the repetition visible.
974
+ */
975
+ releaseMessages(msgIds) {
976
+ let released = 0;
977
+ for (const msgId of msgIds) {
978
+ released += this._stmt(
979
+ `UPDATE inbox SET state = 'recorded', delivered_at = NULL, delivered_tier = NULL, delivered_turn_id = NULL,
980
+ delivered_after_effect_seq = NULL, delivered_before_request_seq = NULL
981
+ WHERE msg_id = ? AND state = 'delivered'`,
982
+ ).run(msgId).changes;
983
+ }
984
+ return released;
985
+ }
986
+
987
+ /**
988
+ * The terminal state for a message no turn will ever carry: the node it was
989
+ * routed to is finished, or failed at its attempt bound, so the promise a
990
+ * `recorded` row makes cannot be kept. Only the rows this call transitions
991
+ * are returned, so two sweepers do not both report the same message.
992
+ */
993
+ markMessagesUndeliverable(nodeId) {
994
+ const claimed = [];
995
+ for (const message of this.pendingMessages(nodeId)) {
996
+ const result = this._stmt("UPDATE inbox SET state = 'undeliverable' WHERE msg_id = ? AND state = 'recorded'").run(
997
+ message.msg_id,
998
+ );
999
+ if (result.changes > 0) claimed.push({ ...message, state: "undeliverable" });
1000
+ }
1001
+ return claimed;
1002
+ }
1003
+
1004
+ messages(nodeId) {
1005
+ return nodeId
1006
+ ? this._stmt("SELECT * FROM inbox WHERE node_id = ? ORDER BY recorded_at, msg_id").all(nodeId)
1007
+ : this._stmt("SELECT * FROM inbox ORDER BY recorded_at, msg_id").all();
1008
+ }
1009
+
1010
+ // ── leases, supervisors, locks ───────────────────────────────────────────
1011
+
1012
+ nextEpoch() {
1013
+ const row = this._stmt("SELECT MAX(epoch) AS epoch FROM supervisors").get();
1014
+ return Number(row?.epoch ?? 0) + 1;
1015
+ }
1016
+
1017
+ registerSupervisor({ pid, driver, childArgs, piVersion }) {
1018
+ const epoch = this.nextEpoch();
1019
+ this._stmt(
1020
+ "INSERT INTO supervisors (pid, epoch, started_at, heartbeat_at, state, driver, child_args, pi_version) VALUES (?, ?, ?, ?, 'live', ?, ?, ?)",
1021
+ ).run(pid, epoch, now(), now(), driver, JSON.stringify(childArgs ?? []), piVersion ?? null);
1022
+ return epoch;
1023
+ }
1024
+
1025
+ heartbeatSupervisor(pid) {
1026
+ this._stmt("UPDATE supervisors SET heartbeat_at = ? WHERE pid = ?").run(now(), pid);
1027
+ }
1028
+
1029
+ setSupervisorState(pid, state) {
1030
+ this._stmt("UPDATE supervisors SET state = ? WHERE pid = ?").run(state, pid);
1031
+ }
1032
+
1033
+ supervisors() {
1034
+ return this._stmt("SELECT * FROM supervisors ORDER BY epoch").all();
1035
+ }
1036
+
1037
+ /**
1038
+ * Take or renew this epoch's lease on one node.
1039
+ *
1040
+ * Exclusivity is the owner answering for the node, not a clock. A lease lives
1041
+ * as long as its owner keeps renewing it, and a predecessor's lease ends in
1042
+ * reconciliation, which asks whether that owner is still there before it
1043
+ * touches anything.
1044
+ *
1045
+ * There is no expiry here, and the stored column of that name is deliberately
1046
+ * neither read nor written: a wall-clock bound was measured into disrepute (one
1047
+ * legitimate tick ran 32.7s under write contention, so any TTL short enough to
1048
+ * name lapsed mid-turn, and a lapsed lease a second supervisor may claim puts
1049
+ * one node under two drivers). Reconciliation refuses to fence a live owner's
1050
+ * turn for the same reason: a clock cannot see whether a child is still
1051
+ * working. The column survives only so the build that shipped it can still
1052
+ * open the store - see the leases DDL.
1053
+ */
1054
+ acquireLease({ nodeId, ownerPid, epoch }) {
1055
+ const existing = this.liveLease(nodeId);
1056
+ if (existing && existing.epoch !== epoch) return null;
1057
+ const leaseId = `${nodeId}:${epoch}`;
1058
+ this._stmt(
1059
+ `INSERT INTO leases (lease_id, node_id, owner_pid, epoch, acquired_at, heartbeat_at, state)
1060
+ VALUES (?, ?, ?, ?, ?, ?, 'live')
1061
+ ON CONFLICT(lease_id) DO UPDATE SET heartbeat_at = excluded.heartbeat_at, state = 'live'`,
1062
+ ).run(leaseId, nodeId, ownerPid, epoch, now(), now());
1063
+ return this._stmt("SELECT * FROM leases WHERE lease_id = ?").get(leaseId);
1064
+ }
1065
+
1066
+ liveLease(nodeId) {
1067
+ return (
1068
+ this._stmt("SELECT * FROM leases WHERE node_id = ? AND state = 'live' ORDER BY epoch DESC LIMIT 1").get(nodeId) ??
1069
+ null
1070
+ );
1071
+ }
1072
+
1073
+ liveLeases() {
1074
+ return this._stmt("SELECT * FROM leases WHERE state = 'live' ORDER BY node_id").all();
1075
+ }
1076
+
1077
+ releaseLease(nodeId, epoch) {
1078
+ this._stmt("UPDATE leases SET state = 'released', released_at = ? WHERE lease_id = ?").run(now(), `${nodeId}:${epoch}`);
1079
+ }
1080
+
1081
+ expireLease(leaseId) {
1082
+ this._stmt("UPDATE leases SET state = 'expired', released_at = ? WHERE lease_id = ?").run(now(), leaseId);
1083
+ }
1084
+
1085
+ acquireLock({ path, ownerNode, ownerEpoch }) {
1086
+ const held = this._stmt("SELECT * FROM locks WHERE path = ? AND released_at IS NULL").get(path);
1087
+ if (held && held.owner_node !== ownerNode) return null;
1088
+ this._stmt(
1089
+ `INSERT INTO locks (path, owner_node, owner_epoch, acquired_at) VALUES (?, ?, ?, ?)
1090
+ ON CONFLICT(path) DO UPDATE SET owner_node = excluded.owner_node, owner_epoch = excluded.owner_epoch,
1091
+ acquired_at = excluded.acquired_at, released_at = NULL`,
1092
+ ).run(path, ownerNode, ownerEpoch, now());
1093
+ return this._stmt("SELECT * FROM locks WHERE path = ?").get(path);
1094
+ }
1095
+
1096
+ releaseLocks(ownerNode) {
1097
+ this._stmt("UPDATE locks SET released_at = ? WHERE owner_node = ? AND released_at IS NULL").run(now(), ownerNode);
1098
+ }
1099
+
1100
+ locks() {
1101
+ return this._stmt("SELECT * FROM locks ORDER BY path").all();
1102
+ }
1103
+ }
1104
+
1105
+ /**
1106
+ * The effect's identity across a crash. A resumed turn that re-issues the same
1107
+ * tool call inside the same turn produces this key again, and the world can
1108
+ * dedupe on it; a naive resume that starts a fresh turn gets a different key,
1109
+ * which is exactly why the journal cannot be the only instrument measuring
1110
+ * duplication and the drill counts the world file too.
1111
+ */
1112
+ export function effectKey({ nodeId, turnId, toolCallId }) {
1113
+ return createHash("sha256").update(`${nodeId}\n${turnId}\n${toolCallId}`).digest("hex").slice(0, 32);
1114
+ }
1115
+
1116
+ /**
1117
+ * Replay policy per tool name, recorded before the attempt. `safe` means the
1118
+ * call can be run again without a second effect; everything else is `never`
1119
+ * until its end event lands, because the journal observed the effect rather
1120
+ * than owning it.
1121
+ */
1122
+ export function replayPolicy(toolName) {
1123
+ return /^(read|ls|find|grep|glob|list|view|get_state|get_messages)$/i.test(toolName ?? "") ? "safe" : "never";
1124
+ }