rollbridge 0.1.70 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -63
- package/TODO.md +0 -13
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +65 -387
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +22 -212
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -435
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +5 -119
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +21 -125
- package/test/owner-replacement.test.js +14 -175
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +72 -737
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
|
@@ -5,7 +5,7 @@ import os from "node:os"
|
|
|
5
5
|
import path from "node:path"
|
|
6
6
|
import {describe, expect, test} from "@velocious/testing"
|
|
7
7
|
import {fileURLToPath} from "node:url"
|
|
8
|
-
import ManagedProcess
|
|
8
|
+
import ManagedProcess from "../src/managed-process.js"
|
|
9
9
|
|
|
10
10
|
describe("managed-process", () => {
|
|
11
11
|
|
|
@@ -308,255 +308,6 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
|
|
|
308
308
|
}
|
|
309
309
|
})
|
|
310
310
|
|
|
311
|
-
test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
|
|
312
|
-
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-wedge-"))
|
|
313
|
-
const readyPath = path.join(dir, "ready")
|
|
314
|
-
/** @type {string[]} */
|
|
315
|
-
const signals = []
|
|
316
|
-
const managed = new ManagedProcess({
|
|
317
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
318
|
-
cwd: undefined,
|
|
319
|
-
env: {},
|
|
320
|
-
id: "worker",
|
|
321
|
-
lifecycle: {drainTimeoutMs: 0},
|
|
322
|
-
logger: () => {},
|
|
323
|
-
outputLines: 50,
|
|
324
|
-
restartDelayMs: 10,
|
|
325
|
-
shouldRestart: () => false,
|
|
326
|
-
stopSignal: "SIGTERM",
|
|
327
|
-
stopTimeoutMs: "indefinite"
|
|
328
|
-
})
|
|
329
|
-
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
330
|
-
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
331
|
-
signals.push(signal)
|
|
332
|
-
await signalProcessGroup(signal, pgid, deadline)
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
try {
|
|
336
|
-
await managed.start()
|
|
337
|
-
// The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
|
|
338
|
-
// child that has not finished booting reaches the default disposition instead, which exits
|
|
339
|
-
// the group and leaves no wedge to observe.
|
|
340
|
-
await waitFor(() => fs.existsSync(readyPath))
|
|
341
|
-
await managed.stop({retirementTimeoutMs: 25})
|
|
342
|
-
|
|
343
|
-
expect(managed.retirementTimedOut).toBe(true)
|
|
344
|
-
expect(signals).toEqual(["SIGTERM"])
|
|
345
|
-
expect(managed.status().pid).toBeDefined()
|
|
346
|
-
} finally {
|
|
347
|
-
const pid = managed.status().pid
|
|
348
|
-
if (pid) process.kill(-pid, "SIGKILL")
|
|
349
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
350
|
-
fs.rmSync(dir, {force: true, recursive: true})
|
|
351
|
-
}
|
|
352
|
-
})
|
|
353
|
-
|
|
354
|
-
test("escalateRetirement terminates the wedged worker the bounded retirement left alive", async () => {
|
|
355
|
-
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalation-wedge-"))
|
|
356
|
-
const readyPath = path.join(dir, "ready")
|
|
357
|
-
/** @type {string[]} */
|
|
358
|
-
const signals = []
|
|
359
|
-
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
360
|
-
const logs = []
|
|
361
|
-
const managed = new ManagedProcess({
|
|
362
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
363
|
-
cwd: undefined,
|
|
364
|
-
env: {},
|
|
365
|
-
id: "worker",
|
|
366
|
-
lifecycle: {drainTimeoutMs: 0},
|
|
367
|
-
logger: (message, data) => logs.push({data, message}),
|
|
368
|
-
outputLines: 50,
|
|
369
|
-
restartDelayMs: 10,
|
|
370
|
-
shouldRestart: () => false,
|
|
371
|
-
stopSignal: "SIGTERM",
|
|
372
|
-
stopTimeoutMs: "indefinite"
|
|
373
|
-
})
|
|
374
|
-
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
375
|
-
|
|
376
|
-
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
377
|
-
signals.push(signal)
|
|
378
|
-
await signalProcessGroup(signal, pgid, deadline)
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
try {
|
|
382
|
-
await managed.start()
|
|
383
|
-
await waitFor(() => fs.existsSync(readyPath))
|
|
384
|
-
await managed.stop({retirementTimeoutMs: 25})
|
|
385
|
-
const wedgedPid = managed.status().pid
|
|
386
|
-
|
|
387
|
-
expect(managed.retirementTimedOut).toBe(true)
|
|
388
|
-
expect(signals).toEqual(["SIGTERM"])
|
|
389
|
-
expect(wedgedPid).toBeDefined()
|
|
390
|
-
|
|
391
|
-
expect(await managed.escalateRetirement({reason: "spec: proven wedge"})).toBe(true)
|
|
392
|
-
expect(signals).toEqual(["SIGTERM", "SIGKILL"])
|
|
393
|
-
expect(logs.some((entry) => entry.message === "process retirement escalated to SIGKILL" && entry.data?.reason === "spec: proven wedge")).toBe(true)
|
|
394
|
-
expect(await managed.waitForProcessGroupExit(/** @type {number} */ (wedgedPid), Date.now() + 5000)).toBe(true)
|
|
395
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
396
|
-
expect(managed.status().pid).toBeUndefined()
|
|
397
|
-
expect(managed.status().state).toBe("stopped")
|
|
398
|
-
} finally {
|
|
399
|
-
const pid = managed.status().pid
|
|
400
|
-
if (pid) process.kill(-pid, "SIGKILL")
|
|
401
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
402
|
-
fs.rmSync(dir, {force: true, recursive: true})
|
|
403
|
-
}
|
|
404
|
-
})
|
|
405
|
-
|
|
406
|
-
test("escalateRetirement refuses a process that has not expired its retirement budget", async () => {
|
|
407
|
-
/** @type {string[]} */
|
|
408
|
-
const signals = []
|
|
409
|
-
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
410
|
-
const logs = []
|
|
411
|
-
const managed = new ManagedProcess({
|
|
412
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
413
|
-
cwd: undefined,
|
|
414
|
-
env: {},
|
|
415
|
-
id: "worker",
|
|
416
|
-
lifecycle: {drainTimeoutMs: 0},
|
|
417
|
-
logger: (message, data) => logs.push({data, message}),
|
|
418
|
-
outputLines: 50,
|
|
419
|
-
restartDelayMs: 10,
|
|
420
|
-
shouldRestart: () => false,
|
|
421
|
-
stopSignal: "SIGTERM",
|
|
422
|
-
stopTimeoutMs: "indefinite"
|
|
423
|
-
})
|
|
424
|
-
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
425
|
-
|
|
426
|
-
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
427
|
-
signals.push(signal)
|
|
428
|
-
await signalProcessGroup(signal, pgid, deadline)
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
try {
|
|
432
|
-
await managed.start()
|
|
433
|
-
await managed.quiesce()
|
|
434
|
-
|
|
435
|
-
expect(managed.retirementTimedOut).toBe(false)
|
|
436
|
-
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
437
|
-
expect(signals).toEqual([])
|
|
438
|
-
expect(logs.some((entry) => entry.message === "retirement escalation refused; process is not past its retirement budget")).toBe(true)
|
|
439
|
-
expect(managed.status().pid).toBeDefined()
|
|
440
|
-
} finally {
|
|
441
|
-
const pid = managed.status().pid
|
|
442
|
-
if (pid) process.kill(-pid, "SIGKILL")
|
|
443
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
444
|
-
}
|
|
445
|
-
})
|
|
446
|
-
|
|
447
|
-
test("escalateRetirement is inert for a process that drained normally", async () => {
|
|
448
|
-
/** @type {string[]} */
|
|
449
|
-
const signals = []
|
|
450
|
-
const managed = new ManagedProcess({
|
|
451
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
452
|
-
cwd: undefined,
|
|
453
|
-
env: {},
|
|
454
|
-
id: "worker",
|
|
455
|
-
lifecycle: {drainTimeoutMs: 0},
|
|
456
|
-
logger: () => {},
|
|
457
|
-
outputLines: 50,
|
|
458
|
-
restartDelayMs: 10,
|
|
459
|
-
shouldRestart: () => false,
|
|
460
|
-
stopSignal: "SIGTERM",
|
|
461
|
-
stopTimeoutMs: 2000
|
|
462
|
-
})
|
|
463
|
-
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
464
|
-
|
|
465
|
-
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
466
|
-
signals.push(signal)
|
|
467
|
-
await signalProcessGroup(signal, pgid, deadline)
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
try {
|
|
471
|
-
await managed.start()
|
|
472
|
-
await managed.stop()
|
|
473
|
-
|
|
474
|
-
expect(managed.status().state).toBe("stopped")
|
|
475
|
-
expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
|
|
476
|
-
expect(signals).toEqual(["SIGTERM"])
|
|
477
|
-
} finally {
|
|
478
|
-
const pid = managed.status().pid
|
|
479
|
-
if (pid) process.kill(-pid, "SIGKILL")
|
|
480
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
481
|
-
}
|
|
482
|
-
})
|
|
483
|
-
|
|
484
|
-
test("a retirement budget caps a finite graceful window instead of being outvoted by it", async () => {
|
|
485
|
-
/** @type {string[]} */
|
|
486
|
-
const signals = []
|
|
487
|
-
/** @type {Array<{msFromStopStart: number, phase: string}>} */
|
|
488
|
-
const deadlines = []
|
|
489
|
-
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-bound-"))
|
|
490
|
-
const readyPath = path.join(dir, "ready")
|
|
491
|
-
const managed = new ManagedProcess({
|
|
492
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
|
|
493
|
-
cwd: undefined,
|
|
494
|
-
env: {},
|
|
495
|
-
id: "worker",
|
|
496
|
-
lifecycle: {drainTimeoutMs: 0},
|
|
497
|
-
logger: () => {},
|
|
498
|
-
outputLines: 50,
|
|
499
|
-
restartDelayMs: 10,
|
|
500
|
-
shouldRestart: () => false,
|
|
501
|
-
stopSignal: "SIGTERM",
|
|
502
|
-
stopTimeoutMs: 2000
|
|
503
|
-
})
|
|
504
|
-
let stopStartedAtMs = 0
|
|
505
|
-
|
|
506
|
-
/**
|
|
507
|
-
* @param {string} phase - The step the deadline was handed to.
|
|
508
|
-
* @param {number | undefined} deadline - Absolute deadline the step received.
|
|
509
|
-
* @returns {void} Records the deadline as an offset from the start of the stop.
|
|
510
|
-
*/
|
|
511
|
-
const recordDeadline = (phase, deadline) => {
|
|
512
|
-
if (deadline !== undefined) deadlines.push({msFromStopStart: deadline - stopStartedAtMs, phase})
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
516
|
-
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
517
|
-
signals.push(signal)
|
|
518
|
-
recordDeadline("signal", deadline)
|
|
519
|
-
await signalProcessGroup(signal, pgid, deadline)
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
const waitForProcessGroupExit = managed.waitForProcessGroupExit.bind(managed)
|
|
523
|
-
managed.waitForProcessGroupExit = async (pgid, deadline) => {
|
|
524
|
-
recordDeadline("wait", deadline)
|
|
525
|
-
return await waitForProcessGroupExit(pgid, deadline)
|
|
526
|
-
}
|
|
527
|
-
|
|
528
|
-
try {
|
|
529
|
-
await managed.start()
|
|
530
|
-
// The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
|
|
531
|
-
// child that has not finished booting reaches the default disposition instead, which exits
|
|
532
|
-
// the group and leaves no retirement wait to observe.
|
|
533
|
-
await waitFor(() => fs.existsSync(readyPath))
|
|
534
|
-
stopStartedAtMs = Date.now()
|
|
535
|
-
|
|
536
|
-
await managed.stop({retirementTimeoutMs: 25})
|
|
537
|
-
|
|
538
|
-
// The 2 000 ms graceful window must not outvote the 25 ms retirement budget. Read the other
|
|
539
|
-
// way round the wait runs the whole window before it escalates to SIGKILL, which is the shape
|
|
540
|
-
// that held the TensorBuzz control plane for 4,5 h after the worker moved from "indefinite" to
|
|
541
|
-
// 4,5 h and killed the in-flight builds the consumer said not to interrupt.
|
|
542
|
-
expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
|
|
543
|
-
for (const entry of deadlines) {
|
|
544
|
-
// The deadline starts at stop entry; a millisecond clock tick between this test's marker
|
|
545
|
-
// and the async stop call may shave 1 ms from the observed offset.
|
|
546
|
-
expect(entry.msFromStopStart).toBeGreaterThanOrEqual(0)
|
|
547
|
-
expect(entry.msFromStopStart).toBeLessThan(1000)
|
|
548
|
-
}
|
|
549
|
-
expect(managed.retirementTimedOut).toBe(true)
|
|
550
|
-
expect(signals).toEqual(["SIGTERM"])
|
|
551
|
-
expect(managed.status().pid).toBeDefined()
|
|
552
|
-
} finally {
|
|
553
|
-
const pid = managed.status().pid
|
|
554
|
-
if (pid) process.kill(-pid, "SIGKILL")
|
|
555
|
-
if (managed.exitPromise) await managed.exitPromise
|
|
556
|
-
fs.rmSync(dir, {force: true, recursive: true})
|
|
557
|
-
}
|
|
558
|
-
})
|
|
559
|
-
|
|
560
311
|
test("a configured stopCommand is used instead of the stop signal", async () => {
|
|
561
312
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-hooks-"))
|
|
562
313
|
const logPath = path.join(dir, "stop.log")
|
|
@@ -1143,14 +894,4 @@ test("the unlimited constant-delay fast path still applies maxDelayMs", () => {
|
|
|
1143
894
|
|
|
1144
895
|
expect(queued).toBe(5)
|
|
1145
896
|
})
|
|
1146
|
-
|
|
1147
|
-
test("the guardian fence releases only a record that is terminal and holds no process group", () => {
|
|
1148
|
-
expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "stopped"})).toBe(true)
|
|
1149
|
-
expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "failed"})).toBe(true)
|
|
1150
|
-
expect(childRecordIsGone({heldProcessGroup: undefined, pid: 4242, state: "running"})).toBe(false)
|
|
1151
|
-
|
|
1152
|
-
// The shape a drained generation leaves behind: the tracked process record is terminal, but the
|
|
1153
|
-
// group it led still holds live members, so the key is still held by that generation.
|
|
1154
|
-
expect(childRecordIsGone({heldProcessGroup: {liveMemberCount: 1, pgid: 4242}, pid: undefined, state: "stopped"})).toBe(false)
|
|
1155
|
-
})
|
|
1156
897
|
})
|
|
@@ -11,7 +11,7 @@ import {fileURLToPath} from "node:url"
|
|
|
11
11
|
import {normalizeConfig} from "../src/config.js"
|
|
12
12
|
import {sendControlCommand} from "../src/control-client.js"
|
|
13
13
|
import RollbridgeDaemon from "../src/daemon.js"
|
|
14
|
-
import GuardianClient
|
|
14
|
+
import GuardianClient from "../src/guardian-client.js"
|
|
15
15
|
import {isProcessRunning, waitForProcessExit} from "./support/process.js"
|
|
16
16
|
|
|
17
17
|
describe("owner-recovery", () => {
|
|
@@ -26,7 +26,7 @@ const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
|
|
|
26
26
|
|
|
27
27
|
test("external owner retirement releases guardian authority without losing its generation", async () => {
|
|
28
28
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-retirement-recovery-"))
|
|
29
|
-
const socketPath =
|
|
29
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
30
30
|
const statePath = path.join(root, "rollbridge.state.json")
|
|
31
31
|
const v1Path = path.join(root, "v1")
|
|
32
32
|
const v2Path = path.join(root, "v2")
|
|
@@ -710,17 +710,13 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
|
|
|
710
710
|
|
|
711
711
|
expect(recovering.daemonPid).toBe(recoveredDaemonPid)
|
|
712
712
|
expect(recovering.ownerRecovery?.ready).toBe(true)
|
|
713
|
-
expect(recovering.generationTransition?.phase).toBe("
|
|
713
|
+
expect(recovering.generationTransition?.phase).toBe("committed")
|
|
714
714
|
expect(typeof guardianPid).toBe("number")
|
|
715
|
-
await expect(() => sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: fixture.socketPath})).toThrow(/Another owner mutation/)
|
|
716
|
-
await expect(() => sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})).toThrow(/Cannot shut down while generation transition recovery is in progress/)
|
|
717
|
-
await expect(() => sendControlCommand({command: {attestation: `sha256:${"a".repeat(64)}`, command: "retire-owner"}, path: fixture.socketPath})).toThrow(/Cannot retire owner while generation transition recovery is in progress/)
|
|
718
|
-
|
|
719
715
|
process.kill(recoveredDaemonPid, "SIGTERM")
|
|
720
716
|
await fs.writeFile(retirementGatePath, "release retirement\n")
|
|
721
717
|
await waitForProcessExit(recoveredDaemonPid, 5000)
|
|
722
718
|
await waitForProcessExit(guardianPid, 5000)
|
|
723
|
-
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:
|
|
719
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1", "retire:v2"])
|
|
724
720
|
} finally {
|
|
725
721
|
await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
|
|
726
722
|
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
|
|
@@ -862,7 +858,7 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
862
858
|
}
|
|
863
859
|
})
|
|
864
860
|
|
|
865
|
-
test("
|
|
861
|
+
test("committed-pending generation retains the incumbent until singleton replacement completes", async () => {
|
|
866
862
|
const fixture = await createFixture()
|
|
867
863
|
|
|
868
864
|
fixture.config.releaseRetention = {keep: 0, maxAgeMs: 0}
|
|
@@ -870,23 +866,16 @@ test("owner recovery reconstructs a stopped release that still owns a committed-
|
|
|
870
866
|
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
871
867
|
const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
872
868
|
const replacementPause = pauseSingletonReplacement(owner, "v2")
|
|
873
|
-
const capturedDrain = captureReleaseDrain(owner, "v1")
|
|
874
869
|
const v1Path = await prepareRelease(fixture.root, "v1")
|
|
875
870
|
const v2Path = await prepareRelease(fixture.root, "v2")
|
|
876
871
|
/** @type {Promise<Record<string, import("../src/json.js").JsonValue>> | undefined} */
|
|
877
872
|
let deployPromise
|
|
878
|
-
/** @type {RollbridgeDaemon | undefined} */
|
|
879
|
-
let recovered
|
|
880
|
-
|
|
881
873
|
try {
|
|
882
874
|
await owner.start()
|
|
883
875
|
await owner.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
|
|
884
876
|
deployPromise = owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
885
877
|
void deployPromise.catch(() => {})
|
|
886
878
|
await replacementPause.started
|
|
887
|
-
await capturedDrain.started
|
|
888
|
-
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
889
|
-
await capturedDrain.completed
|
|
890
879
|
if (owner.pendingWrite) await owner.pendingWrite
|
|
891
880
|
const pending = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
|
|
892
881
|
|
|
@@ -894,38 +883,19 @@ test("owner recovery reconstructs a stopped release that still owns a committed-
|
|
|
894
883
|
expect(pending.generationTransition?.phase).toBe("committed_pending")
|
|
895
884
|
expect(pending.serviceReleaseIds?.beacon).toBe("v2")
|
|
896
885
|
expect(pending.singletonReleaseIds?.singleton).toBe("v1")
|
|
897
|
-
const stoppedRelease = pending.releases.find((release) => release.releaseId === "v1" && release.state === "stopped")
|
|
898
|
-
|
|
899
|
-
if (!stoppedRelease) throw new Error("Missing stopped singleton owner release")
|
|
900
886
|
expect(pending.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2"])
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
908
|
-
expect(recovered.singletonReleaseIds.get("singleton")).toBe("v2")
|
|
909
|
-
// the stopped singleton owner may be pruned after replacement commits
|
|
910
|
-
expect(!recovered.releases.has("v1")).toBeTruthy()
|
|
911
|
-
expect(recovered.portReservations.has(stoppedRelease.ports.jobs)).toBe(false)
|
|
912
|
-
expect(recovered.portReservations.has(stoppedRelease.ports.web)).toBe(false)
|
|
887
|
+
replacementPause.continue()
|
|
888
|
+
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
889
|
+
await deployPromise
|
|
890
|
+
expect(owner.status().generationTransition?.phase).toBe("committed")
|
|
891
|
+
expect(owner.singletonReleaseIds.get("singleton")).toBe("v2")
|
|
913
892
|
} finally {
|
|
914
893
|
replacementPause.continue()
|
|
915
894
|
await deployPromise?.catch(() => {})
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
await shutdown.catch(() => {})
|
|
921
|
-
recovered.guardian?.disconnect()
|
|
922
|
-
} else {
|
|
923
|
-
const shutdown = owner.shutdown()
|
|
924
|
-
|
|
925
|
-
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
926
|
-
await shutdown.catch(() => {})
|
|
927
|
-
owner.guardian?.disconnect()
|
|
928
|
-
}
|
|
895
|
+
const shutdown = owner.shutdown()
|
|
896
|
+
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
897
|
+
await shutdown.catch(() => {})
|
|
898
|
+
owner.guardian?.disconnect()
|
|
929
899
|
await stopFixtureGuardian(fixture.statePath)
|
|
930
900
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
931
901
|
}
|
|
@@ -1003,7 +973,7 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
1003
973
|
}
|
|
1004
974
|
})
|
|
1005
975
|
|
|
1006
|
-
test("owner recovery retains
|
|
976
|
+
test("owner recovery retains the committed generation after singleton work completes", async () => {
|
|
1007
977
|
const fixture = await createFixture()
|
|
1008
978
|
const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
1009
979
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
@@ -1041,10 +1011,12 @@ test("owner recovery retains a stopped previous release until committed-pending
|
|
|
1041
1011
|
deployPromise = owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
1042
1012
|
void deployPromise.catch(() => {})
|
|
1043
1013
|
await replacementComplete
|
|
1044
|
-
const pending = await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed_pending"
|
|
1014
|
+
const pending = await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed_pending", AbortSignal.timeout(5000))
|
|
1045
1015
|
|
|
1046
1016
|
expect(pending.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2"])
|
|
1047
|
-
expect(pending.singletonReleaseIds?.singleton).toBe("
|
|
1017
|
+
expect(pending.singletonReleaseIds?.singleton).toBe("v1")
|
|
1018
|
+
continueReplacement()
|
|
1019
|
+
await deployPromise
|
|
1048
1020
|
await owner.retireCommittedOwner(undefined)
|
|
1049
1021
|
owner.guardian?.disconnect()
|
|
1050
1022
|
|
|
@@ -1098,11 +1070,8 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
|
|
|
1098
1070
|
await fs.writeFile(retirementGatePath, "release retirement\n")
|
|
1099
1071
|
|
|
1100
1072
|
owner = spawnDaemon(fixture.configPath)
|
|
1101
|
-
const recoverySettled = waitForLog(owner, "release generation transition recovery settled")
|
|
1102
|
-
|
|
1103
1073
|
await waitForLog(owner, "control socket listening")
|
|
1104
1074
|
await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed", AbortSignal.timeout(5000))
|
|
1105
|
-
await recoverySettled
|
|
1106
1075
|
// ambiguous retirement must replay exactly once
|
|
1107
1076
|
expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(1)
|
|
1108
1077
|
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
@@ -1683,52 +1652,13 @@ test("ensure-daemon atomically replaces an incompatible owner without losing ret
|
|
|
1683
1652
|
}
|
|
1684
1653
|
})
|
|
1685
1654
|
|
|
1686
|
-
test("startup against a version-drifted guardian without an incumbent refuses and names both versions and the repair command", async () => {
|
|
1687
|
-
const fixture = await createFixture()
|
|
1688
|
-
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
1689
|
-
const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
1690
|
-
const v1Path = await prepareRelease(fixture.root, "v1")
|
|
1691
|
-
/** @type {RollbridgeDaemon | undefined} */
|
|
1692
|
-
let drifted
|
|
1693
|
-
|
|
1694
|
-
try {
|
|
1695
|
-
await owner.start()
|
|
1696
|
-
await owner.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
|
|
1697
|
-
|
|
1698
|
-
const workerPid = releaseProcessPid(owner.status(), "v1", "worker")
|
|
1699
|
-
const guardianVersion = JSON.parse(await fs.readFile(path.join(currentDir, "..", "package.json"), "utf8")).version
|
|
1700
|
-
drifted = new RollbridgeDaemon({
|
|
1701
|
-
config,
|
|
1702
|
-
configPath: fixture.configPath,
|
|
1703
|
-
logger: () => {},
|
|
1704
|
-
runtime: {digest: "f".repeat(64), format: 1, path: path.join(fixture.root, "drift-runtime"), version: "0.0.1"}
|
|
1705
|
-
})
|
|
1706
|
-
|
|
1707
|
-
const refusal = await drifted.start().then(() => undefined, (error) => error)
|
|
1708
|
-
const message = refusal instanceof Error ? refusal.message : ""
|
|
1709
|
-
|
|
1710
|
-
expect(message).toContain(`The persistent Rollbridge guardian is version ${guardianVersion} while this daemon runs rollbridge 0.0.1`)
|
|
1711
|
-
expect(message).toContain("this startup path has no incumbent daemon to hand guardian ownership from")
|
|
1712
|
-
expect(message).toContain(`Run "rollbridge shutdown --config ${fixture.configPath}" and then "rollbridge ensure-daemon --config ${fixture.configPath}"`)
|
|
1713
|
-
expect(isProcessRunning(workerPid)).toBe(true)
|
|
1714
|
-
} finally {
|
|
1715
|
-
drifted?.guardian?.disconnect()
|
|
1716
|
-
await Promise.all([
|
|
1717
|
-
fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n").catch(() => {}),
|
|
1718
|
-
owner.shutdown().catch(() => {})
|
|
1719
|
-
])
|
|
1720
|
-
await stopFixtureGuardian(fixture.statePath)
|
|
1721
|
-
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1722
|
-
}
|
|
1723
|
-
})
|
|
1724
|
-
|
|
1725
1655
|
/**
|
|
1726
1656
|
* @param {{activationFailureRelease?: string}} [options] - Lifecycle fault injection.
|
|
1727
1657
|
* @returns {Promise<{activationGatePath: string, config: Record<string, import("../src/json.js").JsonValue>, configPath: string, lifecycleLogPath: string, root: string, socketPath: string, statePath: string}>} Fixture paths.
|
|
1728
1658
|
*/
|
|
1729
1659
|
async function createFixture(options = {}) {
|
|
1730
1660
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-recovery-"))
|
|
1731
|
-
const socketPath =
|
|
1661
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
1732
1662
|
const configPath = path.join(root, "rollbridge.cjs")
|
|
1733
1663
|
const statePath = path.join(root, "rollbridge.state.json")
|
|
1734
1664
|
const serviceLogPath = path.join(root, "service.log")
|
|
@@ -1876,40 +1806,6 @@ function pauseSingletonReplacement(daemon, releaseId) {
|
|
|
1876
1806
|
return {continue: continueReplacement, started}
|
|
1877
1807
|
}
|
|
1878
1808
|
|
|
1879
|
-
/**
|
|
1880
|
-
* Captures one background release drain through its package-owned promise.
|
|
1881
|
-
* @param {RollbridgeDaemon} daemon - Current owner.
|
|
1882
|
-
* @param {string} releaseId - Release whose drain completion is required.
|
|
1883
|
-
* @returns {{completed: Promise<void>, started: Promise<void>}} Drain boundaries.
|
|
1884
|
-
*/
|
|
1885
|
-
function captureReleaseDrain(daemon, releaseId) {
|
|
1886
|
-
const drainAndPrune = daemon.drainAndPrune.bind(daemon)
|
|
1887
|
-
/** @type {() => void} */
|
|
1888
|
-
let markStarted = () => {}
|
|
1889
|
-
/** @type {(error: Error | string) => void} */
|
|
1890
|
-
let rejectCompleted = () => {}
|
|
1891
|
-
/** @type {() => void} */
|
|
1892
|
-
let resolveCompleted = () => {}
|
|
1893
|
-
const started = new Promise((resolve) => { markStarted = () => resolve(undefined) })
|
|
1894
|
-
const completed = new Promise((resolve, reject) => {
|
|
1895
|
-
resolveCompleted = () => resolve(undefined)
|
|
1896
|
-
rejectCompleted = reject
|
|
1897
|
-
})
|
|
1898
|
-
|
|
1899
|
-
daemon.drainAndPrune = async (release, config) => {
|
|
1900
|
-
if (release.releaseId !== releaseId) return await drainAndPrune(release, config)
|
|
1901
|
-
markStarted()
|
|
1902
|
-
try {
|
|
1903
|
-
await drainAndPrune(release, config)
|
|
1904
|
-
resolveCompleted()
|
|
1905
|
-
} catch (error) {
|
|
1906
|
-
rejectCompleted(error instanceof Error ? error : String(error))
|
|
1907
|
-
throw error
|
|
1908
|
-
}
|
|
1909
|
-
}
|
|
1910
|
-
return {completed, started}
|
|
1911
|
-
}
|
|
1912
|
-
|
|
1913
1809
|
/**
|
|
1914
1810
|
* @param {string} statePath - Fixture state path.
|
|
1915
1811
|
* @returns {Promise<void>} Cleanup completion.
|
|
@@ -1922,7 +1818,7 @@ async function stopFixtureGuardian(statePath) {
|
|
|
1922
1818
|
if (!identity || typeof identity.pid !== "number" || typeof identity.socketPath !== "string" || typeof identity.token !== "string") return
|
|
1923
1819
|
const command = (await fs.readFile(`/proc/${identity.pid}/cmdline`)).toString().replaceAll("\0", " ")
|
|
1924
1820
|
|
|
1925
|
-
if (!command.includes("process-guardian.js") || !command.includes(
|
|
1821
|
+
if (!command.includes("process-guardian.js") || !command.includes(statePath)) throw new Error(`Refusing to stop unverified fixture guardian pid ${identity.pid}`)
|
|
1926
1822
|
const client = new GuardianClient(identity)
|
|
1927
1823
|
|
|
1928
1824
|
await client.connect()
|