rollbridge 0.1.72 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  24. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  25. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  26. package/docs/cli.md +128 -3
  27. package/docs/compose-services.md +222 -0
  28. package/docs/config.md +177 -6
  29. package/docs/troubleshooting.md +97 -6
  30. package/docs/velocious.md +1 -1
  31. package/docs/workers.md +11 -0
  32. package/examples/tensorbuzz.com.js +3 -2
  33. package/package.json +1 -1
  34. package/src/cli.js +68 -2
  35. package/src/compose-unit.js +725 -0
  36. package/src/config.js +346 -8
  37. package/src/daemon.js +410 -73
  38. package/src/doctor.js +74 -0
  39. package/src/drain-policy.js +252 -0
  40. package/src/guardian-client.js +67 -8
  41. package/src/health.js +99 -8
  42. package/src/managed-process.js +197 -16
  43. package/src/process-guardian.js +212 -22
  44. package/src/process-memory.js +62 -2
  45. package/src/process-order.js +26 -0
  46. package/src/release-group.js +435 -37
  47. package/tensorbuzz.yml +20 -1
  48. package/test/completion.test.js +5 -2
  49. package/test/compose-config.test.js +160 -0
  50. package/test/compose-docker.test.js +297 -0
  51. package/test/compose-unit.test.js +415 -0
  52. package/test/config-validation.test.js +135 -5
  53. package/test/drain-policy.test.js +328 -0
  54. package/test/fixtures/deferred-boot-app.js +106 -0
  55. package/test/fixtures/gated-app.js +50 -0
  56. package/test/guardian-client.test.js +119 -5
  57. package/test/health.test.js +74 -1
  58. package/test/managed-process.test.js +260 -1
  59. package/test/owner-recovery.test.js +125 -21
  60. package/test/owner-replacement.test.js +175 -14
  61. package/test/process-memory.test.js +47 -1
  62. package/test/release-group.test.js +408 -2
  63. package/test/rollbridge.test.js +774 -72
  64. package/test/shutdown-completion.test.js +119 -0
  65. package/test/shutdown-retirement-bound.test.js +162 -0
  66. package/test/support/process.js +3 -1
  67. package/test/tensorbuzz-config.test.js +228 -0
@@ -1,8 +1,9 @@
1
1
  // @ts-check
2
2
 
3
3
  import http from "node:http"
4
+ import net from "node:net"
4
5
  import {describe, expect, test} from "@velocious/testing"
5
- import {waitForHealth} from "../src/health.js"
6
+ import {waitForHealth, waitForListeningPort} from "../src/health.js"
6
7
 
7
8
  describe("health", () => {
8
9
 
@@ -31,6 +32,18 @@ async function startHealthServer() {
31
32
  }
32
33
  }
33
34
 
35
+ /**
36
+ * @param {net.Server} server - Server to bind.
37
+ * @returns {Promise<number>} Ephemeral port it bound.
38
+ */
39
+ async function bindPort(server) {
40
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(undefined)))
41
+
42
+ const address = server.address()
43
+
44
+ return address && typeof address === "object" ? address.port : 0
45
+ }
46
+
34
47
  test("waitForHealth delays the first probe by startDelayMs", async () => {
35
48
  const server = await startHealthServer()
36
49
 
@@ -62,4 +75,64 @@ test("waitForHealth probes immediately when startDelayMs is 0", async () => {
62
75
  await server.close()
63
76
  }
64
77
  })
78
+
79
+ test("waitForHealth names the deadline and the time it spent when the app never becomes healthy", async () => {
80
+ const server = http.createServer((_request, response) => {
81
+ response.writeHead(500)
82
+ response.end("bad")
83
+ })
84
+
85
+ try {
86
+ const port = await bindPort(server)
87
+
88
+ /** @type {import("../src/health.js").HealthCheckError} */
89
+ const failure = /** @type {import("../src/health.js").HealthCheckError} */ (await waitForHealth({
90
+ health: {intervalMs: 25, path: "/ping", startDelayMs: 0, timeoutMs: 300},
91
+ host: "127.0.0.1",
92
+ port,
93
+ processId: "web"
94
+ }).catch((error) => error instanceof Error ? error : new Error(String(error))))
95
+
96
+ expect(String(failure.message)).toMatch(/process web: \d+ probes, \d+ms elapsed of the 300ms health deadline/)
97
+ expect(failure.healthCheck?.deadlineMs).toBe(300)
98
+ expect({value: Boolean(failure.healthCheck && failure.healthCheck.elapsedMs >= 300), context: `expected the health gate to have spent its whole budget, was ${failure.healthCheck?.elapsedMs}ms`}).toMatchObject({value: true})
99
+ expect({value: Boolean(failure.healthCheck && failure.healthCheck.attempts >= 1), context: `expected at least one probe, was ${failure.healthCheck?.attempts}`}).toMatchObject({value: true})
100
+ expect(failure.healthCheck?.processId).toBe("web")
101
+ } finally {
102
+ await server.close()
103
+ }
104
+ })
105
+
106
+ test("waitForListeningPort waits for a dependency port on a budget of its own", async () => {
107
+ const server = net.createServer()
108
+ const port = await bindPort(server)
109
+
110
+ await new Promise((resolve) => server.close(() => resolve(undefined)))
111
+
112
+ const waiting = waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 2000})
113
+
114
+ await new Promise((resolve) => setTimeout(resolve, 300))
115
+ await new Promise((resolve) => server.listen(port, "127.0.0.1", () => resolve(undefined)))
116
+
117
+ try {
118
+ const result = await waiting
119
+
120
+ expect({value: Boolean(result.attempts >= 2), context: `expected more than one connection attempt, was ${result.attempts}`}).toMatchObject({value: true})
121
+ expect({value: Boolean(result.elapsedMs >= 250), context: `expected the dependency wait to take ~300ms, was ${result.elapsedMs}ms`}).toMatchObject({value: true})
122
+ } finally {
123
+ await new Promise((resolve) => server.close(() => resolve(undefined)))
124
+ }
125
+ })
126
+
127
+ test("waitForListeningPort names the budget it ran out on when the dependency never binds", async () => {
128
+ const server = net.createServer()
129
+ const port = await bindPort(server)
130
+
131
+ await new Promise((resolve) => server.close(() => resolve(undefined)))
132
+
133
+ /** @type {Error} */
134
+ const failure = /** @type {Error} */ (await waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 200}).catch((error) => error instanceof Error ? error : new Error(String(error))))
135
+
136
+ expect(String(failure.message)).toMatch(/did not accept a connection within 200ms/)
137
+ })
65
138
  })
@@ -5,7 +5,7 @@ import os from "node:os"
5
5
  import path from "node:path"
6
6
  import {describe, expect, test} from "@velocious/testing"
7
7
  import {fileURLToPath} from "node:url"
8
- import ManagedProcess from "../src/managed-process.js"
8
+ import ManagedProcess, {childRecordIsGone} from "../src/managed-process.js"
9
9
 
10
10
  describe("managed-process", () => {
11
11
 
@@ -308,6 +308,255 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
308
308
  }
309
309
  })
310
310
 
311
+ test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
312
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-wedge-"))
313
+ const readyPath = path.join(dir, "ready")
314
+ /** @type {string[]} */
315
+ const signals = []
316
+ const managed = new ManagedProcess({
317
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
318
+ cwd: undefined,
319
+ env: {},
320
+ id: "worker",
321
+ lifecycle: {drainTimeoutMs: 0},
322
+ logger: () => {},
323
+ outputLines: 50,
324
+ restartDelayMs: 10,
325
+ shouldRestart: () => false,
326
+ stopSignal: "SIGTERM",
327
+ stopTimeoutMs: "indefinite"
328
+ })
329
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
330
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
331
+ signals.push(signal)
332
+ await signalProcessGroup(signal, pgid, deadline)
333
+ }
334
+
335
+ try {
336
+ await managed.start()
337
+ // The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
338
+ // child that has not finished booting reaches the default disposition instead, which exits
339
+ // the group and leaves no wedge to observe.
340
+ await waitFor(() => fs.existsSync(readyPath))
341
+ await managed.stop({retirementTimeoutMs: 25})
342
+
343
+ expect(managed.retirementTimedOut).toBe(true)
344
+ expect(signals).toEqual(["SIGTERM"])
345
+ expect(managed.status().pid).toBeDefined()
346
+ } finally {
347
+ const pid = managed.status().pid
348
+ if (pid) process.kill(-pid, "SIGKILL")
349
+ if (managed.exitPromise) await managed.exitPromise
350
+ fs.rmSync(dir, {force: true, recursive: true})
351
+ }
352
+ })
353
+
354
+ test("escalateRetirement terminates the wedged worker the bounded retirement left alive", async () => {
355
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-escalation-wedge-"))
356
+ const readyPath = path.join(dir, "ready")
357
+ /** @type {string[]} */
358
+ const signals = []
359
+ /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
360
+ const logs = []
361
+ const managed = new ManagedProcess({
362
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
363
+ cwd: undefined,
364
+ env: {},
365
+ id: "worker",
366
+ lifecycle: {drainTimeoutMs: 0},
367
+ logger: (message, data) => logs.push({data, message}),
368
+ outputLines: 50,
369
+ restartDelayMs: 10,
370
+ shouldRestart: () => false,
371
+ stopSignal: "SIGTERM",
372
+ stopTimeoutMs: "indefinite"
373
+ })
374
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
375
+
376
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
377
+ signals.push(signal)
378
+ await signalProcessGroup(signal, pgid, deadline)
379
+ }
380
+
381
+ try {
382
+ await managed.start()
383
+ await waitFor(() => fs.existsSync(readyPath))
384
+ await managed.stop({retirementTimeoutMs: 25})
385
+ const wedgedPid = managed.status().pid
386
+
387
+ expect(managed.retirementTimedOut).toBe(true)
388
+ expect(signals).toEqual(["SIGTERM"])
389
+ expect(wedgedPid).toBeDefined()
390
+
391
+ expect(await managed.escalateRetirement({reason: "spec: proven wedge"})).toBe(true)
392
+ expect(signals).toEqual(["SIGTERM", "SIGKILL"])
393
+ expect(logs.some((entry) => entry.message === "process retirement escalated to SIGKILL" && entry.data?.reason === "spec: proven wedge")).toBe(true)
394
+ expect(await managed.waitForProcessGroupExit(/** @type {number} */ (wedgedPid), Date.now() + 5000)).toBe(true)
395
+ if (managed.exitPromise) await managed.exitPromise
396
+ expect(managed.status().pid).toBeUndefined()
397
+ expect(managed.status().state).toBe("stopped")
398
+ } finally {
399
+ const pid = managed.status().pid
400
+ if (pid) process.kill(-pid, "SIGKILL")
401
+ if (managed.exitPromise) await managed.exitPromise
402
+ fs.rmSync(dir, {force: true, recursive: true})
403
+ }
404
+ })
405
+
406
+ test("escalateRetirement refuses a process that has not expired its retirement budget", async () => {
407
+ /** @type {string[]} */
408
+ const signals = []
409
+ /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
410
+ const logs = []
411
+ const managed = new ManagedProcess({
412
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
413
+ cwd: undefined,
414
+ env: {},
415
+ id: "worker",
416
+ lifecycle: {drainTimeoutMs: 0},
417
+ logger: (message, data) => logs.push({data, message}),
418
+ outputLines: 50,
419
+ restartDelayMs: 10,
420
+ shouldRestart: () => false,
421
+ stopSignal: "SIGTERM",
422
+ stopTimeoutMs: "indefinite"
423
+ })
424
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
425
+
426
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
427
+ signals.push(signal)
428
+ await signalProcessGroup(signal, pgid, deadline)
429
+ }
430
+
431
+ try {
432
+ await managed.start()
433
+ await managed.quiesce()
434
+
435
+ expect(managed.retirementTimedOut).toBe(false)
436
+ expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
437
+ expect(signals).toEqual([])
438
+ expect(logs.some((entry) => entry.message === "retirement escalation refused; process is not past its retirement budget")).toBe(true)
439
+ expect(managed.status().pid).toBeDefined()
440
+ } finally {
441
+ const pid = managed.status().pid
442
+ if (pid) process.kill(-pid, "SIGKILL")
443
+ if (managed.exitPromise) await managed.exitPromise
444
+ }
445
+ })
446
+
447
+ test("escalateRetirement is inert for a process that drained normally", async () => {
448
+ /** @type {string[]} */
449
+ const signals = []
450
+ const managed = new ManagedProcess({
451
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
452
+ cwd: undefined,
453
+ env: {},
454
+ id: "worker",
455
+ lifecycle: {drainTimeoutMs: 0},
456
+ logger: () => {},
457
+ outputLines: 50,
458
+ restartDelayMs: 10,
459
+ shouldRestart: () => false,
460
+ stopSignal: "SIGTERM",
461
+ stopTimeoutMs: 2000
462
+ })
463
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
464
+
465
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
466
+ signals.push(signal)
467
+ await signalProcessGroup(signal, pgid, deadline)
468
+ }
469
+
470
+ try {
471
+ await managed.start()
472
+ await managed.stop()
473
+
474
+ expect(managed.status().state).toBe("stopped")
475
+ expect(await managed.escalateRetirement({reason: "spec"})).toBe(false)
476
+ expect(signals).toEqual(["SIGTERM"])
477
+ } finally {
478
+ const pid = managed.status().pid
479
+ if (pid) process.kill(-pid, "SIGKILL")
480
+ if (managed.exitPromise) await managed.exitPromise
481
+ }
482
+ })
483
+
484
+ test("a retirement budget caps a finite graceful window instead of being outvoted by it", async () => {
485
+ /** @type {string[]} */
486
+ const signals = []
487
+ /** @type {Array<{msFromStopStart: number, phase: string}>} */
488
+ const deadlines = []
489
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-retirement-bound-"))
490
+ const readyPath = path.join(dir, "ready")
491
+ const managed = new ManagedProcess({
492
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => {}); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
493
+ cwd: undefined,
494
+ env: {},
495
+ id: "worker",
496
+ lifecycle: {drainTimeoutMs: 0},
497
+ logger: () => {},
498
+ outputLines: 50,
499
+ restartDelayMs: 10,
500
+ shouldRestart: () => false,
501
+ stopSignal: "SIGTERM",
502
+ stopTimeoutMs: 2000
503
+ })
504
+ let stopStartedAtMs = 0
505
+
506
+ /**
507
+ * @param {string} phase - The step the deadline was handed to.
508
+ * @param {number | undefined} deadline - Absolute deadline the step received.
509
+ * @returns {void} Records the deadline as an offset from the start of the stop.
510
+ */
511
+ const recordDeadline = (phase, deadline) => {
512
+ if (deadline !== undefined) deadlines.push({msFromStopStart: deadline - stopStartedAtMs, phase})
513
+ }
514
+
515
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
516
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
517
+ signals.push(signal)
518
+ recordDeadline("signal", deadline)
519
+ await signalProcessGroup(signal, pgid, deadline)
520
+ }
521
+
522
+ const waitForProcessGroupExit = managed.waitForProcessGroupExit.bind(managed)
523
+ managed.waitForProcessGroupExit = async (pgid, deadline) => {
524
+ recordDeadline("wait", deadline)
525
+ return await waitForProcessGroupExit(pgid, deadline)
526
+ }
527
+
528
+ try {
529
+ await managed.start()
530
+ // The fixture installs its SIGTERM disposition before it announces readiness. Signalling a
531
+ // child that has not finished booting reaches the default disposition instead, which exits
532
+ // the group and leaves no retirement wait to observe.
533
+ await waitFor(() => fs.existsSync(readyPath))
534
+ stopStartedAtMs = Date.now()
535
+
536
+ await managed.stop({retirementTimeoutMs: 25})
537
+
538
+ // The 2 000 ms graceful window must not outvote the 25 ms retirement budget. Read the other
539
+ // way round the wait runs the whole window before it escalates to SIGKILL, which is the shape
540
+ // that held the TensorBuzz control plane for 4,5 h after the worker moved from "indefinite" to
541
+ // 4,5 h and killed the in-flight builds the consumer said not to interrupt.
542
+ expect(deadlines.map((entry) => entry.phase)).toEqual(["signal", "wait"])
543
+ for (const entry of deadlines) {
544
+ // The deadline starts at stop entry; a millisecond clock tick between this test's marker
545
+ // and the async stop call may shave 1 ms from the observed offset.
546
+ expect(entry.msFromStopStart).toBeGreaterThanOrEqual(0)
547
+ expect(entry.msFromStopStart).toBeLessThan(1000)
548
+ }
549
+ expect(managed.retirementTimedOut).toBe(true)
550
+ expect(signals).toEqual(["SIGTERM"])
551
+ expect(managed.status().pid).toBeDefined()
552
+ } finally {
553
+ const pid = managed.status().pid
554
+ if (pid) process.kill(-pid, "SIGKILL")
555
+ if (managed.exitPromise) await managed.exitPromise
556
+ fs.rmSync(dir, {force: true, recursive: true})
557
+ }
558
+ })
559
+
311
560
  test("a configured stopCommand is used instead of the stop signal", async () => {
312
561
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-hooks-"))
313
562
  const logPath = path.join(dir, "stop.log")
@@ -894,4 +1143,14 @@ test("the unlimited constant-delay fast path still applies maxDelayMs", () => {
894
1143
 
895
1144
  expect(queued).toBe(5)
896
1145
  })
1146
+
1147
+ test("the guardian fence releases only a record that is terminal and holds no process group", () => {
1148
+ expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "stopped"})).toBe(true)
1149
+ expect(childRecordIsGone({heldProcessGroup: undefined, pid: undefined, state: "failed"})).toBe(true)
1150
+ expect(childRecordIsGone({heldProcessGroup: undefined, pid: 4242, state: "running"})).toBe(false)
1151
+
1152
+ // The shape a drained generation leaves behind: the tracked process record is terminal, but the
1153
+ // group it led still holds live members, so the key is still held by that generation.
1154
+ expect(childRecordIsGone({heldProcessGroup: {liveMemberCount: 1, pgid: 4242}, pid: undefined, state: "stopped"})).toBe(false)
1155
+ })
897
1156
  })
@@ -11,7 +11,7 @@ import {fileURLToPath} from "node:url"
11
11
  import {normalizeConfig} from "../src/config.js"
12
12
  import {sendControlCommand} from "../src/control-client.js"
13
13
  import RollbridgeDaemon from "../src/daemon.js"
14
- import GuardianClient from "../src/guardian-client.js"
14
+ import GuardianClient, {usableSocketPath} from "../src/guardian-client.js"
15
15
  import {isProcessRunning, waitForProcessExit} from "./support/process.js"
16
16
 
17
17
  describe("owner-recovery", () => {
@@ -26,7 +26,7 @@ const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
26
26
 
27
27
  test("external owner retirement releases guardian authority without losing its generation", async () => {
28
28
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-retirement-recovery-"))
29
- const socketPath = path.join(root, "rollbridge.sock")
29
+ const socketPath = usableSocketPath(path.join(root, "rollbridge.sock"))
30
30
  const statePath = path.join(root, "rollbridge.state.json")
31
31
  const v1Path = path.join(root, "v1")
32
32
  const v2Path = path.join(root, "v2")
@@ -710,13 +710,17 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
710
710
 
711
711
  expect(recovering.daemonPid).toBe(recoveredDaemonPid)
712
712
  expect(recovering.ownerRecovery?.ready).toBe(true)
713
- expect(recovering.generationTransition?.phase).toBe("committed")
713
+ expect(recovering.generationTransition?.phase).toBe("committed_pending")
714
714
  expect(typeof guardianPid).toBe("number")
715
+ await expect(() => sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: fixture.socketPath})).toThrow(/Another owner mutation/)
716
+ await expect(() => sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})).toThrow(/Cannot shut down while generation transition recovery is in progress/)
717
+ await expect(() => sendControlCommand({command: {attestation: `sha256:${"a".repeat(64)}`, command: "retire-owner"}, path: fixture.socketPath})).toThrow(/Cannot retire owner while generation transition recovery is in progress/)
718
+
715
719
  process.kill(recoveredDaemonPid, "SIGTERM")
716
720
  await fs.writeFile(retirementGatePath, "release retirement\n")
717
721
  await waitForProcessExit(recoveredDaemonPid, 5000)
718
722
  await waitForProcessExit(guardianPid, 5000)
719
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1", "retire:v2"])
723
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v2", "retire:v1"])
720
724
  } finally {
721
725
  await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
722
726
  await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
@@ -858,7 +862,7 @@ test("owner recovery preserves complete private transition authority across a ca
858
862
  }
859
863
  })
860
864
 
861
- test("committed-pending generation retains the incumbent until singleton replacement completes", async () => {
865
+ test("owner recovery reconstructs a stopped release that still owns a committed-pending singleton", async () => {
862
866
  const fixture = await createFixture()
863
867
 
864
868
  fixture.config.releaseRetention = {keep: 0, maxAgeMs: 0}
@@ -866,16 +870,23 @@ test("committed-pending generation retains the incumbent until singleton replace
866
870
  const config = normalizeConfig(fixture.config, fixture.configPath)
867
871
  const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
868
872
  const replacementPause = pauseSingletonReplacement(owner, "v2")
873
+ const capturedDrain = captureReleaseDrain(owner, "v1")
869
874
  const v1Path = await prepareRelease(fixture.root, "v1")
870
875
  const v2Path = await prepareRelease(fixture.root, "v2")
871
876
  /** @type {Promise<Record<string, import("../src/json.js").JsonValue>> | undefined} */
872
877
  let deployPromise
878
+ /** @type {RollbridgeDaemon | undefined} */
879
+ let recovered
880
+
873
881
  try {
874
882
  await owner.start()
875
883
  await owner.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
876
884
  deployPromise = owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
877
885
  void deployPromise.catch(() => {})
878
886
  await replacementPause.started
887
+ await capturedDrain.started
888
+ await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
889
+ await capturedDrain.completed
879
890
  if (owner.pendingWrite) await owner.pendingWrite
880
891
  const pending = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
881
892
 
@@ -883,19 +894,38 @@ test("committed-pending generation retains the incumbent until singleton replace
883
894
  expect(pending.generationTransition?.phase).toBe("committed_pending")
884
895
  expect(pending.serviceReleaseIds?.beacon).toBe("v2")
885
896
  expect(pending.singletonReleaseIds?.singleton).toBe("v1")
897
+ const stoppedRelease = pending.releases.find((release) => release.releaseId === "v1" && release.state === "stopped")
898
+
899
+ if (!stoppedRelease) throw new Error("Missing stopped singleton owner release")
886
900
  expect(pending.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2"])
887
- replacementPause.continue()
888
- await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
889
- await deployPromise
890
- expect(owner.status().generationTransition?.phase).toBe("committed")
891
- expect(owner.singletonReleaseIds.get("singleton")).toBe("v2")
901
+ await owner.retireCommittedOwner(undefined)
902
+ owner.guardian?.disconnect()
903
+
904
+ recovered = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
905
+ await recovered.start()
906
+
907
+ expect(recovered.status().generationTransition?.phase).toBe("committed")
908
+ expect(recovered.singletonReleaseIds.get("singleton")).toBe("v2")
909
+ // the stopped singleton owner may be pruned after replacement commits
910
+ expect(!recovered.releases.has("v1")).toBeTruthy()
911
+ expect(recovered.portReservations.has(stoppedRelease.ports.jobs)).toBe(false)
912
+ expect(recovered.portReservations.has(stoppedRelease.ports.web)).toBe(false)
892
913
  } finally {
893
914
  replacementPause.continue()
894
915
  await deployPromise?.catch(() => {})
895
- const shutdown = owner.shutdown()
896
- await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
897
- await shutdown.catch(() => {})
898
- owner.guardian?.disconnect()
916
+ if (recovered) {
917
+ const shutdown = recovered.shutdown()
918
+
919
+ await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
920
+ await shutdown.catch(() => {})
921
+ recovered.guardian?.disconnect()
922
+ } else {
923
+ const shutdown = owner.shutdown()
924
+
925
+ await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
926
+ await shutdown.catch(() => {})
927
+ owner.guardian?.disconnect()
928
+ }
899
929
  await stopFixtureGuardian(fixture.statePath)
900
930
  await fs.rm(fixture.root, {force: true, recursive: true})
901
931
  }
@@ -973,7 +1003,7 @@ test("owner recovery uses the owning release singleton definition during a commi
973
1003
  }
974
1004
  })
975
1005
 
976
- test("owner recovery retains the committed generation after singleton work completes", async () => {
1006
+ test("owner recovery retains a stopped previous release until committed-pending singleton work completes", async () => {
977
1007
  const fixture = await createFixture()
978
1008
  const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
979
1009
  const worker = processes.find((processConfig) => processConfig.id === "worker")
@@ -1011,12 +1041,10 @@ test("owner recovery retains the committed generation after singleton work compl
1011
1041
  deployPromise = owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
1012
1042
  void deployPromise.catch(() => {})
1013
1043
  await replacementComplete
1014
- const pending = await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed_pending", AbortSignal.timeout(5000))
1044
+ const pending = await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed_pending" && state.releases?.some((release) => release.releaseId === "v1" && release.state === "stopped"), AbortSignal.timeout(5000))
1015
1045
 
1016
1046
  expect(pending.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2"])
1017
- expect(pending.singletonReleaseIds?.singleton).toBe("v1")
1018
- continueReplacement()
1019
- await deployPromise
1047
+ expect(pending.singletonReleaseIds?.singleton).toBe("v2")
1020
1048
  await owner.retireCommittedOwner(undefined)
1021
1049
  owner.guardian?.disconnect()
1022
1050
 
@@ -1070,8 +1098,11 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
1070
1098
  await fs.writeFile(retirementGatePath, "release retirement\n")
1071
1099
 
1072
1100
  owner = spawnDaemon(fixture.configPath)
1101
+ const recoverySettled = waitForLog(owner, "release generation transition recovery settled")
1102
+
1073
1103
  await waitForLog(owner, "control socket listening")
1074
1104
  await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed", AbortSignal.timeout(5000))
1105
+ await recoverySettled
1075
1106
  // ambiguous retirement must replay exactly once
1076
1107
  expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(1)
1077
1108
  expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
@@ -1652,13 +1683,52 @@ test("ensure-daemon atomically replaces an incompatible owner without losing ret
1652
1683
  }
1653
1684
  })
1654
1685
 
1686
+ test("startup against a version-drifted guardian without an incumbent refuses and names both versions and the repair command", async () => {
1687
+ const fixture = await createFixture()
1688
+ const config = normalizeConfig(fixture.config, fixture.configPath)
1689
+ const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
1690
+ const v1Path = await prepareRelease(fixture.root, "v1")
1691
+ /** @type {RollbridgeDaemon | undefined} */
1692
+ let drifted
1693
+
1694
+ try {
1695
+ await owner.start()
1696
+ await owner.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
1697
+
1698
+ const workerPid = releaseProcessPid(owner.status(), "v1", "worker")
1699
+ const guardianVersion = JSON.parse(await fs.readFile(path.join(currentDir, "..", "package.json"), "utf8")).version
1700
+ drifted = new RollbridgeDaemon({
1701
+ config,
1702
+ configPath: fixture.configPath,
1703
+ logger: () => {},
1704
+ runtime: {digest: "f".repeat(64), format: 1, path: path.join(fixture.root, "drift-runtime"), version: "0.0.1"}
1705
+ })
1706
+
1707
+ const refusal = await drifted.start().then(() => undefined, (error) => error)
1708
+ const message = refusal instanceof Error ? refusal.message : ""
1709
+
1710
+ expect(message).toContain(`The persistent Rollbridge guardian is version ${guardianVersion} while this daemon runs rollbridge 0.0.1`)
1711
+ expect(message).toContain("this startup path has no incumbent daemon to hand guardian ownership from")
1712
+ expect(message).toContain(`Run "rollbridge shutdown --config ${fixture.configPath}" and then "rollbridge ensure-daemon --config ${fixture.configPath}"`)
1713
+ expect(isProcessRunning(workerPid)).toBe(true)
1714
+ } finally {
1715
+ drifted?.guardian?.disconnect()
1716
+ await Promise.all([
1717
+ fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n").catch(() => {}),
1718
+ owner.shutdown().catch(() => {})
1719
+ ])
1720
+ await stopFixtureGuardian(fixture.statePath)
1721
+ await fs.rm(fixture.root, {force: true, recursive: true})
1722
+ }
1723
+ })
1724
+
1655
1725
  /**
1656
1726
  * @param {{activationFailureRelease?: string}} [options] - Lifecycle fault injection.
1657
1727
  * @returns {Promise<{activationGatePath: string, config: Record<string, import("../src/json.js").JsonValue>, configPath: string, lifecycleLogPath: string, root: string, socketPath: string, statePath: string}>} Fixture paths.
1658
1728
  */
1659
1729
  async function createFixture(options = {}) {
1660
1730
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-recovery-"))
1661
- const socketPath = path.join(root, "rollbridge.sock")
1731
+ const socketPath = usableSocketPath(path.join(root, "rollbridge.sock"))
1662
1732
  const configPath = path.join(root, "rollbridge.cjs")
1663
1733
  const statePath = path.join(root, "rollbridge.state.json")
1664
1734
  const serviceLogPath = path.join(root, "service.log")
@@ -1806,6 +1876,40 @@ function pauseSingletonReplacement(daemon, releaseId) {
1806
1876
  return {continue: continueReplacement, started}
1807
1877
  }
1808
1878
 
1879
+ /**
1880
+ * Captures one background release drain through its package-owned promise.
1881
+ * @param {RollbridgeDaemon} daemon - Current owner.
1882
+ * @param {string} releaseId - Release whose drain completion is required.
1883
+ * @returns {{completed: Promise<void>, started: Promise<void>}} Drain boundaries.
1884
+ */
1885
+ function captureReleaseDrain(daemon, releaseId) {
1886
+ const drainAndPrune = daemon.drainAndPrune.bind(daemon)
1887
+ /** @type {() => void} */
1888
+ let markStarted = () => {}
1889
+ /** @type {(error: Error | string) => void} */
1890
+ let rejectCompleted = () => {}
1891
+ /** @type {() => void} */
1892
+ let resolveCompleted = () => {}
1893
+ const started = new Promise((resolve) => { markStarted = () => resolve(undefined) })
1894
+ const completed = new Promise((resolve, reject) => {
1895
+ resolveCompleted = () => resolve(undefined)
1896
+ rejectCompleted = reject
1897
+ })
1898
+
1899
+ daemon.drainAndPrune = async (release, config) => {
1900
+ if (release.releaseId !== releaseId) return await drainAndPrune(release, config)
1901
+ markStarted()
1902
+ try {
1903
+ await drainAndPrune(release, config)
1904
+ resolveCompleted()
1905
+ } catch (error) {
1906
+ rejectCompleted(error instanceof Error ? error : String(error))
1907
+ throw error
1908
+ }
1909
+ }
1910
+ return {completed, started}
1911
+ }
1912
+
1809
1913
  /**
1810
1914
  * @param {string} statePath - Fixture state path.
1811
1915
  * @returns {Promise<void>} Cleanup completion.
@@ -1818,7 +1922,7 @@ async function stopFixtureGuardian(statePath) {
1818
1922
  if (!identity || typeof identity.pid !== "number" || typeof identity.socketPath !== "string" || typeof identity.token !== "string") return
1819
1923
  const command = (await fs.readFile(`/proc/${identity.pid}/cmdline`)).toString().replaceAll("\0", " ")
1820
1924
 
1821
- if (!command.includes("process-guardian.js") || !command.includes(statePath)) throw new Error(`Refusing to stop unverified fixture guardian pid ${identity.pid}`)
1925
+ if (!command.includes("process-guardian.js") || !command.includes(identity.socketPath)) throw new Error(`Refusing to stop unverified fixture guardian pid ${identity.pid}`)
1822
1926
  const client = new GuardianClient(identity)
1823
1927
 
1824
1928
  await client.connect()