rollbridge 0.1.69 → 0.1.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +1 -53
  2. package/TODO.md +0 -12
  3. package/docs/cli.md +3 -128
  4. package/docs/config.md +6 -177
  5. package/docs/troubleshooting.md +6 -97
  6. package/docs/velocious.md +1 -1
  7. package/docs/workers.md +0 -11
  8. package/examples/tensorbuzz.com.js +2 -3
  9. package/package.json +1 -1
  10. package/src/cli.js +2 -68
  11. package/src/config.js +8 -347
  12. package/src/daemon.js +69 -370
  13. package/src/doctor.js +0 -74
  14. package/src/guardian-client.js +8 -67
  15. package/src/health.js +8 -99
  16. package/src/managed-process.js +16 -197
  17. package/src/process-guardian.js +21 -211
  18. package/src/process-memory.js +2 -62
  19. package/src/release-group.js +37 -427
  20. package/tensorbuzz.yml +1 -20
  21. package/test/completion.test.js +2 -5
  22. package/test/config-validation.test.js +5 -135
  23. package/test/guardian-client.test.js +4 -118
  24. package/test/health.test.js +1 -74
  25. package/test/managed-process.test.js +1 -260
  26. package/test/owner-recovery.test.js +28 -132
  27. package/test/owner-replacement.test.js +11 -172
  28. package/test/process-memory.test.js +1 -47
  29. package/test/release-group.test.js +2 -408
  30. package/test/rollbridge.test.js +82 -766
  31. package/test/shutdown-completion.test.js +0 -52
  32. package/test/support/process.js +1 -3
  33. package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
  34. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
  35. package/changelog.d/20261104-compose-service-units.md +0 -21
  36. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
  37. package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
  38. package/changelog.d/20261109-path-filtered-builds.md +0 -11
  39. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
  40. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
  41. package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
  42. package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
  43. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
  44. package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
  45. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
  46. package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
  47. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
  48. package/changelog.d/20261116-guardian-version-handoff.md +0 -37
  49. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
  50. package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
  51. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
  52. package/docs/compose-services.md +0 -222
  53. package/src/compose-unit.js +0 -725
  54. package/src/drain-policy.js +0 -252
  55. package/src/process-order.js +0 -26
  56. package/test/compose-config.test.js +0 -160
  57. package/test/compose-docker.test.js +0 -297
  58. package/test/compose-unit.test.js +0 -415
  59. package/test/drain-policy.test.js +0 -328
  60. package/test/fixtures/deferred-boot-app.js +0 -106
  61. package/test/fixtures/gated-app.js +0 -50
  62. package/test/tensorbuzz-config.test.js +0 -228
@@ -19,12 +19,10 @@ describe("rollbridge", () => {
19
19
  const currentDir = path.dirname(fileURLToPath(import.meta.url))
20
20
  const binPath = path.join(currentDir, "..", "bin", "rollbridge")
21
21
  const dependentAppPath = path.join(currentDir, "fixtures", "dependent-app.js")
22
- const deferredBootAppPath = path.join(currentDir, "fixtures", "deferred-boot-app.js")
23
22
  const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
24
23
  const memoryHogPath = path.join(currentDir, "fixtures", "memory-hog.js")
25
24
  const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
26
25
  const singletonAppPath = path.join(currentDir, "fixtures", "singleton-app.js")
27
- const gatedAppPath = path.join(currentDir, "fixtures", "gated-app.js")
28
26
  const linuxTest = process.platform === "linux" ? test : test.skip
29
27
 
30
28
  test("a nonBlockingDrain worker stops immediately while its release is still draining", async () => {
@@ -35,7 +33,6 @@ test("a nonBlockingDrain worker stops immediately while its release is still dra
35
33
 
36
34
  try {
37
35
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
38
- await waitForFile(fixture.workerReadyPath)
39
36
 
40
37
  // An open WebSocket keeps v1's connection drain pending after v2 takes over.
41
38
  socket = await openWebSocket(daemon)
@@ -62,63 +59,6 @@ test("a nonBlockingDrain worker stops immediately while its release is still dra
62
59
  }
63
60
  })
64
61
 
65
- test("an explicit stop returns within its retirement budget instead of waiting on a worker that is still draining", async () => {
66
- const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 30000})
67
- const config = normalizeConfig({
68
- ...fixture.config,
69
- processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
70
- ? {...processConfig, gracefulStopMs: "indefinite"}
71
- : processConfig),
72
- proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 100}
73
- })
74
- /** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
75
- const logs = []
76
- const daemon = new RollbridgeDaemon({config, logger: (message, data) => logs.push({message, data})})
77
- /** @type {number | undefined} */
78
- let workerPid
79
-
80
- try {
81
- await daemon.start()
82
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
83
- await waitForFile(fixture.workerReadyPath)
84
-
85
- workerPid = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.pid
86
-
87
- if (typeof workerPid !== "number") throw new Error("Expected the worker to be running before the stop")
88
-
89
- const startedAtMs = Date.now()
90
-
91
- await daemon.stopRelease("v1")
92
-
93
- // Before this the command had no deadline for a worker whose graceful window is "indefinite",
94
- // so it never returned and held the owner-mutation fence for the whole drain, refusing every
95
- // later deploy for hours (TensorBuzz production, 2026-11-16). The bound is far below the
96
- // worker's own 30 000 ms stop delay because the stop must still let the generation's other
97
- // processes exit; what it must not do is wait for the worker at all.
98
- expect(Date.now() - startedAtMs).toBeLessThan(10000)
99
-
100
- await retirementRecordSettled(daemon, "v1", "worker")
101
-
102
- const stopped = statusRelease(daemon, "v1")
103
- const stoppedWorker = stopped.processes.find((processStatus) => processStatus.id === "worker")
104
-
105
- expect(stopped.state).toBe("stopped")
106
- expect(stoppedWorker?.retirementTimedOut).toBe(true)
107
- // The bounded stop records the retirement timeout; the shutdown regression separately proves
108
- // that a non-blocking worker can remain alive after the acknowledgement.
109
- expect(logs.filter((entry) => entry.message === "release stop gave up waiting for a retiring process")).toMatchObject([
110
- {data: {processId: "worker", releaseId: "v1", timeoutMs: 100}}
111
- ])
112
- expect(logs.filter((entry) => entry.message === "release stopped")).toMatchObject([{data: {releaseId: "v1"}}])
113
- } finally {
114
- await daemon.shutdown()
115
- if (workerPid !== undefined) {
116
- try { process.kill(-workerPid, "SIGKILL") } catch (_error) { /* The group already exited. */ }
117
- }
118
- await fs.rm(fixture.root, {force: true, recursive: true})
119
- }
120
- })
121
-
122
62
  test("deploy switches new HTTP traffic while old WebSockets drain", async () => {
123
63
  const fixture = await createFixture()
124
64
  const daemon = await startDaemon(fixture.config)
@@ -144,6 +84,27 @@ test("deploy switches new HTTP traffic while old WebSockets drain", async () =>
144
84
  }
145
85
  })
146
86
 
87
+ test("deploy does not wait for retirement and force-stops an old generation by the drain deadline", async () => {
88
+ const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 10000})
89
+ const config = normalizeConfig({...fixture.config, proxy: {...fixture.config.proxy, drainTimeoutMs: 50, forceStopTimeoutMs: 50}})
90
+ const daemon = await startDaemon(config)
91
+
92
+ try {
93
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
94
+ const startedAt = Date.now()
95
+
96
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
97
+
98
+ expect(Date.now() - startedAt).toBeLessThan(1000)
99
+ expect(await fetchText(daemon, "/release")).toBe("v2")
100
+ await waitFor(() => statusRelease(daemon, "v1").state === "stopped", 8000)
101
+ expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("stopped")
102
+ } finally {
103
+ await daemon.shutdown()
104
+ await fs.rm(fixture.root, {force: true, recursive: true})
105
+ }
106
+ })
107
+
147
108
  test("failed health check leaves the previous release active", async () => {
148
109
  const fixture = await createFixture()
149
110
  const daemon = await startDaemon(fixture.config)
@@ -185,7 +146,7 @@ test("deploy reloads process config and retires the previous worker with the ref
185
146
 
186
147
  await writeConfigFile(refreshedConfig, fixture.root)
187
148
  await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
188
- await waitFor(() => statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state === "stopped", 1000)
149
+ await waitFor(() => statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state === "stopped", 5000)
189
150
 
190
151
  expect(daemon.config.processes.find((processConfig) => processConfig.id === "worker")?.gracefulStopMs).toBe(50)
191
152
  expect(await fetchText(daemon, "/release")).toBe("v2")
@@ -302,139 +263,6 @@ test("failed release startup logs process output and cleanup status", async () =
302
263
  }
303
264
  })
304
265
 
305
- test("the dependency wait is charged to its own budget, not to the proxied boot budget", async () => {
306
- const fixture = await createFixture()
307
- const config = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 1200, webBootMs: 600, webHealthTimeoutMs: 900})
308
- /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
309
- const logs = []
310
- const daemon = new RollbridgeDaemon({config, logger: (message, data = {}) => logs.push({data, message})})
311
-
312
- await daemon.start()
313
-
314
- try {
315
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
316
-
317
- const dependencyLog = logs.find((entry) => entry.message === "release process dependency ready")
318
- const healthLog = logs.find((entry) => entry.message === "release candidate health ready")
319
-
320
- if (!dependencyLog?.data || !healthLog?.data) throw new Error("expected the dependency wait and the boot time to be logged")
321
- // The dependency took ~1200ms and the app boot ~600ms. The boot budget is 900ms, so this
322
- // deploy only succeeds because the dependency wait is charged to a budget of its own.
323
- expect(Number(dependencyLog.data.elapsedMs)).toBeGreaterThanOrEqual(1000)
324
- expect(Number(healthLog.data.bootMs)).toBeLessThan(900)
325
- expect(Number(healthLog.data.waitedForDependenciesMs)).toBeGreaterThanOrEqual(1000)
326
- expect(await fetchText(daemon, "/release")).toBe("v1")
327
- } finally {
328
- await daemon.shutdown()
329
- await fs.rm(fixture.root, {force: true, recursive: true})
330
- }
331
- })
332
-
333
- test("a release whose app boots past the old 30000ms window publishes on the default boot budget while the incumbent serves", {timeoutMs: 180000}, async () => {
334
- const fixture = await createFixture()
335
- // Measured on TensorBuzz production (server3, 2026-11-06): the candidate web process was
336
- // still initialising when the 30000ms release health deadline expired and was killed at
337
- // 30287ms without ever logging a listening line. 32000ms reproduces a boot the old default
338
- // killed; nothing here configures a health budget, so the release runs on the default.
339
- const lateBootMs = 32000
340
- const incumbentConfig = deferredBootConfig(fixture, {dependencyTimeoutMs: 30000, serviceBootMs: 0, webBootMs: 0})
341
- const configPath = await writeConfigFile(incumbentConfig, fixture.root)
342
- /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
343
- const logs = []
344
- const daemon = new RollbridgeDaemon({config: incumbentConfig, configPath, logger: (message, data = {}) => logs.push({data, message})})
345
-
346
- await daemon.start()
347
-
348
- try {
349
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
350
- expect(await fetchText(daemon, "/release")).toBe("v1")
351
-
352
- await writeConfigFile(deferredBootConfig(fixture, {dependencyTimeoutMs: 30000, serviceBootMs: 0, webBootMs: lateBootMs}), fixture.root)
353
- const deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
354
-
355
- // The candidate is still booting a second later: the incumbent keeps serving traffic.
356
- await new Promise((resolve) => setTimeout(resolve, 1000))
357
- expect(await fetchText(daemon, "/release")).toBe("v1")
358
- await deployment
359
-
360
- const ready = logs.find((entry) => entry.message === "release candidate health ready" && entry.data?.releaseId === "v2")
361
-
362
- if (!ready?.data) throw new Error("expected the v2 boot time to be recorded in the event log")
363
- expect(Number(ready.data.bootMs)).toBeGreaterThanOrEqual(lateBootMs)
364
- expect(Number(ready.data.healthGateMs)).toBeGreaterThanOrEqual(lateBootMs)
365
- expect(Number(ready.data.deadlineMs)).toBe(120000)
366
- expect(daemon.status().activeReleaseId).toBe("v2")
367
- expect(await fetchText(daemon, "/release")).toBe("v2")
368
- } finally {
369
- await daemon.shutdown()
370
- await fs.rm(fixture.root, {force: true, recursive: true})
371
- }
372
- })
373
-
374
- test("a dependency that never opens fails the deploy on its own budget and keeps the incumbent serving", async () => {
375
- const fixture = await createFixture()
376
- const initialConfig = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 0, webHealthTimeoutMs: 900})
377
- const configPath = await writeConfigFile(initialConfig, fixture.root)
378
- /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
379
- const logs = []
380
- const daemon = new RollbridgeDaemon({config: initialConfig, configPath, logger: (message, data = {}) => logs.push({data, message})})
381
-
382
- await daemon.start()
383
-
384
- try {
385
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
386
- expect(await fetchText(daemon, "/release")).toBe("v1")
387
-
388
- // v2's service never binds its port, so the dependency gate runs out before the app is started.
389
- await writeConfigFile(deferredBootConfig(fixture, {dependencyTimeoutMs: 300, serviceBootMs: 60000, webBootMs: 0, webHealthTimeoutMs: 900}), fixture.root)
390
- await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/did not accept a connection within 300ms/)
391
-
392
- expect(daemon.status().activeReleaseId).toBe("v1")
393
- expect(await fetchText(daemon, "/release")).toBe("v1")
394
-
395
- const failure = logs.find((entry) => entry.message === "release startup failed" && entry.data?.phase === "before cleanup")
396
-
397
- if (!failure?.data) throw new Error("expected the dependency failure to name its budget")
398
- expect(failure.data.stage).toBe("dependency")
399
- expect(failure.data.deadlineMs).toBe(300)
400
- expect(Number(failure.data.dependencyWaitMs)).toBeGreaterThanOrEqual(300)
401
- expect(failure.data.readyProcesses).toEqual(["beacon"])
402
- expect(failure.data.unreadyProcesses).toEqual([])
403
- } finally {
404
- await daemon.shutdown()
405
- await fs.rm(fixture.root, {force: true, recursive: true})
406
- }
407
- })
408
-
409
- test("a startup timeout names the deadline, the elapsed boot, the ready processes, and the last output", async () => {
410
- const fixture = await createFixture()
411
- const config = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 1200, webHealthTimeoutMs: 400})
412
- /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
413
- const logs = []
414
- const daemon = new RollbridgeDaemon({config, logger: (message, data = {}) => logs.push({data, message})})
415
-
416
- await daemon.start()
417
-
418
- try {
419
- await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/health deadline/)
420
-
421
- const failure = logs.find((entry) => entry.message === "release startup failed" && entry.data?.phase === "before cleanup")
422
-
423
- if (!failure?.data) throw new Error("expected the startup failure to name its budget")
424
- expect(failure.data.stage).toBe("health")
425
- expect(failure.data.deadlineMs).toBe(400)
426
- expect(Number(failure.data.elapsedMs)).toBeGreaterThanOrEqual(400)
427
- expect(Number(failure.data.probes)).toBeGreaterThanOrEqual(1)
428
- expect(failure.data.readyProcesses).toEqual(["beacon"])
429
- expect(failure.data.unreadyProcesses).toEqual(["web"])
430
- expect(String(failure.data.lastOutput)).toMatch(/accepted a connection/)
431
- expect(String(failure.data.lastOutputAt)).toMatch(/^\d{4}-\d{2}-\d{2}T/)
432
- } finally {
433
- await daemon.shutdown()
434
- await fs.rm(fixture.root, {force: true, recursive: true})
435
- }
436
- })
437
-
438
266
  test("singleton processes restart without overlap during deploy", async () => {
439
267
  const fixture = await createFixture({includeSingleton: true})
440
268
  const daemon = await startDaemon(fixture.config)
@@ -461,70 +289,33 @@ test("singleton processes restart without overlap during deploy", async () => {
461
289
  }
462
290
  })
463
291
 
464
- // Budget for waiting on the old jobs generation's quiet marker in the spec below. Measured with the
465
- // marker read directly: the spec completes in 1.740s on this host, so this is not a boot budget but a
466
- // generous ceiling on a handoff that already happened; the number only decides how long a genuinely
467
- // stuck quiescence takes to report itself, and it now reports the marker path instead of the runner's
468
- // ambient 60s default.
469
- const SINGLETON_QUIET_MARKER_TIMEOUT_MS = 20000
470
-
471
- test("candidate activation quiesces the old jobs generation before a blocked singleton replacement completes", async () => {
472
- const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, includeSingleton: true, webDependsOnService: true})
473
- const singletonGatePath = path.join(fixture.root, "singleton-replacement.gate")
474
- const singletonQuietPath = path.join(fixture.root, "singleton-replacement.quiet")
475
- const fifo = spawn("mkfifo", [singletonGatePath])
476
-
477
- await once(fifo, "exit")
478
-
292
+ test("candidate activation switches traffic before a blocked singleton replacement completes", async () => {
293
+ const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, webDependsOnService: true})
294
+ const retirementGatePath = path.join(fixture.root, "retirement.gate")
295
+ const retirementWaitingPath = path.join(fixture.root, "retirement.waiting")
479
296
  const config = normalizeConfig({
480
297
  ...fixture.config,
481
- processes: fixture.config.processes.map((processConfig) => processConfig.id === "jobs-main"
482
- ? {...processConfig, lifecycle: {quietCommand: `printf 'quiet\\n' > ${JSON.stringify(singletonQuietPath)}; read -r _ < ${JSON.stringify(singletonGatePath)}`}}
298
+ processes: fixture.config.processes.map((processConfig) => processConfig.id === "beacon"
299
+ ? {...processConfig, lifecycle: {activateCommand: processConfig.lifecycle.activateCommand, quietCommand: `printf 'waiting\\n' > ${JSON.stringify(retirementWaitingPath)}; while [ ! -f ${JSON.stringify(retirementGatePath)} ]; do sleep 0.02; done; printf 'retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(fixture.lifecycleLogPath)}`}}
483
300
  : processConfig)
484
301
  })
485
302
  const daemon = await startDaemon(config)
486
- /** @type {Promise<Record<string, import("../src/json.js").JsonValue>> | undefined} */
487
303
  let deployPromise
488
- let singletonGateReleased = false
489
304
 
490
305
  try {
491
306
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
492
-
493
307
  deployPromise = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
494
- let deploySettled = false
495
-
496
- void deployPromise.then(() => { deploySettled = true }, () => { deploySettled = true })
497
-
498
- // Wait on the durable marker the old jobs generation writes, not on an inotify event for it.
499
- // fs.watch surfaces nothing in runtimes whose filesystem does not deliver events (measured in
500
- // this container: a minimal fs.watch receives 0 events for a file written by an external
501
- // process, on both overlayfs /tmp and ext3 /data, with max_user_watches at 1048576), and an
502
- // unguarded event loop turns that into a bare runner timeout naming no stage. Reading the
503
- // marker is the same observation with a fallback the event never had, and the three assertions
504
- // below are unchanged: candidate traffic active, deploy still pending, old generation quiesced.
505
- const quietDeadline = Date.now() + SINGLETON_QUIET_MARKER_TIMEOUT_MS
506
- let sawQuietMarker = false
507
-
508
- while (Date.now() < quietDeadline && !sawQuietMarker) {
509
- sawQuietMarker = await fs.readFile(singletonQuietPath, "utf8").then((content) => content.includes("quiet"), () => false)
510
- if (!sawQuietMarker) await new Promise((resolve) => setTimeout(resolve, 10))
511
- }
512
-
513
- if (!sawQuietMarker) throw new Error(`Old jobs generation never wrote its quiet marker within ${SINGLETON_QUIET_MARKER_TIMEOUT_MS}ms: ${singletonQuietPath}`)
308
+ await waitFor(async () => (await fs.readFile(retirementWaitingPath, "utf8").catch(() => "")) === "waiting\n")
514
309
 
515
310
  // Candidate traffic must already be active.
516
311
  expect(daemon.status().activeReleaseId).toBe("v2")
517
- // Deploy must remain pending on singleton replacement.
518
- expect(deploySettled).toBe(false)
519
- // Old jobs-main must quiesce before singleton replacement completes.
520
- expect(await fs.readFile(fixture.serviceQuietPath, "utf8")).toBe("v1\n")
521
-
522
- await fs.writeFile(singletonGatePath, "continue\n")
523
- singletonGateReleased = true
312
+ await fs.writeFile(retirementGatePath, "continue\n")
524
313
  await deployPromise
314
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).join(",") === "activate:v1,activate:v2,retire:v1")
315
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
525
316
  } finally {
526
- if (deployPromise && !singletonGateReleased) {
527
- await fs.writeFile(singletonGatePath, "continue\n")
317
+ if (deployPromise) {
318
+ await fs.writeFile(retirementGatePath, "continue\n")
528
319
  await deployPromise.catch(() => {})
529
320
  }
530
321
 
@@ -673,7 +464,7 @@ test("handoff services stop after release-local dependents finish draining", asy
673
464
  })
674
465
 
675
466
  test("candidate activation retires jobs-main with its workers without waiting for the generation to drain", async () => {
676
- const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, nonBlockingDrainWorker: true, webDependsOnService: true, workerStopDelayMs: 1000})
467
+ const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, nonBlockingDrainWorker: true, webDependsOnService: true, workerStopDelayMs: 5000})
677
468
  const daemon = await startDaemon(fixture.config)
678
469
 
679
470
  try {
@@ -686,20 +477,28 @@ test("candidate activation retires jobs-main with its workers without waiting fo
686
477
  expect(oldWorker?.pid).toBeTruthy()
687
478
 
688
479
  await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
480
+ await waitFor(async () => (await fs.readFile(fixture.serviceQuietPath, "utf8").catch(() => "")) === "v1\n", 5000)
689
481
 
690
482
  // Traffic must switch only after the complete candidate is healthy.
691
483
  expect(daemon.status().activeReleaseId).toBe("v2")
692
- // Old jobs-main must quiesce immediately after candidate activation.
693
- expect(await fs.readFile(fixture.serviceQuietPath, "utf8")).toBe("v1\n")
484
+ // Old jobs-main is retired after traffic switches, without holding deployment completion.
485
+ await waitFor(async () => {
486
+ try {
487
+ return (await fs.readFile(fixture.serviceQuietPath, "utf8")) === "v1\n"
488
+ } catch (error) {
489
+ if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return false
490
+ throw error
491
+ }
492
+ })
694
493
 
695
494
  const retired = statusRelease(daemon, "v1")
696
495
 
697
496
  // Deployment completion must not wait for the old jobs generation.
698
497
  expect(retired.state).toBe("draining")
699
- // Old jobs-main must remain alive and quiesced with its draining workers.
700
- expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).toBe("quiesced")
701
- // Old worker must remain in its original generation until accepted work settles.
702
- expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state).not.toBe("stopped")
498
+ // Old jobs-main remains in retirement while its bounded drain runs.
499
+ expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state)
500
+ // The old worker may already have completed its bounded drain; the deploy did not wait for it.
501
+ expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "worker")?.state)
703
502
  // Old and new workers must retain distinct jobs-main endpoints.
704
503
  expect(statusRelease(daemon, "v2").ports.beacon).not.toBe(retired.ports.beacon)
705
504
  } finally {
@@ -708,7 +507,7 @@ test("candidate activation retires jobs-main with its workers without waiting fo
708
507
  }
709
508
  })
710
509
 
711
- test("opt-in generation lifecycle acknowledges old retirement before activating the candidate", async () => {
510
+ test("opt-in generation lifecycle activates the candidate before retiring the old generation", async () => {
712
511
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, nonBlockingDrainWorker: true, webDependsOnService: true})
713
512
  const daemon = await startDaemon(fixture.config)
714
513
 
@@ -719,7 +518,8 @@ test("opt-in generation lifecycle acknowledges old retirement before activating
719
518
 
720
519
  await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
721
520
 
722
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
521
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).join(",") === "activate:v1,activate:v2,retire:v1")
522
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
723
523
  expect(daemon.status().activeReleaseId).toBe("v2")
724
524
  expect(daemon.status().generationTransition?.phase).toBe("committed")
725
525
  } finally {
@@ -813,7 +613,8 @@ test("exact committed retry finishes pending singleton replacement before succes
813
613
 
814
614
  expect(daemon.status().generationTransition?.phase).toBe("committed")
815
615
  expect(daemon.status().singletons[0]?.process.state).toBe("running")
816
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
616
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).join(",") === "activate:v1,activate:v2,retire:v1")
617
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
817
618
  } finally {
818
619
  await daemon.shutdown()
819
620
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -837,27 +638,22 @@ test("first generation is not committed when its activation acknowledgement fail
837
638
  }
838
639
  })
839
640
 
840
- test("retirement acknowledgement failure retains the exact transition, blocks other deploys, and exact resume continues it", async () => {
641
+ test("retirement acknowledgement failure is named without blocking the committed deployment", async () => {
841
642
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceQuietFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
842
643
  const daemon = await startDaemon(fixture.config)
843
644
 
844
645
  try {
845
646
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
846
- await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/retirement quiescence failed/)
647
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
847
648
 
848
649
  const failed = daemon.status()
849
650
 
850
- expect(failed.activeReleaseId).toBe("v1")
851
- expect(failed.generationTransition?.phase).toBe("retiring_previous")
852
- expect(String(failed.generationTransition?.error)).toMatch(/quiet command exited non-zero/)
853
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1"])
854
- await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
855
-
651
+ expect(failed.activeReleaseId).toBe("v2")
652
+ expect(failed.generationTransition?.phase).toBe("committed")
653
+ await waitFor(() => /quiet command exited non-zero/u.test(String(statusRelease(daemon, "v1").retirementError)))
654
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2"])
856
655
  await fs.writeFile(fixture.retirementGatePath, "allow\n")
857
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
858
-
859
656
  expect(daemon.status().activeReleaseId).toBe("v2")
860
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
861
657
  } finally {
862
658
  await daemon.shutdown()
863
659
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -894,7 +690,7 @@ test("candidate activation failure reports restoration failure and exact recover
894
690
  expect(String(activationEvent?.data.error)).toMatch(/activate command exited non-zero/)
895
691
  expect(String(restorationEvent?.data.activationError)).toMatch(/activate command exited non-zero/)
896
692
  expect(String(restorationEvent?.data.error)).toMatch(/incumbent restoration rejected/)
897
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2"])
693
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2"])
898
694
  await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
899
695
  const failedCandidate = daemon.releases.get("v2")
900
696
 
@@ -923,7 +719,7 @@ test("candidate activation failure reports restoration failure and exact recover
923
719
  const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
924
720
 
925
721
  expect(persisted?.generationTransition).toBe(undefined)
926
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
722
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "retire:v1", "activate:v1"])
927
723
 
928
724
  const idempotent = await sendControlCommand({
929
725
  command: {
@@ -954,155 +750,7 @@ test("candidate activation failure reports restoration failure and exact recover
954
750
  }
955
751
  })
956
752
 
957
- test("a candidate retired by a failed activation is drained to stopped instead of left holding live processes", async () => {
958
- const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
959
- const daemon = await startDaemon(fixture.config)
960
-
961
- try {
962
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
963
- const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
964
-
965
- if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
966
- incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
967
- await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
968
-
969
- expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
970
-
971
- // The failed candidate is not a restoration target, so the compensation starts its bounded
972
- // drain before attempting the restoration that then throws. On the base tree the drain only
973
- // ran on the success path, so this candidate stayed `draining` with a live process and the
974
- // assertion below timed out.
975
- await waitFor(() => statusRelease(daemon, "v2").state === "stopped")
976
- const drainedEvent = daemon.eventLog.recent().find((event) => event.message === "release drained" && String(event.data?.releaseId) === "v2")
977
-
978
- expect(drainedEvent).toBeDefined()
979
- } finally {
980
- await daemon.shutdown()
981
- await fs.rm(fixture.root, {force: true, recursive: true})
982
- }
983
- })
984
-
985
- test("recovery against a committed journal naming the live incumbent reports already_recovered and admits a rerun", async () => {
986
- const fixture = await createFixture({gatedWeb: true, webHealthTimeoutMs: 500})
987
- const daemon = await startDaemon(fixture.config)
988
-
989
- try {
990
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
991
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
992
-
993
- // v3 never becomes healthy: the deploy aborts before its own transition is journaled,
994
- // so the journal still holds the committed transition that published v2, exactly like
995
- // the TensorBuzz production incident.
996
- await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/health check failed/i)
997
- expect(daemon.status().generationTransition?.phase).toBe("committed")
998
- expect(daemon.status().generationTransition?.candidateReleaseId).toBe("v2")
999
- expect(daemon.status().generationTransition?.previousReleaseId).toBe("v1")
1000
- expect(daemon.status().activeReleaseId).toBe("v2")
1001
-
1002
- const exactRecovery = (overrides = {}) => sendControlCommand({
1003
- command: {
1004
- command: "recover-generation-transition",
1005
- previousReleaseId: "v2",
1006
- releaseId: "v3",
1007
- releasePath: fixture.root,
1008
- revision: "v3",
1009
- ...overrides
1010
- },
1011
- path: fixture.config.control.path
1012
- })
1013
-
1014
- // A fence naming a release other than the live incumbent is not a recoverable
1015
- // transition and keeps failing closed.
1016
- await expect(exactRecovery({previousReleaseId: "v1"})).rejects.toThrow(/not a safe failed pre-commit transition/i)
1017
- await expect(exactRecovery({previousReleaseId: "not-a-release"})).rejects.toThrow(/refusing stale recovery/i)
1018
-
1019
- // Rampway derives the recovery fence from the release that is actually running.
1020
- const recovery = await exactRecovery()
1021
-
1022
- expect(recovery.recoveryStatus).toBe("already_recovered")
1023
- expect(recovery.activeReleaseId).toBe("v2")
1024
- expect(daemon.status().activeReleaseId).toBe("v2")
1025
- expect(daemon.status().generationTransition?.phase).toBe("committed")
1026
- expect(await fetchText(daemon, "/release")).toBe("v2")
1027
-
1028
- // The acceptance flow: the deploy is recovered by rerunning it, with no state edits.
1029
- await fs.writeFile(fixture.gatePath, "allow\n")
1030
- await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
1031
- expect(daemon.status().activeReleaseId).toBe("v3")
1032
- expect(daemon.status().generationTransition?.phase).toBe("committed")
1033
- expect(daemon.status().generationTransition?.candidateReleaseId).toBe("v3")
1034
- expect(await fetchText(daemon, "/release")).toBe("v3")
1035
- } finally {
1036
- await daemon.shutdown()
1037
- await fs.rm(fixture.root, {force: true, recursive: true})
1038
- }
1039
- })
1040
-
1041
- test("recovery keeps the stale-fence guard failing closed for a genuinely in-flight transition", async () => {
1042
- const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
1043
- const daemon = await startDaemon(fixture.config)
1044
-
1045
- try {
1046
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
1047
- const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
1048
- const reactivate = incumbentCoordinator?.reactivateStrict.bind(incumbentCoordinator)
1049
-
1050
- if (!(incumbentCoordinator && reactivate)) throw new Error("Missing required fixture: incumbentCoordinator && reactivate")
1051
- incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
1052
- const deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
1053
-
1054
- await expect(deployment).rejects.toThrow(/incumbent v1 restoration failed: incumbent restoration rejected/i)
1055
-
1056
- // The failed candidate is still journaled in flight, so the fence must keep failing.
1057
- expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
1058
-
1059
- const exactRecovery = (overrides = {}) => sendControlCommand({
1060
- command: {
1061
- command: "recover-generation-transition",
1062
- previousReleaseId: "v1",
1063
- releaseId: "v2",
1064
- releasePath: fixture.root,
1065
- revision: "v2",
1066
- ...overrides
1067
- },
1068
- path: fixture.config.control.path
1069
- })
1070
-
1071
- await expect(exactRecovery({previousReleaseId: "v2"})).rejects.toThrow(/refusing stale recovery/i)
1072
- await expect(exactRecovery({previousReleaseId: "not-a-release"})).rejects.toThrow(/refusing stale recovery/i)
1073
- expect(daemon.status().activeReleaseId).toBe("v1")
1074
- expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
1075
- } finally {
1076
- await daemon.shutdown()
1077
- await fs.rm(fixture.root, {force: true, recursive: true})
1078
- }
1079
- })
1080
-
1081
- test("failed compensation stops incumbent proxy traffic when jobs restoration fails", async () => {
1082
- const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
1083
- const daemon = await startDaemon(fixture.config)
1084
-
1085
- try {
1086
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
1087
- const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
1088
-
1089
- if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
1090
- incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
1091
- await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
1092
-
1093
- expect(daemon.status().activeReleaseId).toBe("v1")
1094
- expect(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.state).toBe("stopped")
1095
- expect(String(daemon.status().generationTransition?.compensationError)).toMatch(/incumbent restoration rejected/)
1096
- expect(daemon.eventLog.recent().find((event) => event.message === "release generation compensation restoration failed")?.data).toMatchObject({traffic: "unavailable"})
1097
- const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
1098
- expect(unavailable.status).toBe(502)
1099
- } finally {
1100
- await daemon.shutdown()
1101
- await fs.rm(fixture.root, {force: true, recursive: true})
1102
- }
1103
- })
1104
-
1105
- test.skip("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
753
+ test("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
1106
754
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true})
1107
755
  const daemon = await startDaemon(fixture.config)
1108
756
 
@@ -1203,7 +851,7 @@ test.skip("explicit recovery stops the exact failed candidate and fences degrade
1203
851
  expect(daemon.status().generationTransition?.phase).toBe("committed")
1204
852
  expect(await fetchText(daemon, "/release")).toBe("v3")
1205
853
  // Fresh deployment must not re-retire a degraded incumbent generation.
1206
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "retire:bad-v3", "activate:v3"])
854
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "retire:bad-v3", "activate:v3"])
1207
855
  } finally {
1208
856
  await daemon.shutdown()
1209
857
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -1224,7 +872,7 @@ test("candidate activation failure compensates to the incumbent and admits a dif
1224
872
  expect(compensated.generationTransition).toBe(undefined)
1225
873
  expect(await fetchText(daemon, "/release")).toBe("v1")
1226
874
  expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("running")
1227
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
875
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "activate:v1"])
1228
876
 
1229
877
  await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
1230
878
 
@@ -1246,7 +894,6 @@ test("ambiguous candidate activation retires the candidate before reactivating t
1246
894
 
1247
895
  expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual([
1248
896
  "activate:v1",
1249
- "retire:v1",
1250
897
  "activate:v2",
1251
898
  "retire:v2",
1252
899
  "activate:v1"
@@ -1300,7 +947,7 @@ test("candidate activation recovery keeps the fence when a worker-specific resum
1300
947
  expect(status.generationTransition?.phase).toBe("restoring_previous")
1301
948
  expect(String(status.generationTransition?.activationError)).toMatch(/activate command exited non-zero/)
1302
949
  expect(String(status.generationTransition?.compensationError)).toMatch(/reactivate command exited non-zero/)
1303
- expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("quiesced")
950
+ expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("running")
1304
951
  expect(String(restorationEvent?.data.activationError)).toMatch(/activate command exited non-zero/)
1305
952
  expect(String(restorationEvent?.data.error)).toMatch(/reactivate command exited non-zero/)
1306
953
  await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
@@ -1367,14 +1014,8 @@ test("unresolved generation transition fences stop, restart, and rollback mutati
1367
1014
  await expect(daemon.restartProcesses({processId: "beacon"})).rejects.toThrow(/cannot restart.*generation transition.*unresolved/i)
1368
1015
  await expect(daemon.rollback({releaseId: "v2"})).rejects.toThrow(/cannot rollback.*generation transition.*unresolved/i)
1369
1016
 
1370
- expect(statusRelease(daemon, "v1").processes.find((entry) => entry.id === "web")?.state).toBe("stopped")
1371
- const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
1372
-
1373
- expect(unavailable.status).toBe(502)
1374
-
1375
- // The fence refused every mutation; the candidate this deploy retired is stopped by its own
1376
- // compensation drain, not by a fenced command.
1377
- await waitFor(() => statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state === "stopped")
1017
+ expect(statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state).not.toBe("stopped")
1018
+ expect(await fetchText(daemon, "/release")).toBe("v1")
1378
1019
  } finally {
1379
1020
  await daemon.shutdown()
1380
1021
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -1461,7 +1102,7 @@ test("retired generation coordinator remains fenced after exit", async () => {
1461
1102
  expect(exit.code).toBe(null)
1462
1103
  expect(exit.id).toBe("beacon")
1463
1104
  expect(exit.signal).toBe("SIGKILL")
1464
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
1105
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
1465
1106
  expect(stopped.lifecycleRole).toBe("retired")
1466
1107
  expect(stopped.pid).toBe(undefined)
1467
1108
  expect(stopped.restarts).toBe(0)
@@ -1544,14 +1185,13 @@ test("handoff-service quiescence failure is visible and leaves the generation al
1544
1185
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
1545
1186
  const result = await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
1546
1187
 
1547
- const retired = statusRelease(daemon, "v1")
1188
+ await waitFor(() => logs.some((entry) => entry.message === "release drain failed" && entry.data?.releaseId === "v1"), 5000)
1548
1189
 
1549
- expect(retired.state).toBe("draining")
1550
- expect(String(retired.retirementError)).toMatch(/quiet command exited non-zero.*23/)
1551
- expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).toBe("stopping")
1552
- expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state).not.toBe("stopped")
1553
- expect(logs.some((entry) => entry.message === "release retirement quiescence failed" && entry.data?.releaseId === "v1")).toBeTruthy()
1554
- expect(result.retirement).toEqual({error: retired.retirementError, releaseId: "v1", status: "quiescence_failed"})
1190
+ const retired = statusRelease(daemon, "v1")
1191
+ expect(String(retired.retirementError)).toMatch(/quiet command (?:exited non-zero.*23|timed out)/)
1192
+ expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state)
1193
+ expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "worker")?.state)
1194
+ expect(result).toEqual({activeReleaseId: "v2", previousReleaseId: "v1"})
1555
1195
  } finally {
1556
1196
  await daemon.shutdown()
1557
1197
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -2123,246 +1763,6 @@ test("rollback to a still-draining release stops the old instance instead of orp
2123
1763
  }
2124
1764
  })
2125
1765
 
2126
- test("deploy is refused before activation once the draining-generation bound is reached", async () => {
2127
- const fixture = await createFixture()
2128
- const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 1}})
2129
- const daemon = await startDaemon(config)
2130
- /** @type {WebSocket | undefined} */
2131
- let socket
2132
-
2133
- try {
2134
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2135
-
2136
- // A held WebSocket keeps v1 draining after v2 takes traffic over: the measured production
2137
- // shape where a drain outlasts the deploy cadence.
2138
- socket = await openWebSocket(daemon)
2139
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2140
-
2141
- expect(statusRelease(daemon, "v1").state).toBe("draining")
2142
-
2143
- const beforeRefusal = daemon.status()
2144
-
2145
- await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/already draining \(v1 progressing \(1 live connection still owned by the generation\)\)\. Activating v3 would make 2, above the releaseRetention\.maxConcurrentDrainingReleaseGroups bound of 1/)
2146
-
2147
- const afterRefusal = daemon.status()
2148
-
2149
- // Refusing pre-handoff means nothing was started: no new generation, incumbent intact.
2150
- expect(afterRefusal.activeReleaseId).toBe("v2")
2151
- expect(afterRefusal.releases.length).toBe(beforeRefusal.releases.length)
2152
- expect(afterRefusal.releases.some((release) => release.releaseId === "v3")).toBe(false)
2153
- expect(await fetchText(daemon, "/release")).toBe("v2")
2154
- } finally {
2155
- if (socket) socket.close()
2156
- await daemon.shutdown()
2157
- await fs.rm(fixture.root, {force: true, recursive: true})
2158
- }
2159
- })
2160
-
2161
- test("a draining-generation bound of 0 leaves concurrent drains unbounded", async () => {
2162
- const fixture = await createFixture()
2163
- const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 0}})
2164
- const daemon = await startDaemon(config)
2165
- /** @type {WebSocket | undefined} */
2166
- let socket
2167
-
2168
- try {
2169
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2170
- socket = await openWebSocket(daemon)
2171
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2172
- await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
2173
-
2174
- expect(daemon.status().activeReleaseId).toBe("v3")
2175
- expect(statusRelease(daemon, "v1").state).toBe("draining")
2176
- } finally {
2177
- if (socket) socket.close()
2178
- await daemon.shutdown()
2179
- await fs.rm(fixture.root, {force: true, recursive: true})
2180
- }
2181
- })
2182
-
2183
- test("a deploy is the event that asks a retiring generation for its escalation decision, and nothing else does", async () => {
2184
- const config = normalizeConfig({
2185
- application: "demo",
2186
- control: {path: path.join(os.tmpdir(), "rollbridge-escalate-gate.sock")},
2187
- processes: [{command: `${JSON.stringify(process.execPath)} -e "setInterval(() => {}, 1000)"`, id: "web", policy: "proxied", port: {from: 0, to: 0}}],
2188
- proxy: {host: "127.0.0.1", port: 0}
2189
- })
2190
- const daemon = new RollbridgeDaemon({config, logger: () => {}})
2191
- /** @type {Array<{reason: string, releaseId: string}>} */
2192
- const decided = []
2193
-
2194
- /**
2195
- * @param {string} releaseId - Generation id.
2196
- * @param {import("../src/release-group.js").ReleaseState} state - Recorded state of the generation.
2197
- * @returns {import("../src/release-group.js").default} A generation that only records the decision request.
2198
- */
2199
- // eslint-disable-next-line local/no-unknown-jsdoc-type
2200
- const retiringRelease = (releaseId, state) => /** @type {import("../src/release-group.js").default} */ (/** @type {unknown} */ ({
2201
- escalateWedgedGeneration: async (/** @type {{reason: string}} */ options) => {
2202
- decided.push({reason: options.reason, releaseId})
2203
-
2204
- return []
2205
- },
2206
- releaseId,
2207
- state
2208
- }))
2209
-
2210
- daemon.releases.set("draining", retiringRelease("draining", "draining"))
2211
- daemon.releases.set("drained", retiringRelease("drained", "stopped"))
2212
- daemon.releases.set("active", retiringRelease("active", "active"))
2213
-
2214
- // Off by default: enabling the escalation is the operator's decision, not the daemon's.
2215
- await daemon.escalateWedgedGenerations(config, "candidate")
2216
-
2217
- expect(decided).toEqual([])
2218
-
2219
- const escalating = normalizeConfig({...config, releaseRetention: {...config.releaseRetention, escalateWedgedGenerations: true}})
2220
-
2221
- await daemon.escalateWedgedGenerations(escalating, "candidate")
2222
-
2223
- // Every generation still holding a retirement — draining, drained with residue, or failed —
2224
- // is asked; the active one and the candidate itself are not.
2225
- expect(decided).toEqual([{reason: "deploy of candidate", releaseId: "draining"}, {reason: "deploy of candidate", releaseId: "drained"}])
2226
- })
2227
-
2228
- test("status reports the drain classification of each draining generation", async () => {
2229
- const fixture = await createFixture()
2230
- const daemon = await startDaemon(fixture.config)
2231
- /** @type {WebSocket | undefined} */
2232
- let socket
2233
-
2234
- try {
2235
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2236
- socket = await openWebSocket(daemon)
2237
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2238
-
2239
- const draining = statusRelease(daemon, "v1")
2240
-
2241
- expect(draining.state).toBe("draining")
2242
- expect(draining.drain?.status).toBe("progressing")
2243
- expect(draining.drain?.reason).toBe("1 live connection still owned by the generation")
2244
- expect(statusRelease(daemon, "v2").drain).toBe(undefined)
2245
- } finally {
2246
- if (socket) socket.close()
2247
- await daemon.shutdown()
2248
- await fs.rm(fixture.root, {force: true, recursive: true})
2249
- }
2250
- })
2251
-
2252
- test("rollback refuses to restore a drained generation that still holds live processes", async () => {
2253
- const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
2254
- const config = normalizeConfig({
2255
- ...fixture.config,
2256
- processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
2257
- ? {...processConfig, gracefulStopMs: "indefinite"}
2258
- : processConfig),
2259
- proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
2260
- })
2261
- const daemon = await startDaemon(config)
2262
-
2263
- try {
2264
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2265
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2266
-
2267
- // The retirement budget expires, Rollbridge records the generation as drained, and the
2268
- // worker stays alive with its in-flight work — the shape a restore must not re-deploy.
2269
- // The tracked process (the shell wrapper) is already gone, so the only honest record of
2270
- // the leak is the process group that outlived it.
2271
- await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
2272
- await retirementRecordSettled(daemon, "v1", "worker")
2273
-
2274
- const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
2275
-
2276
- if (!worker?.heldProcessGroup) {
2277
- throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
2278
- }
2279
-
2280
- expect(worker.pid).toBeUndefined()
2281
- expect(worker.retirementTimedOut).toBe(true)
2282
- expect(worker.state).toBe("stopped")
2283
- await expect(daemon.rollback({releaseId: "v1"})).rejects.toThrow(/recorded as drained while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch\. The guardian's active release is "v2"/)
2284
- expect(daemon.status().activeReleaseId).toBe("v2")
2285
- expect(await fetchText(daemon, "/release")).toBe("v2")
2286
- } finally {
2287
- await daemon.shutdown()
2288
- await fs.rm(fixture.root, {force: true, recursive: true})
2289
- }
2290
- })
2291
-
2292
- test("deploy refuses to activate a release id whose draining generation still holds live processes", async () => {
2293
- const fixture = await createFixture()
2294
- const daemon = await startDaemon(fixture.config)
2295
- /** @type {WebSocket | undefined} */
2296
- let socket
2297
-
2298
- try {
2299
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2300
-
2301
- // A held WebSocket keeps v1 draining after v2 takes traffic over, so v1's generation is
2302
- // still alive when a restore names its id — the measured 2026-11-15 20:29Z and 20:41Z
2303
- // shape, where the re-activation aborted with a bare guardian provenance mismatch.
2304
- socket = await openWebSocket(daemon)
2305
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2306
-
2307
- const draining = statusRelease(daemon, "v1")
2308
- const heldPid = draining.processes.find((processStatus) => processStatus.id === "web")?.pid
2309
-
2310
- if (!heldPid) throw new Error("the draining generation should still hold its web process")
2311
-
2312
- await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state draining while 1 process of that generation still hold live work \(web: state running, pid \d+\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"; deploy a new release id or reconcile the deployment pointer with `rollbridge status`\./)
2313
-
2314
- const afterRefusal = daemon.status()
2315
-
2316
- // Refusing before activation leaves one generation per release id: nothing was started,
2317
- // the incumbent keeps serving, and the draining generation is still the one status reports.
2318
- expect(afterRefusal.activeReleaseId).toBe("v2")
2319
- expect(afterRefusal.releases.filter((release) => release.releaseId === "v1").length).toBe(1)
2320
- expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "web")?.pid).toBe(heldPid)
2321
- expect(await fetchText(daemon, "/release")).toBe("v2")
2322
- } finally {
2323
- if (socket) socket.close()
2324
- await daemon.shutdown()
2325
- await fs.rm(fixture.root, {force: true, recursive: true})
2326
- }
2327
- })
2328
-
2329
- test("a restore deploy naming a drained generation that outlived its tracked process is refused by name", async () => {
2330
- const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
2331
- const config = normalizeConfig({
2332
- ...fixture.config,
2333
- processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
2334
- ? {...processConfig, gracefulStopMs: "indefinite"}
2335
- : processConfig),
2336
- proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
2337
- })
2338
- const daemon = await startDaemon(config)
2339
-
2340
- try {
2341
- await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2342
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2343
-
2344
- // v1 is recorded as drained while its worker keeps its in-flight work in the group its
2345
- // shell wrapper led. The 2026-10-25 carve-out that lets a dead generation's process key be
2346
- // re-registered does not apply here: the group is alive, so the key is still held.
2347
- await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
2348
- await retirementRecordSettled(daemon, "v1", "worker")
2349
-
2350
- const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
2351
-
2352
- if (!worker?.heldProcessGroup) {
2353
- throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
2354
- }
2355
-
2356
- await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state stopped while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"/)
2357
- expect(daemon.status().activeReleaseId).toBe("v2")
2358
- expect(daemon.status().releases.filter((release) => release.releaseId === "v1").length).toBe(1)
2359
- expect(await fetchText(daemon, "/release")).toBe("v2")
2360
- } finally {
2361
- await daemon.shutdown()
2362
- await fs.rm(fixture.root, {force: true, recursive: true})
2363
- }
2364
- })
2365
-
2366
1766
  test("the rollback control command switches traffic over the socket", async () => {
2367
1767
  const fixture = await createFixture()
2368
1768
  const daemon = await startDaemon(fixture.config)
@@ -2570,20 +1970,18 @@ test("deploy can ensure the daemon before sending the release command", async ()
2570
1970
  })
2571
1971
 
2572
1972
  /**
2573
- * @param {{companionReplicas?: number, gatedWeb?: boolean, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
2574
- * @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string, workerReadyPath: string}>} Fixture data.
1973
+ * @param {{companionReplicas?: number, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
1974
+ * @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
2575
1975
  */
2576
1976
  async function createFixture(options = {}) {
2577
1977
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-test-"))
2578
1978
  const serviceLogPath = path.join(root, "service.log")
2579
1979
  const serviceQuietPath = path.join(root, "service.quiet")
2580
1980
  const lifecycleLogPath = path.join(root, "service.lifecycle")
2581
- const workerReadyPath = path.join(root, "worker.ready")
2582
1981
  const activationGatePath = path.join(root, "activation.allow")
2583
1982
  const retirementGatePath = path.join(root, "retirement.allow")
2584
1983
  const singletonLogPath = path.join(root, "singleton.log")
2585
1984
  const statePath = path.join(root, "rollbridge.state.json")
2586
- const gatePath = path.join(root, "stc788-gate")
2587
1985
  /** @type {Array<Record<string, import("../src/json.js").JsonValue>>} */
2588
1986
  const processes = []
2589
1987
 
@@ -2629,7 +2027,7 @@ async function createFixture(options = {}) {
2629
2027
 
2630
2028
  if (options.nonBlockingDrainWorker) {
2631
2029
  processes.push({
2632
- command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0})); require("node:fs").writeFileSync(${JSON.stringify(workerReadyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
2030
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0})); setInterval(() => {}, 1000)`)}`,
2633
2031
  id: "worker",
2634
2032
  ...(options.workerReactivationLifecycle ? {lifecycle: {
2635
2033
  quietCommand: `printf 'worker-retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`,
@@ -2653,11 +2051,9 @@ async function createFixture(options = {}) {
2653
2051
  }
2654
2052
 
2655
2053
  processes.push({
2656
- command: options.webCommand || (options.gatedWeb
2657
- ? `${JSON.stringify(process.execPath)} ${JSON.stringify(gatedAppPath)} --gate=${JSON.stringify(gatePath)}`
2658
- : options.webDependsOnService
2659
- ? `${JSON.stringify(process.execPath)} ${JSON.stringify(dependentAppPath)}`
2660
- : `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`),
2054
+ command: options.webCommand || (options.webDependsOnService
2055
+ ? `${JSON.stringify(process.execPath)} ${JSON.stringify(dependentAppPath)}`
2056
+ : `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`),
2661
2057
  health: {
2662
2058
  intervalMs: 50,
2663
2059
  path: "/ping",
@@ -2697,51 +2093,7 @@ async function createFixture(options = {}) {
2697
2093
  ...((options.persistState || options.handoffServiceActivate) ? {ownerRecovery: {reconnectGraceMs: 30000}, statePath} : {})
2698
2094
  })
2699
2095
 
2700
- return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath, workerReadyPath}
2701
- }
2702
-
2703
- /**
2704
- * Config where both processes run the deferred-boot fixture, so a measured boot budget can be
2705
- * compared against a dependency wait that must not be charged to it. Omitting
2706
- * webHealthTimeoutMs leaves both the per-process and the release health budget unconfigured, so
2707
- * the normalized config carries the default boot budget instead of a test-chosen one.
2708
- * @param {{config: import("../src/config.js").RollbridgeConfig, root: string}} fixture - Fixture.
2709
- * @param {{dependencyTimeoutMs: number, serviceBootMs: number, webBootMs: number, webHealthTimeoutMs?: number}} options - Budgets.
2710
- * @returns {import("../src/config.js").RollbridgeConfig} Config for the deferred-boot processes.
2711
- */
2712
- function deferredBootConfig(fixture, options) {
2713
- const appCommand = `${JSON.stringify(process.execPath)} ${JSON.stringify(deferredBootAppPath)}`
2714
- /** @type {Partial<import("../src/config.js").ProxyConfig>} */
2715
- const proxyWithoutHealthBudget = structuredClone(fixture.config.proxy)
2716
-
2717
- delete proxyWithoutHealthBudget.healthTimeoutMs
2718
-
2719
- return normalizeConfig({
2720
- ...fixture.config,
2721
- ...(options.webHealthTimeoutMs === undefined ? {proxy: proxyWithoutHealthBudget} : {}),
2722
- processes: [
2723
- {
2724
- command: appCommand,
2725
- deployStrategy: "handoff",
2726
- env: {ROLLBRIDGE_BOOT_MS: String(options.serviceBootMs)},
2727
- id: "beacon",
2728
- policy: "service",
2729
- port: {from: 15000, to: 15099}
2730
- },
2731
- {
2732
- command: appCommand,
2733
- dependsOn: ["beacon"],
2734
- dependencyTimeoutMs: options.dependencyTimeoutMs,
2735
- env: {ROLLBRIDGE_BOOT_MS: String(options.webBootMs), ROLLBRIDGE_DEPENDENCY_PORT: "{{ports.beacon}}"},
2736
- ...(options.webHealthTimeoutMs === undefined
2737
- ? {}
2738
- : {health: {intervalMs: 50, path: "/ping", timeoutMs: options.webHealthTimeoutMs}}),
2739
- id: "web",
2740
- policy: "proxied",
2741
- port: {from: 0, to: 0}
2742
- }
2743
- ]
2744
- })
2096
+ return {activationGatePath, config, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
2745
2097
  }
2746
2098
 
2747
2099
  /**
@@ -2815,27 +2167,6 @@ function statusRelease(daemon, releaseId) {
2815
2167
  return release
2816
2168
  }
2817
2169
 
2818
- /**
2819
- * Waits until the daemon has observed the exit of a retiring process's tracked wrapper.
2820
- *
2821
- * A bounded retirement stop returns the moment its budget expires — that is the whole point of
2822
- * the bound — while the wrapper it just signalled is still on its way out, so for a few
2823
- * milliseconds the record still carries the wrapper pid and reports the process `quiesced`. The
2824
- * residue the drain specs assert (the process group that outlived the tracked process) is
2825
- * recorded by that exit observation, not by the stop call, so a spec that reads the record
2826
- * straight after the stop returns races the observation it is describing. Measured on
2827
- * 2026-11-16: `stopRelease` returned after 106 ms with `{pid: 13640, state: "quiesced"}` and the
2828
- * record settled to `{heldProcessGroup: {liveMemberCount: 1, pgid: 13640}, state: "stopped"}`
2829
- * within the next 50 ms.
2830
- * @param {RollbridgeDaemon} daemon - Running daemon.
2831
- * @param {string} releaseId - Retained generation to observe.
2832
- * @param {string} processId - Process whose wrapper exit must be observed.
2833
- * @returns {Promise<void>} Resolves once the process record has settled.
2834
- */
2835
- async function retirementRecordSettled(daemon, releaseId, processId) {
2836
- await waitFor(() => statusRelease(daemon, releaseId).processes.find((processStatus) => processStatus.id === processId)?.state === "stopped")
2837
- }
2838
-
2839
2170
  /**
2840
2171
  * Maps process id to pid across the active release, services, and singletons.
2841
2172
  * @param {import("../src/daemon.js").DaemonStatus} status - Daemon status payload.
@@ -2908,21 +2239,6 @@ async function waitFor(callback, timeoutMs = 3000) {
2908
2239
  throw new Error("Timed out waiting for condition")
2909
2240
  }
2910
2241
 
2911
- /**
2912
- * @param {string} filePath - File to await.
2913
- * @returns {Promise<void>} Resolves once the file exists.
2914
- */
2915
- async function waitForFile(filePath) {
2916
- await waitFor(async () => {
2917
- try {
2918
- await fs.access(filePath)
2919
- return true
2920
- } catch {
2921
- return false
2922
- }
2923
- })
2924
- }
2925
-
2926
2242
  /**
2927
2243
  * Resolves from retained output or the exact event that records it, and rejects if the
2928
2244
  * supervised process exits first.