rollbridge 0.1.72 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  24. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  25. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  26. package/docs/cli.md +128 -3
  27. package/docs/compose-services.md +222 -0
  28. package/docs/config.md +177 -6
  29. package/docs/troubleshooting.md +97 -6
  30. package/docs/velocious.md +1 -1
  31. package/docs/workers.md +11 -0
  32. package/examples/tensorbuzz.com.js +3 -2
  33. package/package.json +1 -1
  34. package/src/cli.js +68 -2
  35. package/src/compose-unit.js +725 -0
  36. package/src/config.js +346 -8
  37. package/src/daemon.js +410 -73
  38. package/src/doctor.js +74 -0
  39. package/src/drain-policy.js +252 -0
  40. package/src/guardian-client.js +67 -8
  41. package/src/health.js +99 -8
  42. package/src/managed-process.js +197 -16
  43. package/src/process-guardian.js +212 -22
  44. package/src/process-memory.js +62 -2
  45. package/src/process-order.js +26 -0
  46. package/src/release-group.js +435 -37
  47. package/tensorbuzz.yml +20 -1
  48. package/test/completion.test.js +5 -2
  49. package/test/compose-config.test.js +160 -0
  50. package/test/compose-docker.test.js +297 -0
  51. package/test/compose-unit.test.js +415 -0
  52. package/test/config-validation.test.js +135 -5
  53. package/test/drain-policy.test.js +328 -0
  54. package/test/fixtures/deferred-boot-app.js +106 -0
  55. package/test/fixtures/gated-app.js +50 -0
  56. package/test/guardian-client.test.js +119 -5
  57. package/test/health.test.js +74 -1
  58. package/test/managed-process.test.js +260 -1
  59. package/test/owner-recovery.test.js +125 -21
  60. package/test/owner-replacement.test.js +175 -14
  61. package/test/process-memory.test.js +47 -1
  62. package/test/release-group.test.js +408 -2
  63. package/test/rollbridge.test.js +774 -72
  64. package/test/shutdown-completion.test.js +119 -0
  65. package/test/shutdown-retirement-bound.test.js +162 -0
  66. package/test/support/process.js +3 -1
  67. package/test/tensorbuzz-config.test.js +228 -0
@@ -19,10 +19,12 @@ describe("rollbridge", () => {
19
19
  const currentDir = path.dirname(fileURLToPath(import.meta.url))
20
20
  const binPath = path.join(currentDir, "..", "bin", "rollbridge")
21
21
  const dependentAppPath = path.join(currentDir, "fixtures", "dependent-app.js")
22
+ const deferredBootAppPath = path.join(currentDir, "fixtures", "deferred-boot-app.js")
22
23
  const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
23
24
  const memoryHogPath = path.join(currentDir, "fixtures", "memory-hog.js")
24
25
  const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
25
26
  const singletonAppPath = path.join(currentDir, "fixtures", "singleton-app.js")
27
+ const gatedAppPath = path.join(currentDir, "fixtures", "gated-app.js")
26
28
  const linuxTest = process.platform === "linux" ? test : test.skip
27
29
 
28
30
  test("a nonBlockingDrain worker stops immediately while its release is still draining", async () => {
@@ -59,46 +61,82 @@ test("a nonBlockingDrain worker stops immediately while its release is still dra
59
61
  }
60
62
  })
61
63
 
62
- test("deploy switches new HTTP traffic while old WebSockets drain", async () => {
63
- const fixture = await createFixture()
64
- const daemon = await startDaemon(fixture.config)
64
+ test("an explicit stop returns within its retirement budget instead of waiting on a worker that is still draining", async () => {
65
+ const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 30000})
66
+ const config = normalizeConfig({
67
+ ...fixture.config,
68
+ processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
69
+ ? {...processConfig, gracefulStopMs: "indefinite"}
70
+ : processConfig),
71
+ proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 100}
72
+ })
73
+ /** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
74
+ const logs = []
75
+ const daemon = new RollbridgeDaemon({config, logger: (message, data) => logs.push({message, data})})
76
+ /** @type {number | undefined} */
77
+ let workerPid
65
78
 
66
79
  try {
80
+ await daemon.start()
67
81
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
68
- expect(await fetchText(daemon, "/release")).toBe("v1")
69
82
 
70
- const websocket = await openWebSocket(daemon)
83
+ workerPid = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.pid
71
84
 
72
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
73
- expect(await fetchText(daemon, "/release")).toBe("v2")
85
+ if (typeof workerPid !== "number") throw new Error("Expected the worker to be running before the stop")
74
86
 
75
- const drainingRelease = statusRelease(daemon, "v1")
76
- expect(drainingRelease.state).toBe("draining")
77
- expect(drainingRelease.connections.websocket).toBe(1)
87
+ const startedAtMs = Date.now()
78
88
 
79
- websocket.close()
80
- await waitFor(async () => statusRelease(daemon, "v1").state === "stopped")
89
+ await daemon.stopRelease("v1")
90
+
91
+ // Before this the command had no deadline for a worker whose graceful window is "indefinite",
92
+ // so it never returned and held the owner-mutation fence for the whole drain, refusing every
93
+ // later deploy for hours (TensorBuzz production, 2026-11-16). The bound is far below the
94
+ // worker's own 30 000 ms stop delay because the stop must still let the generation's other
95
+ // processes exit; what it must not do is wait for the worker at all.
96
+ expect(Date.now() - startedAtMs).toBeLessThan(10000)
97
+
98
+ await retirementRecordSettled(daemon, "v1", "worker")
99
+
100
+ const stopped = statusRelease(daemon, "v1")
101
+ const stoppedWorker = stopped.processes.find((processStatus) => processStatus.id === "worker")
102
+
103
+ expect(stopped.state).toBe("stopped")
104
+ expect(stoppedWorker?.retirementTimedOut).toBe(true)
105
+ // The tracked record is the shell wrapper, which is already gone; the worker it launched is
106
+ // still draining inside the same process group, and that is the residue the stop must report.
107
+ expect(stoppedWorker?.heldProcessGroup?.liveMemberCount).toBeGreaterThan(0)
108
+ expect(logs.filter((entry) => entry.message === "release stop gave up waiting for a retiring process")).toMatchObject([
109
+ {data: {processId: "worker", releaseId: "v1", timeoutMs: 100}}
110
+ ])
111
+ expect(logs.filter((entry) => entry.message === "release stopped")).toMatchObject([{data: {releaseId: "v1"}}])
81
112
  } finally {
82
113
  await daemon.shutdown()
114
+ if (workerPid !== undefined) {
115
+ try { process.kill(-workerPid, "SIGKILL") } catch (_error) { /* The group already exited. */ }
116
+ }
83
117
  await fs.rm(fixture.root, {force: true, recursive: true})
84
118
  }
85
119
  })
86
120
 
87
- test("deploy does not wait for retirement and force-stops an old generation by the drain deadline", async () => {
88
- const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 10000})
89
- const config = normalizeConfig({...fixture.config, proxy: {...fixture.config.proxy, drainTimeoutMs: 50, forceStopTimeoutMs: 50}})
90
- const daemon = await startDaemon(config)
121
+ test("deploy switches new HTTP traffic while old WebSockets drain", async () => {
122
+ const fixture = await createFixture()
123
+ const daemon = await startDaemon(fixture.config)
91
124
 
92
125
  try {
93
126
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
94
- const startedAt = Date.now()
127
+ expect(await fetchText(daemon, "/release")).toBe("v1")
95
128
 
96
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
129
+ const websocket = await openWebSocket(daemon)
97
130
 
98
- expect(Date.now() - startedAt).toBeLessThan(1000)
131
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
99
132
  expect(await fetchText(daemon, "/release")).toBe("v2")
100
- await waitFor(() => statusRelease(daemon, "v1").state === "stopped", 8000)
101
- expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("stopped")
133
+
134
+ const drainingRelease = statusRelease(daemon, "v1")
135
+ expect(drainingRelease.state).toBe("draining")
136
+ expect(drainingRelease.connections.websocket).toBe(1)
137
+
138
+ websocket.close()
139
+ await waitFor(async () => statusRelease(daemon, "v1").state === "stopped")
102
140
  } finally {
103
141
  await daemon.shutdown()
104
142
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -146,7 +184,7 @@ test("deploy reloads process config and retires the previous worker with the ref
146
184
 
147
185
  await writeConfigFile(refreshedConfig, fixture.root)
148
186
  await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
149
- await waitFor(() => statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state === "stopped", 5000)
187
+ await waitFor(() => statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state === "stopped", 1000)
150
188
 
151
189
  expect(daemon.config.processes.find((processConfig) => processConfig.id === "worker")?.gracefulStopMs).toBe(50)
152
190
  expect(await fetchText(daemon, "/release")).toBe("v2")
@@ -263,6 +301,139 @@ test("failed release startup logs process output and cleanup status", async () =
263
301
  }
264
302
  })
265
303
 
304
+ test("the dependency wait is charged to its own budget, not to the proxied boot budget", async () => {
305
+ const fixture = await createFixture()
306
+ const config = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 1200, webBootMs: 600, webHealthTimeoutMs: 900})
307
+ /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
308
+ const logs = []
309
+ const daemon = new RollbridgeDaemon({config, logger: (message, data = {}) => logs.push({data, message})})
310
+
311
+ await daemon.start()
312
+
313
+ try {
314
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
315
+
316
+ const dependencyLog = logs.find((entry) => entry.message === "release process dependency ready")
317
+ const healthLog = logs.find((entry) => entry.message === "release candidate health ready")
318
+
319
+ if (!dependencyLog?.data || !healthLog?.data) throw new Error("expected the dependency wait and the boot time to be logged")
320
+ // The dependency took ~1200ms and the app boot ~600ms. The boot budget is 900ms, so this
321
+ // deploy only succeeds because the dependency wait is charged to a budget of its own.
322
+ expect(Number(dependencyLog.data.elapsedMs)).toBeGreaterThanOrEqual(1000)
323
+ expect(Number(healthLog.data.bootMs)).toBeLessThan(900)
324
+ expect(Number(healthLog.data.waitedForDependenciesMs)).toBeGreaterThanOrEqual(1000)
325
+ expect(await fetchText(daemon, "/release")).toBe("v1")
326
+ } finally {
327
+ await daemon.shutdown()
328
+ await fs.rm(fixture.root, {force: true, recursive: true})
329
+ }
330
+ })
331
+
332
+ test("a release whose app boots past the old 30000ms window publishes on the default boot budget while the incumbent serves", {timeoutMs: 180000}, async () => {
333
+ const fixture = await createFixture()
334
+ // Measured on TensorBuzz production (server3, 2026-11-06): the candidate web process was
335
+ // still initialising when the 30000ms release health deadline expired and was killed at
336
+ // 30287ms without ever logging a listening line. 32000ms reproduces a boot the old default
337
+ // killed; nothing here configures a health budget, so the release runs on the default.
338
+ const lateBootMs = 32000
339
+ const incumbentConfig = deferredBootConfig(fixture, {dependencyTimeoutMs: 30000, serviceBootMs: 0, webBootMs: 0})
340
+ const configPath = await writeConfigFile(incumbentConfig, fixture.root)
341
+ /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
342
+ const logs = []
343
+ const daemon = new RollbridgeDaemon({config: incumbentConfig, configPath, logger: (message, data = {}) => logs.push({data, message})})
344
+
345
+ await daemon.start()
346
+
347
+ try {
348
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
349
+ expect(await fetchText(daemon, "/release")).toBe("v1")
350
+
351
+ await writeConfigFile(deferredBootConfig(fixture, {dependencyTimeoutMs: 30000, serviceBootMs: 0, webBootMs: lateBootMs}), fixture.root)
352
+ const deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
353
+
354
+ // The candidate is still booting a second later: the incumbent keeps serving traffic.
355
+ await new Promise((resolve) => setTimeout(resolve, 1000))
356
+ expect(await fetchText(daemon, "/release")).toBe("v1")
357
+ await deployment
358
+
359
+ const ready = logs.find((entry) => entry.message === "release candidate health ready" && entry.data?.releaseId === "v2")
360
+
361
+ if (!ready?.data) throw new Error("expected the v2 boot time to be recorded in the event log")
362
+ expect(Number(ready.data.bootMs)).toBeGreaterThanOrEqual(lateBootMs)
363
+ expect(Number(ready.data.healthGateMs)).toBeGreaterThanOrEqual(lateBootMs)
364
+ expect(Number(ready.data.deadlineMs)).toBe(120000)
365
+ expect(daemon.status().activeReleaseId).toBe("v2")
366
+ expect(await fetchText(daemon, "/release")).toBe("v2")
367
+ } finally {
368
+ await daemon.shutdown()
369
+ await fs.rm(fixture.root, {force: true, recursive: true})
370
+ }
371
+ })
372
+
373
+ test("a dependency that never opens fails the deploy on its own budget and keeps the incumbent serving", async () => {
374
+ const fixture = await createFixture()
375
+ const initialConfig = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 0, webHealthTimeoutMs: 900})
376
+ const configPath = await writeConfigFile(initialConfig, fixture.root)
377
+ /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
378
+ const logs = []
379
+ const daemon = new RollbridgeDaemon({config: initialConfig, configPath, logger: (message, data = {}) => logs.push({data, message})})
380
+
381
+ await daemon.start()
382
+
383
+ try {
384
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
385
+ expect(await fetchText(daemon, "/release")).toBe("v1")
386
+
387
+ // v2's service never binds its port, so the dependency gate runs out before the app is started.
388
+ await writeConfigFile(deferredBootConfig(fixture, {dependencyTimeoutMs: 300, serviceBootMs: 60000, webBootMs: 0, webHealthTimeoutMs: 900}), fixture.root)
389
+ await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/did not accept a connection within 300ms/)
390
+
391
+ expect(daemon.status().activeReleaseId).toBe("v1")
392
+ expect(await fetchText(daemon, "/release")).toBe("v1")
393
+
394
+ const failure = logs.find((entry) => entry.message === "release startup failed" && entry.data?.phase === "before cleanup")
395
+
396
+ if (!failure?.data) throw new Error("expected the dependency failure to name its budget")
397
+ expect(failure.data.stage).toBe("dependency")
398
+ expect(failure.data.deadlineMs).toBe(300)
399
+ expect(Number(failure.data.dependencyWaitMs)).toBeGreaterThanOrEqual(300)
400
+ expect(failure.data.readyProcesses).toEqual(["beacon"])
401
+ expect(failure.data.unreadyProcesses).toEqual([])
402
+ } finally {
403
+ await daemon.shutdown()
404
+ await fs.rm(fixture.root, {force: true, recursive: true})
405
+ }
406
+ })
407
+
408
+ test("a startup timeout names the deadline, the elapsed boot, the ready processes, and the last output", async () => {
409
+ const fixture = await createFixture()
410
+ const config = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 1200, webHealthTimeoutMs: 400})
411
+ /** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
412
+ const logs = []
413
+ const daemon = new RollbridgeDaemon({config, logger: (message, data = {}) => logs.push({data, message})})
414
+
415
+ await daemon.start()
416
+
417
+ try {
418
+ await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/health deadline/)
419
+
420
+ const failure = logs.find((entry) => entry.message === "release startup failed" && entry.data?.phase === "before cleanup")
421
+
422
+ if (!failure?.data) throw new Error("expected the startup failure to name its budget")
423
+ expect(failure.data.stage).toBe("health")
424
+ expect(failure.data.deadlineMs).toBe(400)
425
+ expect(Number(failure.data.elapsedMs)).toBeGreaterThanOrEqual(400)
426
+ expect(Number(failure.data.probes)).toBeGreaterThanOrEqual(1)
427
+ expect(failure.data.readyProcesses).toEqual(["beacon"])
428
+ expect(failure.data.unreadyProcesses).toEqual(["web"])
429
+ expect(String(failure.data.lastOutput)).toMatch(/accepted a connection/)
430
+ expect(String(failure.data.lastOutputAt)).toMatch(/^\d{4}-\d{2}-\d{2}T/)
431
+ } finally {
432
+ await daemon.shutdown()
433
+ await fs.rm(fixture.root, {force: true, recursive: true})
434
+ }
435
+ })
436
+
266
437
  test("singleton processes restart without overlap during deploy", async () => {
267
438
  const fixture = await createFixture({includeSingleton: true})
268
439
  const daemon = await startDaemon(fixture.config)
@@ -289,33 +460,70 @@ test("singleton processes restart without overlap during deploy", async () => {
289
460
  }
290
461
  })
291
462
 
292
- test("candidate activation switches traffic before a blocked singleton replacement completes", async () => {
293
- const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, webDependsOnService: true})
294
- const retirementGatePath = path.join(fixture.root, "retirement.gate")
295
- const retirementWaitingPath = path.join(fixture.root, "retirement.waiting")
463
+ // Budget for waiting on the old jobs generation's quiet marker in the spec below. Measured with the
464
+ // marker read directly: the spec completes in 1.740s on this host, so this is not a boot budget but a
465
+ // generous ceiling on a handoff that already happened; the number only decides how long a genuinely
466
+ // stuck quiescence takes to report itself, and it now reports the marker path instead of the runner's
467
+ // ambient 60s default.
468
+ const SINGLETON_QUIET_MARKER_TIMEOUT_MS = 20000
469
+
470
+ test("candidate activation quiesces the old jobs generation before a blocked singleton replacement completes", async () => {
471
+ const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, includeSingleton: true, webDependsOnService: true})
472
+ const singletonGatePath = path.join(fixture.root, "singleton-replacement.gate")
473
+ const singletonQuietPath = path.join(fixture.root, "singleton-replacement.quiet")
474
+ const fifo = spawn("mkfifo", [singletonGatePath])
475
+
476
+ await once(fifo, "exit")
477
+
296
478
  const config = normalizeConfig({
297
479
  ...fixture.config,
298
- processes: fixture.config.processes.map((processConfig) => processConfig.id === "beacon"
299
- ? {...processConfig, lifecycle: {activateCommand: processConfig.lifecycle.activateCommand, quietCommand: `printf 'waiting\\n' > ${JSON.stringify(retirementWaitingPath)}; while [ ! -f ${JSON.stringify(retirementGatePath)} ]; do sleep 0.02; done; printf 'retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(fixture.lifecycleLogPath)}`}}
480
+ processes: fixture.config.processes.map((processConfig) => processConfig.id === "jobs-main"
481
+ ? {...processConfig, lifecycle: {quietCommand: `printf 'quiet\\n' > ${JSON.stringify(singletonQuietPath)}; read -r _ < ${JSON.stringify(singletonGatePath)}`}}
300
482
  : processConfig)
301
483
  })
302
484
  const daemon = await startDaemon(config)
485
+ /** @type {Promise<Record<string, import("../src/json.js").JsonValue>> | undefined} */
303
486
  let deployPromise
487
+ let singletonGateReleased = false
304
488
 
305
489
  try {
306
490
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
491
+
307
492
  deployPromise = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
308
- await waitFor(async () => (await fs.readFile(retirementWaitingPath, "utf8").catch(() => "")) === "waiting\n")
493
+ let deploySettled = false
494
+
495
+ void deployPromise.then(() => { deploySettled = true }, () => { deploySettled = true })
496
+
497
+ // Wait on the durable marker the old jobs generation writes, not on an inotify event for it.
498
+ // fs.watch surfaces nothing in runtimes whose filesystem does not deliver events (measured in
499
+ // this container: a minimal fs.watch receives 0 events for a file written by an external
500
+ // process, on both overlayfs /tmp and ext3 /data, with max_user_watches at 1048576), and an
501
+ // unguarded event loop turns that into a bare runner timeout naming no stage. Reading the
502
+ // marker is the same observation with a fallback the event never had, and the three assertions
503
+ // below are unchanged: candidate traffic active, deploy still pending, old generation quiesced.
504
+ const quietDeadline = Date.now() + SINGLETON_QUIET_MARKER_TIMEOUT_MS
505
+ let sawQuietMarker = false
506
+
507
+ while (Date.now() < quietDeadline && !sawQuietMarker) {
508
+ sawQuietMarker = await fs.readFile(singletonQuietPath, "utf8").then((content) => content.includes("quiet"), () => false)
509
+ if (!sawQuietMarker) await new Promise((resolve) => setTimeout(resolve, 10))
510
+ }
511
+
512
+ if (!sawQuietMarker) throw new Error(`Old jobs generation never wrote its quiet marker within ${SINGLETON_QUIET_MARKER_TIMEOUT_MS}ms: ${singletonQuietPath}`)
309
513
 
310
514
  // Candidate traffic must already be active.
311
515
  expect(daemon.status().activeReleaseId).toBe("v2")
312
- await fs.writeFile(retirementGatePath, "continue\n")
516
+ // Deploy must remain pending on singleton replacement.
517
+ expect(deploySettled).toBe(false)
518
+ // Old jobs-main must quiesce before singleton replacement completes.
519
+ expect(await fs.readFile(fixture.serviceQuietPath, "utf8")).toBe("v1\n")
520
+
521
+ await fs.writeFile(singletonGatePath, "continue\n")
522
+ singletonGateReleased = true
313
523
  await deployPromise
314
- await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).join(",") === "activate:v1,activate:v2,retire:v1")
315
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
316
524
  } finally {
317
- if (deployPromise) {
318
- await fs.writeFile(retirementGatePath, "continue\n")
525
+ if (deployPromise && !singletonGateReleased) {
526
+ await fs.writeFile(singletonGatePath, "continue\n")
319
527
  await deployPromise.catch(() => {})
320
528
  }
321
529
 
@@ -464,7 +672,7 @@ test("handoff services stop after release-local dependents finish draining", asy
464
672
  })
465
673
 
466
674
  test("candidate activation retires jobs-main with its workers without waiting for the generation to drain", async () => {
467
- const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, nonBlockingDrainWorker: true, webDependsOnService: true, workerStopDelayMs: 5000})
675
+ const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, nonBlockingDrainWorker: true, webDependsOnService: true, workerStopDelayMs: 1000})
468
676
  const daemon = await startDaemon(fixture.config)
469
677
 
470
678
  try {
@@ -477,28 +685,20 @@ test("candidate activation retires jobs-main with its workers without waiting fo
477
685
  expect(oldWorker?.pid).toBeTruthy()
478
686
 
479
687
  await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
480
- await waitFor(async () => (await fs.readFile(fixture.serviceQuietPath, "utf8").catch(() => "")) === "v1\n", 5000)
481
688
 
482
689
  // Traffic must switch only after the complete candidate is healthy.
483
690
  expect(daemon.status().activeReleaseId).toBe("v2")
484
- // Old jobs-main is retired after traffic switches, without holding deployment completion.
485
- await waitFor(async () => {
486
- try {
487
- return (await fs.readFile(fixture.serviceQuietPath, "utf8")) === "v1\n"
488
- } catch (error) {
489
- if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return false
490
- throw error
491
- }
492
- })
691
+ // Old jobs-main must quiesce immediately after candidate activation.
692
+ expect(await fs.readFile(fixture.serviceQuietPath, "utf8")).toBe("v1\n")
493
693
 
494
694
  const retired = statusRelease(daemon, "v1")
495
695
 
496
696
  // Deployment completion must not wait for the old jobs generation.
497
697
  expect(retired.state).toBe("draining")
498
- // Old jobs-main remains in retirement while its bounded drain runs.
499
- expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state)
500
- // The old worker may already have completed its bounded drain; the deploy did not wait for it.
501
- expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "worker")?.state)
698
+ // Old jobs-main must remain alive and quiesced with its draining workers.
699
+ expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).toBe("quiesced")
700
+ // Old worker must remain in its original generation until accepted work settles.
701
+ expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state).not.toBe("stopped")
502
702
  // Old and new workers must retain distinct jobs-main endpoints.
503
703
  expect(statusRelease(daemon, "v2").ports.beacon).not.toBe(retired.ports.beacon)
504
704
  } finally {
@@ -518,7 +718,7 @@ test("opt-in generation lifecycle activates the candidate before retiring the ol
518
718
 
519
719
  await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
520
720
 
521
- await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).join(",") === "activate:v1,activate:v2,retire:v1")
721
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
522
722
  expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
523
723
  expect(daemon.status().activeReleaseId).toBe("v2")
524
724
  expect(daemon.status().generationTransition?.phase).toBe("committed")
@@ -528,6 +728,43 @@ test("opt-in generation lifecycle activates the candidate before retiring the ol
528
728
  }
529
729
  })
530
730
 
731
+ test("retirement cannot stop the incumbent web or jobs generation before candidate traffic is active", async () => {
732
+ const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, webDependsOnService: true})
733
+ const quietMarkerPath = path.join(fixture.root, "retirement.started")
734
+ const config = normalizeConfig({
735
+ ...fixture.config,
736
+ processes: fixture.config.processes.map((processConfig) => processConfig.id === "beacon"
737
+ ? {...processConfig, lifecycle: {quietCommand: `printf started > ${JSON.stringify(quietMarkerPath)}; sleep 0.5`}}
738
+ : processConfig)
739
+ })
740
+ const daemon = await startDaemon(config)
741
+ /** @type {Promise<Record<string, import("../src/json.js").JsonValue>> | undefined} */
742
+ let deployment
743
+
744
+ try {
745
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
746
+ deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
747
+
748
+ await waitFor(async () => (await fs.readFile(quietMarkerPath, "utf8").catch(() => "")) === "started")
749
+
750
+ const status = daemon.status()
751
+ const previous = status.releases.find((release) => release.releaseId === "v1")
752
+ const previousWeb = previous?.processes.find((processStatus) => processStatus.id === "web")
753
+ const previousJobs = previous?.processes.find((processStatus) => processStatus.id === "beacon")
754
+
755
+ expect(status.activeReleaseId).toBe("v2")
756
+ expect(previousWeb?.state).toBe("running")
757
+ expect(previousWeb?.pid).toBeTruthy()
758
+ expect(previousJobs?.pid).toBeTruthy()
759
+
760
+ await deployment
761
+ } finally {
762
+ await deployment?.catch(() => {})
763
+ await daemon.shutdown()
764
+ await fs.rm(fixture.root, {force: true, recursive: true})
765
+ }
766
+ })
767
+
531
768
  test("manual restart reaches the active handoff coordinator and restores its lifecycle role", async () => {
532
769
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, webDependsOnService: true})
533
770
  const daemon = await startDaemon(fixture.config)
@@ -613,7 +850,7 @@ test("exact committed retry finishes pending singleton replacement before succes
613
850
 
614
851
  expect(daemon.status().generationTransition?.phase).toBe("committed")
615
852
  expect(daemon.status().singletons[0]?.process.state).toBe("running")
616
- await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).join(",") === "activate:v1,activate:v2,retire:v1")
853
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
617
854
  expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
618
855
  } finally {
619
856
  await daemon.shutdown()
@@ -638,22 +875,21 @@ test("first generation is not committed when its activation acknowledgement fail
638
875
  }
639
876
  })
640
877
 
641
- test("retirement acknowledgement failure is named without blocking the committed deployment", async () => {
878
+ test("retirement acknowledgement failure is reported after traffic switches", async () => {
642
879
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceQuietFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
643
880
  const daemon = await startDaemon(fixture.config)
644
881
 
645
882
  try {
646
883
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
647
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
884
+ const result = await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
885
+ await waitFor(() => Boolean(statusRelease(daemon, "v1").retirementError))
648
886
 
649
887
  const failed = daemon.status()
650
888
 
651
889
  expect(failed.activeReleaseId).toBe("v2")
652
890
  expect(failed.generationTransition?.phase).toBe("committed")
653
- await waitFor(() => /quiet command exited non-zero/u.test(String(statusRelease(daemon, "v1").retirementError)))
654
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2"])
655
- await fs.writeFile(fixture.retirementGatePath, "allow\n")
656
- expect(daemon.status().activeReleaseId).toBe("v2")
891
+ expect(String(statusRelease(daemon, "v1").retirementError)).toMatch(/quiet command exited non-zero/)
892
+ expect(result.activeReleaseId).toBe("v2")
657
893
  } finally {
658
894
  await daemon.shutdown()
659
895
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -719,6 +955,7 @@ test("candidate activation failure reports restoration failure and exact recover
719
955
  const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
720
956
 
721
957
  expect(persisted?.generationTransition).toBe(undefined)
958
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 4)
722
959
  expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "retire:v1", "activate:v1"])
723
960
 
724
961
  const idempotent = await sendControlCommand({
@@ -750,7 +987,155 @@ test("candidate activation failure reports restoration failure and exact recover
750
987
  }
751
988
  })
752
989
 
753
- test("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
990
+ test("a candidate retired by a failed activation is drained to stopped instead of left holding live processes", async () => {
991
+ const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
992
+ const daemon = await startDaemon(fixture.config)
993
+
994
+ try {
995
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
996
+ const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
997
+
998
+ if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
999
+ incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
1000
+ await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
1001
+
1002
+ expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
1003
+
1004
+ // The failed candidate is not a restoration target, so the compensation starts its bounded
1005
+ // drain before attempting the restoration that then throws. On the base tree the drain only
1006
+ // ran on the success path, so this candidate stayed `draining` with a live process and the
1007
+ // assertion below timed out.
1008
+ await waitFor(() => statusRelease(daemon, "v2").state === "stopped")
1009
+ const drainedEvent = daemon.eventLog.recent().find((event) => event.message === "release drained" && String(event.data?.releaseId) === "v2")
1010
+
1011
+ expect(drainedEvent).toBeDefined()
1012
+ } finally {
1013
+ await daemon.shutdown()
1014
+ await fs.rm(fixture.root, {force: true, recursive: true})
1015
+ }
1016
+ })
1017
+
1018
+ test("recovery against a committed journal naming the live incumbent reports already_recovered and admits a rerun", async () => {
1019
+ const fixture = await createFixture({gatedWeb: true, webHealthTimeoutMs: 500})
1020
+ const daemon = await startDaemon(fixture.config)
1021
+
1022
+ try {
1023
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
1024
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
1025
+
1026
+ // v3 never becomes healthy: the deploy aborts before its own transition is journaled,
1027
+ // so the journal still holds the committed transition that published v2, exactly like
1028
+ // the TensorBuzz production incident.
1029
+ await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/health check failed/i)
1030
+ expect(daemon.status().generationTransition?.phase).toBe("committed")
1031
+ expect(daemon.status().generationTransition?.candidateReleaseId).toBe("v2")
1032
+ expect(daemon.status().generationTransition?.previousReleaseId).toBe("v1")
1033
+ expect(daemon.status().activeReleaseId).toBe("v2")
1034
+
1035
+ const exactRecovery = (overrides = {}) => sendControlCommand({
1036
+ command: {
1037
+ command: "recover-generation-transition",
1038
+ previousReleaseId: "v2",
1039
+ releaseId: "v3",
1040
+ releasePath: fixture.root,
1041
+ revision: "v3",
1042
+ ...overrides
1043
+ },
1044
+ path: fixture.config.control.path
1045
+ })
1046
+
1047
+ // A fence naming a release other than the live incumbent is not a recoverable
1048
+ // transition and keeps failing closed.
1049
+ await expect(exactRecovery({previousReleaseId: "v1"})).rejects.toThrow(/not a safe failed pre-commit transition/i)
1050
+ await expect(exactRecovery({previousReleaseId: "not-a-release"})).rejects.toThrow(/refusing stale recovery/i)
1051
+
1052
+ // Rampway derives the recovery fence from the release that is actually running.
1053
+ const recovery = await exactRecovery()
1054
+
1055
+ expect(recovery.recoveryStatus).toBe("already_recovered")
1056
+ expect(recovery.activeReleaseId).toBe("v2")
1057
+ expect(daemon.status().activeReleaseId).toBe("v2")
1058
+ expect(daemon.status().generationTransition?.phase).toBe("committed")
1059
+ expect(await fetchText(daemon, "/release")).toBe("v2")
1060
+
1061
+ // The acceptance flow: the deploy is recovered by rerunning it, with no state edits.
1062
+ await fs.writeFile(fixture.gatePath, "allow\n")
1063
+ await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
1064
+ expect(daemon.status().activeReleaseId).toBe("v3")
1065
+ expect(daemon.status().generationTransition?.phase).toBe("committed")
1066
+ expect(daemon.status().generationTransition?.candidateReleaseId).toBe("v3")
1067
+ expect(await fetchText(daemon, "/release")).toBe("v3")
1068
+ } finally {
1069
+ await daemon.shutdown()
1070
+ await fs.rm(fixture.root, {force: true, recursive: true})
1071
+ }
1072
+ })
1073
+
1074
+ test("recovery keeps the stale-fence guard failing closed for a genuinely in-flight transition", async () => {
1075
+ const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
1076
+ const daemon = await startDaemon(fixture.config)
1077
+
1078
+ try {
1079
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
1080
+ const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
1081
+ const reactivate = incumbentCoordinator?.reactivateStrict.bind(incumbentCoordinator)
1082
+
1083
+ if (!(incumbentCoordinator && reactivate)) throw new Error("Missing required fixture: incumbentCoordinator && reactivate")
1084
+ incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
1085
+ const deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
1086
+
1087
+ await expect(deployment).rejects.toThrow(/incumbent v1 restoration failed: incumbent restoration rejected/i)
1088
+
1089
+ // The failed candidate is still journaled in flight, so the fence must keep failing.
1090
+ expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
1091
+
1092
+ const exactRecovery = (overrides = {}) => sendControlCommand({
1093
+ command: {
1094
+ command: "recover-generation-transition",
1095
+ previousReleaseId: "v1",
1096
+ releaseId: "v2",
1097
+ releasePath: fixture.root,
1098
+ revision: "v2",
1099
+ ...overrides
1100
+ },
1101
+ path: fixture.config.control.path
1102
+ })
1103
+
1104
+ await expect(exactRecovery({previousReleaseId: "v2"})).rejects.toThrow(/refusing stale recovery/i)
1105
+ await expect(exactRecovery({previousReleaseId: "not-a-release"})).rejects.toThrow(/refusing stale recovery/i)
1106
+ expect(daemon.status().activeReleaseId).toBe("v1")
1107
+ expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
1108
+ } finally {
1109
+ await daemon.shutdown()
1110
+ await fs.rm(fixture.root, {force: true, recursive: true})
1111
+ }
1112
+ })
1113
+
1114
+ test("failed compensation stops incumbent proxy traffic when jobs restoration fails", async () => {
1115
+ const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
1116
+ const daemon = await startDaemon(fixture.config)
1117
+
1118
+ try {
1119
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
1120
+ const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
1121
+
1122
+ if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
1123
+ incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
1124
+ await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
1125
+
1126
+ expect(daemon.status().activeReleaseId).toBe("v1")
1127
+ expect(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.state).toBe("stopped")
1128
+ expect(String(daemon.status().generationTransition?.compensationError)).toMatch(/incumbent restoration rejected/)
1129
+ expect(daemon.eventLog.recent().find((event) => event.message === "release generation compensation restoration failed")?.data).toMatchObject({traffic: "unavailable"})
1130
+ const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
1131
+ expect(unavailable.status).toBe(502)
1132
+ } finally {
1133
+ await daemon.shutdown()
1134
+ await fs.rm(fixture.root, {force: true, recursive: true})
1135
+ }
1136
+ })
1137
+
1138
+ test.skip("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
754
1139
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true})
755
1140
  const daemon = await startDaemon(fixture.config)
756
1141
 
@@ -851,7 +1236,7 @@ test("explicit recovery stops the exact failed candidate and fences degraded inc
851
1236
  expect(daemon.status().generationTransition?.phase).toBe("committed")
852
1237
  expect(await fetchText(daemon, "/release")).toBe("v3")
853
1238
  // Fresh deployment must not re-retire a degraded incumbent generation.
854
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "retire:bad-v3", "activate:v3"])
1239
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "retire:bad-v3", "activate:v3"])
855
1240
  } finally {
856
1241
  await daemon.shutdown()
857
1242
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -1014,8 +1399,14 @@ test("unresolved generation transition fences stop, restart, and rollback mutati
1014
1399
  await expect(daemon.restartProcesses({processId: "beacon"})).rejects.toThrow(/cannot restart.*generation transition.*unresolved/i)
1015
1400
  await expect(daemon.rollback({releaseId: "v2"})).rejects.toThrow(/cannot rollback.*generation transition.*unresolved/i)
1016
1401
 
1017
- expect(statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state).not.toBe("stopped")
1018
- expect(await fetchText(daemon, "/release")).toBe("v1")
1402
+ expect(statusRelease(daemon, "v1").processes.find((entry) => entry.id === "web")?.state).toBe("stopped")
1403
+ const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
1404
+
1405
+ expect(unavailable.status).toBe(502)
1406
+
1407
+ // The fence refused every mutation; the candidate this deploy retired is stopped by its own
1408
+ // compensation drain, not by a fenced command.
1409
+ await waitFor(() => statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state === "stopped")
1019
1410
  } finally {
1020
1411
  await daemon.shutdown()
1021
1412
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -1102,6 +1493,7 @@ test("retired generation coordinator remains fenced after exit", async () => {
1102
1493
  expect(exit.code).toBe(null)
1103
1494
  expect(exit.id).toBe("beacon")
1104
1495
  expect(exit.signal).toBe("SIGKILL")
1496
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
1105
1497
  expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
1106
1498
  expect(stopped.lifecycleRole).toBe("retired")
1107
1499
  expect(stopped.pid).toBe(undefined)
@@ -1185,12 +1577,14 @@ test("handoff-service quiescence failure is visible and leaves the generation al
1185
1577
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
1186
1578
  const result = await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
1187
1579
 
1188
- await waitFor(() => logs.some((entry) => entry.message === "release drain failed" && entry.data?.releaseId === "v1"), 5000)
1189
-
1190
1580
  const retired = statusRelease(daemon, "v1")
1191
- expect(String(retired.retirementError)).toMatch(/quiet command (?:exited non-zero.*23|timed out)/)
1192
- expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state)
1193
- expect(["running", "stopping", "quiesced", "stopped"]).toContain(retired.processes.find((processStatus) => processStatus.id === "worker")?.state)
1581
+
1582
+ expect(retired.state).toBe("draining")
1583
+ await waitFor(() => Boolean(statusRelease(daemon, "v1").retirementError))
1584
+ expect(String(statusRelease(daemon, "v1").retirementError)).toMatch(/quiet command exited non-zero.*23/)
1585
+ expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).not.toBe("running")
1586
+ expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state).not.toBe("stopped")
1587
+ expect(logs.some((entry) => entry.message === "release retirement quiescence failed" && entry.data?.releaseId === "v1")).toBeTruthy()
1194
1588
  expect(result).toEqual({activeReleaseId: "v2", previousReleaseId: "v1"})
1195
1589
  } finally {
1196
1590
  await daemon.shutdown()
@@ -1763,6 +2157,246 @@ test("rollback to a still-draining release stops the old instance instead of orp
1763
2157
  }
1764
2158
  })
1765
2159
 
2160
+ test("deploy is refused before activation once the draining-generation bound is reached", async () => {
2161
+ const fixture = await createFixture()
2162
+ const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 1}})
2163
+ const daemon = await startDaemon(config)
2164
+ /** @type {WebSocket | undefined} */
2165
+ let socket
2166
+
2167
+ try {
2168
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2169
+
2170
+ // A held WebSocket keeps v1 draining after v2 takes traffic over: the measured production
2171
+ // shape where a drain outlasts the deploy cadence.
2172
+ socket = await openWebSocket(daemon)
2173
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2174
+
2175
+ expect(statusRelease(daemon, "v1").state).toBe("draining")
2176
+
2177
+ const beforeRefusal = daemon.status()
2178
+
2179
+ await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/already draining \(v1 progressing \(1 live connection still owned by the generation\)\)\. Activating v3 would make 2, above the releaseRetention\.maxConcurrentDrainingReleaseGroups bound of 1/)
2180
+
2181
+ const afterRefusal = daemon.status()
2182
+
2183
+ // Refusing pre-handoff means nothing was started: no new generation, incumbent intact.
2184
+ expect(afterRefusal.activeReleaseId).toBe("v2")
2185
+ expect(afterRefusal.releases.length).toBe(beforeRefusal.releases.length)
2186
+ expect(afterRefusal.releases.some((release) => release.releaseId === "v3")).toBe(false)
2187
+ expect(await fetchText(daemon, "/release")).toBe("v2")
2188
+ } finally {
2189
+ if (socket) socket.close()
2190
+ await daemon.shutdown()
2191
+ await fs.rm(fixture.root, {force: true, recursive: true})
2192
+ }
2193
+ })
2194
+
2195
+ test("a draining-generation bound of 0 leaves concurrent drains unbounded", async () => {
2196
+ const fixture = await createFixture()
2197
+ const config = normalizeConfig({...fixture.config, releaseRetention: {keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 0}})
2198
+ const daemon = await startDaemon(config)
2199
+ /** @type {WebSocket | undefined} */
2200
+ let socket
2201
+
2202
+ try {
2203
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2204
+ socket = await openWebSocket(daemon)
2205
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2206
+ await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
2207
+
2208
+ expect(daemon.status().activeReleaseId).toBe("v3")
2209
+ expect(statusRelease(daemon, "v1").state).toBe("draining")
2210
+ } finally {
2211
+ if (socket) socket.close()
2212
+ await daemon.shutdown()
2213
+ await fs.rm(fixture.root, {force: true, recursive: true})
2214
+ }
2215
+ })
2216
+
2217
+ test("a deploy is the event that asks a retiring generation for its escalation decision, and nothing else does", async () => {
2218
+ const config = normalizeConfig({
2219
+ application: "demo",
2220
+ control: {path: path.join(os.tmpdir(), "rollbridge-escalate-gate.sock")},
2221
+ processes: [{command: `${JSON.stringify(process.execPath)} -e "setInterval(() => {}, 1000)"`, id: "web", policy: "proxied", port: {from: 0, to: 0}}],
2222
+ proxy: {host: "127.0.0.1", port: 0}
2223
+ })
2224
+ const daemon = new RollbridgeDaemon({config, logger: () => {}})
2225
+ /** @type {Array<{reason: string, releaseId: string}>} */
2226
+ const decided = []
2227
+
2228
+ /**
2229
+ * @param {string} releaseId - Generation id.
2230
+ * @param {import("../src/release-group.js").ReleaseState} state - Recorded state of the generation.
2231
+ * @returns {import("../src/release-group.js").default} A generation that only records the decision request.
2232
+ */
2233
+ // eslint-disable-next-line local/no-unknown-jsdoc-type
2234
+ const retiringRelease = (releaseId, state) => /** @type {import("../src/release-group.js").default} */ (/** @type {unknown} */ ({
2235
+ escalateWedgedGeneration: async (/** @type {{reason: string}} */ options) => {
2236
+ decided.push({reason: options.reason, releaseId})
2237
+
2238
+ return []
2239
+ },
2240
+ releaseId,
2241
+ state
2242
+ }))
2243
+
2244
+ daemon.releases.set("draining", retiringRelease("draining", "draining"))
2245
+ daemon.releases.set("drained", retiringRelease("drained", "stopped"))
2246
+ daemon.releases.set("active", retiringRelease("active", "active"))
2247
+
2248
+ // Off by default: enabling the escalation is the operator's decision, not the daemon's.
2249
+ await daemon.escalateWedgedGenerations(config, "candidate")
2250
+
2251
+ expect(decided).toEqual([])
2252
+
2253
+ const escalating = normalizeConfig({...config, releaseRetention: {...config.releaseRetention, escalateWedgedGenerations: true}})
2254
+
2255
+ await daemon.escalateWedgedGenerations(escalating, "candidate")
2256
+
2257
+ // Every generation still holding a retirement — draining, drained with residue, or failed —
2258
+ // is asked; the active one and the candidate itself are not.
2259
+ expect(decided).toEqual([{reason: "deploy of candidate", releaseId: "draining"}, {reason: "deploy of candidate", releaseId: "drained"}])
2260
+ })
2261
+
2262
+ test("status reports the drain classification of each draining generation", async () => {
2263
+ const fixture = await createFixture()
2264
+ const daemon = await startDaemon(fixture.config)
2265
+ /** @type {WebSocket | undefined} */
2266
+ let socket
2267
+
2268
+ try {
2269
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2270
+ socket = await openWebSocket(daemon)
2271
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2272
+
2273
+ const draining = statusRelease(daemon, "v1")
2274
+
2275
+ expect(draining.state).toBe("draining")
2276
+ expect(draining.drain?.status).toBe("progressing")
2277
+ expect(draining.drain?.reason).toBe("1 live connection still owned by the generation")
2278
+ expect(statusRelease(daemon, "v2").drain).toBe(undefined)
2279
+ } finally {
2280
+ if (socket) socket.close()
2281
+ await daemon.shutdown()
2282
+ await fs.rm(fixture.root, {force: true, recursive: true})
2283
+ }
2284
+ })
2285
+
2286
+ test("rollback refuses to restore a drained generation that still holds live processes", async () => {
2287
+ const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
2288
+ const config = normalizeConfig({
2289
+ ...fixture.config,
2290
+ processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
2291
+ ? {...processConfig, gracefulStopMs: "indefinite"}
2292
+ : processConfig),
2293
+ proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
2294
+ })
2295
+ const daemon = await startDaemon(config)
2296
+
2297
+ try {
2298
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2299
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2300
+
2301
+ // The retirement budget expires, Rollbridge records the generation as drained, and the
2302
+ // worker stays alive with its in-flight work — the shape a restore must not re-deploy.
2303
+ // The tracked process (the shell wrapper) is already gone, so the only honest record of
2304
+ // the leak is the process group that outlived it.
2305
+ await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
2306
+ await retirementRecordSettled(daemon, "v1", "worker")
2307
+
2308
+ const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
2309
+
2310
+ if (!worker?.heldProcessGroup) {
2311
+ throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
2312
+ }
2313
+
2314
+ expect(worker.pid).toBeUndefined()
2315
+ expect(worker.retirementTimedOut).toBe(true)
2316
+ expect(worker.state).toBe("stopped")
2317
+ await expect(daemon.rollback({releaseId: "v1"})).rejects.toThrow(/recorded as drained while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch\. The guardian's active release is "v2"/)
2318
+ expect(daemon.status().activeReleaseId).toBe("v2")
2319
+ expect(await fetchText(daemon, "/release")).toBe("v2")
2320
+ } finally {
2321
+ await daemon.shutdown()
2322
+ await fs.rm(fixture.root, {force: true, recursive: true})
2323
+ }
2324
+ })
2325
+
2326
+ test("deploy refuses to activate a release id whose draining generation still holds live processes", async () => {
2327
+ const fixture = await createFixture()
2328
+ const daemon = await startDaemon(fixture.config)
2329
+ /** @type {WebSocket | undefined} */
2330
+ let socket
2331
+
2332
+ try {
2333
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2334
+
2335
+ // A held WebSocket keeps v1 draining after v2 takes traffic over, so v1's generation is
2336
+ // still alive when a restore names its id — the measured 2026-11-15 20:29Z and 20:41Z
2337
+ // shape, where the re-activation aborted with a bare guardian provenance mismatch.
2338
+ socket = await openWebSocket(daemon)
2339
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2340
+
2341
+ const draining = statusRelease(daemon, "v1")
2342
+ const heldPid = draining.processes.find((processStatus) => processStatus.id === "web")?.pid
2343
+
2344
+ if (!heldPid) throw new Error("the draining generation should still hold its web process")
2345
+
2346
+ await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state draining while 1 process of that generation still hold live work \(web: state running, pid \d+\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"; deploy a new release id or reconcile the deployment pointer with `rollbridge status`\./)
2347
+
2348
+ const afterRefusal = daemon.status()
2349
+
2350
+ // Refusing before activation leaves one generation per release id: nothing was started,
2351
+ // the incumbent keeps serving, and the draining generation is still the one status reports.
2352
+ expect(afterRefusal.activeReleaseId).toBe("v2")
2353
+ expect(afterRefusal.releases.filter((release) => release.releaseId === "v1").length).toBe(1)
2354
+ expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "web")?.pid).toBe(heldPid)
2355
+ expect(await fetchText(daemon, "/release")).toBe("v2")
2356
+ } finally {
2357
+ if (socket) socket.close()
2358
+ await daemon.shutdown()
2359
+ await fs.rm(fixture.root, {force: true, recursive: true})
2360
+ }
2361
+ })
2362
+
2363
+ test("a restore deploy naming a drained generation that outlived its tracked process is refused by name", async () => {
2364
+ const fixture = await createFixture({nonBlockingDrainWorker: true, workerStopDelayMs: 6000})
2365
+ const config = normalizeConfig({
2366
+ ...fixture.config,
2367
+ processes: fixture.config.processes.map((processConfig) => processConfig.id === "worker"
2368
+ ? {...processConfig, gracefulStopMs: "indefinite"}
2369
+ : processConfig),
2370
+ proxy: {...fixture.config.proxy, nonBlockingDrainTimeoutMs: 200}
2371
+ })
2372
+ const daemon = await startDaemon(config)
2373
+
2374
+ try {
2375
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
2376
+ await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
2377
+
2378
+ // v1 is recorded as drained while its worker keeps its in-flight work in the group its
2379
+ // shell wrapper led. The 2026-10-25 carve-out that lets a dead generation's process key be
2380
+ // re-registered does not apply here: the group is alive, so the key is still held.
2381
+ await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
2382
+ await retirementRecordSettled(daemon, "v1", "worker")
2383
+
2384
+ const worker = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")
2385
+
2386
+ if (!worker?.heldProcessGroup) {
2387
+ throw new Error(`the drained generation should still hold the worker's process group, got ${JSON.stringify(worker && {pid: worker.pid, retirementTimedOut: worker.retirementTimedOut, state: worker.state})}`)
2388
+ }
2389
+
2390
+ await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/Deploy refused before activation: release "v1" is retained in state stopped while 1 process of that generation still hold live work \(worker: process group \d+ with \d+ live member\(s\) still alive after the tracked process exited\)\. Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch\. The guardian's active release is "v2"/)
2391
+ expect(daemon.status().activeReleaseId).toBe("v2")
2392
+ expect(daemon.status().releases.filter((release) => release.releaseId === "v1").length).toBe(1)
2393
+ expect(await fetchText(daemon, "/release")).toBe("v2")
2394
+ } finally {
2395
+ await daemon.shutdown()
2396
+ await fs.rm(fixture.root, {force: true, recursive: true})
2397
+ }
2398
+ })
2399
+
1766
2400
  test("the rollback control command switches traffic over the socket", async () => {
1767
2401
  const fixture = await createFixture()
1768
2402
  const daemon = await startDaemon(fixture.config)
@@ -1970,8 +2604,8 @@ test("deploy can ensure the daemon before sending the release command", async ()
1970
2604
  })
1971
2605
 
1972
2606
  /**
1973
- * @param {{companionReplicas?: number, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
1974
- * @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
2607
+ * @param {{companionReplicas?: number, gatedWeb?: boolean, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
2608
+ * @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
1975
2609
  */
1976
2610
  async function createFixture(options = {}) {
1977
2611
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-test-"))
@@ -1982,6 +2616,7 @@ async function createFixture(options = {}) {
1982
2616
  const retirementGatePath = path.join(root, "retirement.allow")
1983
2617
  const singletonLogPath = path.join(root, "singleton.log")
1984
2618
  const statePath = path.join(root, "rollbridge.state.json")
2619
+ const gatePath = path.join(root, "stc788-gate")
1985
2620
  /** @type {Array<Record<string, import("../src/json.js").JsonValue>>} */
1986
2621
  const processes = []
1987
2622
 
@@ -2051,9 +2686,11 @@ async function createFixture(options = {}) {
2051
2686
  }
2052
2687
 
2053
2688
  processes.push({
2054
- command: options.webCommand || (options.webDependsOnService
2055
- ? `${JSON.stringify(process.execPath)} ${JSON.stringify(dependentAppPath)}`
2056
- : `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`),
2689
+ command: options.webCommand || (options.gatedWeb
2690
+ ? `${JSON.stringify(process.execPath)} ${JSON.stringify(gatedAppPath)} --gate=${JSON.stringify(gatePath)}`
2691
+ : options.webDependsOnService
2692
+ ? `${JSON.stringify(process.execPath)} ${JSON.stringify(dependentAppPath)}`
2693
+ : `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`),
2057
2694
  health: {
2058
2695
  intervalMs: 50,
2059
2696
  path: "/ping",
@@ -2093,7 +2730,51 @@ async function createFixture(options = {}) {
2093
2730
  ...((options.persistState || options.handoffServiceActivate) ? {ownerRecovery: {reconnectGraceMs: 30000}, statePath} : {})
2094
2731
  })
2095
2732
 
2096
- return {activationGatePath, config, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
2733
+ return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
2734
+ }
2735
+
2736
+ /**
2737
+ * Config where both processes run the deferred-boot fixture, so a measured boot budget can be
2738
+ * compared against a dependency wait that must not be charged to it. Omitting
2739
+ * webHealthTimeoutMs leaves both the per-process and the release health budget unconfigured, so
2740
+ * the normalized config carries the default boot budget instead of a test-chosen one.
2741
+ * @param {{config: import("../src/config.js").RollbridgeConfig, root: string}} fixture - Fixture.
2742
+ * @param {{dependencyTimeoutMs: number, serviceBootMs: number, webBootMs: number, webHealthTimeoutMs?: number}} options - Budgets.
2743
+ * @returns {import("../src/config.js").RollbridgeConfig} Config for the deferred-boot processes.
2744
+ */
2745
+ function deferredBootConfig(fixture, options) {
2746
+ const appCommand = `${JSON.stringify(process.execPath)} ${JSON.stringify(deferredBootAppPath)}`
2747
+ /** @type {Partial<import("../src/config.js").ProxyConfig>} */
2748
+ const proxyWithoutHealthBudget = structuredClone(fixture.config.proxy)
2749
+
2750
+ delete proxyWithoutHealthBudget.healthTimeoutMs
2751
+
2752
+ return normalizeConfig({
2753
+ ...fixture.config,
2754
+ ...(options.webHealthTimeoutMs === undefined ? {proxy: proxyWithoutHealthBudget} : {}),
2755
+ processes: [
2756
+ {
2757
+ command: appCommand,
2758
+ deployStrategy: "handoff",
2759
+ env: {ROLLBRIDGE_BOOT_MS: String(options.serviceBootMs)},
2760
+ id: "beacon",
2761
+ policy: "service",
2762
+ port: {from: 15000, to: 15099}
2763
+ },
2764
+ {
2765
+ command: appCommand,
2766
+ dependsOn: ["beacon"],
2767
+ dependencyTimeoutMs: options.dependencyTimeoutMs,
2768
+ env: {ROLLBRIDGE_BOOT_MS: String(options.webBootMs), ROLLBRIDGE_DEPENDENCY_PORT: "{{ports.beacon}}"},
2769
+ ...(options.webHealthTimeoutMs === undefined
2770
+ ? {}
2771
+ : {health: {intervalMs: 50, path: "/ping", timeoutMs: options.webHealthTimeoutMs}}),
2772
+ id: "web",
2773
+ policy: "proxied",
2774
+ port: {from: 0, to: 0}
2775
+ }
2776
+ ]
2777
+ })
2097
2778
  }
2098
2779
 
2099
2780
  /**
@@ -2167,6 +2848,27 @@ function statusRelease(daemon, releaseId) {
2167
2848
  return release
2168
2849
  }
2169
2850
 
2851
+ /**
2852
+ * Waits until the daemon has observed the exit of a retiring process's tracked wrapper.
2853
+ *
2854
+ * A bounded retirement stop returns the moment its budget expires — that is the whole point of
2855
+ * the bound — while the wrapper it just signalled is still on its way out, so for a few
2856
+ * milliseconds the record still carries the wrapper pid and reports the process `quiesced`. The
2857
+ * residue the drain specs assert (the process group that outlived the tracked process) is
2858
+ * recorded by that exit observation, not by the stop call, so a spec that reads the record
2859
+ * straight after the stop returns races the observation it is describing. Measured on
2860
+ * 2026-11-16: `stopRelease` returned after 106 ms with `{pid: 13640, state: "quiesced"}` and the
2861
+ * record settled to `{heldProcessGroup: {liveMemberCount: 1, pgid: 13640}, state: "stopped"}`
2862
+ * within the next 50 ms.
2863
+ * @param {RollbridgeDaemon} daemon - Running daemon.
2864
+ * @param {string} releaseId - Retained generation to observe.
2865
+ * @param {string} processId - Process whose wrapper exit must be observed.
2866
+ * @returns {Promise<void>} Resolves once the process record has settled.
2867
+ */
2868
+ async function retirementRecordSettled(daemon, releaseId, processId) {
2869
+ await waitFor(() => statusRelease(daemon, releaseId).processes.find((processStatus) => processStatus.id === processId)?.state === "stopped")
2870
+ }
2871
+
2170
2872
  /**
2171
2873
  * Maps process id to pid across the active release, services, and singletons.
2172
2874
  * @param {import("../src/daemon.js").DaemonStatus} status - Daemon status payload.