rollbridge 0.1.69 → 0.1.70

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -183,8 +183,9 @@ handoffs and exits only after the worker pool has drained.
183
183
 
184
184
  For a coordinator that starts quiescent, give the handoff jobs-main paired
185
185
  `lifecycle.activateCommand` and `quietCommand` hooks. After candidate health,
186
- Rollbridge waits for old retirement, waits for candidate activation, and commits
187
- the active proxy target without an awaited boundary. The durable transition is
186
+ Rollbridge activates the candidate, commits the active proxy target, and only
187
+ then starts the old generation's bounded retirement asynchronously. The durable
188
+ transition is
188
189
  generation-scoped and resumable; failures remain visible and block unrelated
189
190
  deploys. Post-commit singleton replacement is also journaled and must complete
190
191
  before an exact retry reports success. If the active coordinator restarts,
@@ -465,14 +466,14 @@ On `rollbridge deploy`, the required ordering is:
465
466
  1. starts any missing persistent service and the candidate's handoff services;
466
467
  2. starts the new release's `companion`s, then its `proxied` process, and
467
468
  health-checks the proxied process;
468
- 3. when `activateCommand` is configured, retires and acknowledges the previous
469
- jobs generation, then activates and acknowledges the candidate;
470
- 4. synchronously switches new traffic to the new release (hook-free configs keep
471
- the existing switch-then-retire behavior);
469
+ 3. when `activateCommand` is configured, activates and acknowledges the
470
+ candidate;
471
+ 4. synchronously switches new traffic to the new release;
472
472
  5. replaces `singleton`s (stops the old one, then starts the new one);
473
- 6. returns success without waiting for the previous generation or its independent
474
- HTTP/WebSocket drain; Rollbridge supervises all retained drains in the
475
- background and reaps each generation only after its handoffs and workers end.
473
+ 6. starts the previous generation's bounded retirement asynchronously after the
474
+ traffic switch; deploy success never waits for quiescence or drain, and
475
+ Rollbridge supervises all retained drains in the background and reaps each
476
+ generation only after its handoffs and workers end.
476
477
 
477
478
  If the new release fails to start or health-check, the previous release stays
478
479
  active and any service started during this deploy is rolled back.
package/TODO.md CHANGED
@@ -39,6 +39,7 @@ This roadmap tracks planned Rollbridge features and documentation. Rollbridge sh
39
39
  - [x] Implement opt-in durable guardian recovery for exact same-authority daemon process replacement without stopping retained generations.
40
40
  - [x] Implement guardian-fenced atomic incompatible owner/config/control-socket/package/runtime replacement without stopping retained generations.
41
41
  - [x] Add the authenticated one-time disruptive bridge for pre-replacement guardians; all protocol-capable replacements remain atomic.
42
+ - [x] Activate and traffic-switch a healthy candidate before asynchronously starting bounded old-generation retirement.
42
43
  - [x] Replicas and stable worker indexes. (Supported on port-less `companion` processes; `proxied`/`singleton`/ported processes stay single.)
43
44
  - [x] Allow one process config to start multiple replicas (`replicas`, companion-only for now).
44
45
  - [x] Expose `ROLLBRIDGE_REPLICA_INDEX`, replica count, and per-replica template context (`{{replicaIndex}}`/`{{replicaCount}}`).
@@ -0,0 +1 @@
1
+ - Activate and switch traffic to a ready replacement generation before quiescing the incumbent. A stalled retirement can no longer stop the active web or jobs generation before its successor is serving traffic; retirement failures remain observable on the retired release after the handoff.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rollbridge",
3
- "version": "0.1.69",
3
+ "version": "0.1.70",
4
4
  "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
5
5
  "keywords": [
6
6
  "deploy",
package/src/daemon.js CHANGED
@@ -506,8 +506,29 @@ export default class RollbridgeDaemon {
506
506
  /**
507
507
  * Prepares from private guardian state, starts listeners, then asks the committed owner
508
508
  * to atomically fence itself and transfer authority.
509
+ * @returns {Promise<void>} Resolves after the replacement is committed.
509
510
  */
510
511
  async replaceIncompatibleOwner() {
512
+ const maxAttempts = 3
513
+ let lastError
514
+
515
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
516
+ try {
517
+ return await this.replaceIncompatibleOwnerAttempt()
518
+ } catch (error) {
519
+ lastError = error
520
+ const diagnostic = error instanceof Error ? error.message : String(error)
521
+
522
+ if (!/^Guardian owner state changed after prepare \(prepared revision \d+, current revision \d+\); abort the stale candidate before preparing again$/.test(diagnostic) || attempt === maxAttempts) {
523
+ throw error
524
+ }
525
+ this.logger("owner replacement candidate became stale; re-preparing", {attempt, maxAttempts, error: diagnostic})
526
+ }
527
+ }
528
+ throw lastError
529
+ }
530
+
531
+ async replaceIncompatibleOwnerAttempt() {
511
532
  if (!this.statePath || !this.config.ownerRecovery) throw new Error("Atomic owner replacement requires ownerRecovery and the same statePath transaction anchor")
512
533
  this.controlCommandsReady = false
513
534
  const state = await readState(this.statePath)
@@ -1908,8 +1929,10 @@ export default class RollbridgeDaemon {
1908
1929
  let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
1909
1930
 
1910
1931
  if (transition.phase === "candidate_ready") {
1911
- if (previousRelease && !transition.degradedIncumbent) await this.updateGenerationTransition("retiring_previous")
1912
- else await this.updateGenerationTransition("previous_retired")
1932
+ // The candidate has already started and passed its health gates. Keep the
1933
+ // incumbent authoritative until the candidate is activated and traffic is
1934
+ // switched; retirement is deliberately launched from committed_pending.
1935
+ await this.updateGenerationTransition("activating_candidate")
1913
1936
  }
1914
1937
 
1915
1938
  if (transition.phase === "retiring_previous") {
@@ -1983,22 +2006,20 @@ export default class RollbridgeDaemon {
1983
2006
 
1984
2007
  if (transition.phase === "committed_pending") {
1985
2008
  await this.refreshServiceDefinitions(release)
1986
- if (!activationLifecycle && previousRelease?.state === "active") {
1987
- try {
1988
- const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
1989
-
1990
- await previousRelease.beginRetirement(retirementConfig)
1991
- } catch (error) {
1992
- retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
1993
- this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
1994
- }
2009
+ if (previousRelease?.state === "active") {
2010
+ const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
2011
+
2012
+ // Traffic is already on the healthy candidate. Retirement must not hold
2013
+ // the deploy mutation or its owner fence; its own bounded drain reports
2014
+ // failures and escalates any process that misses the retirement budget.
2015
+ void previousRelease.beginRetirement(retirementConfig)
2016
+ .catch((error) => {
2017
+ retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
2018
+ this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
2019
+ })
2020
+ .then(() => this.drainAndPrune(previousRelease, retirementConfig))
1995
2021
  }
1996
2022
  await this.checkpointGenerationTransition()
1997
- if (previousRelease) {
1998
- const retirementConfig = previousRelease.config
1999
-
2000
- void this.drainAndPrune(previousRelease, retirementConfig)
2001
- }
2002
2023
  try {
2003
2024
  await this.replaceSingletons(release)
2004
2025
  } catch (error) {
@@ -3129,13 +3150,13 @@ export default class RollbridgeDaemon {
3129
3150
  await captureShutdownError(cleanupErrors, "proxy close", async () => this.proxy.close())
3130
3151
  const dependentStopResults = await Promise.allSettled([
3131
3152
  ...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
3132
- ...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})),
3133
- ...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs}))
3153
+ ...[...this.startingReleases].map((release) => release.stop()),
3154
+ ...[...this.releases.values()].map((release) => release.stop())
3134
3155
  ])
3135
3156
  const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
3136
3157
  const stopResults = [...dependentStopResults, ...serviceStopResults]
3137
3158
  const guardian = this.guardian
3138
- if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs}))
3159
+ if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
3139
3160
  await captureShutdownError(cleanupErrors, "proxy server close", () => this.closeServer(this.proxyServer))
3140
3161
 
3141
3162
  // Wait for any in-flight write first so it can't recreate or overwrite the final state (no
@@ -489,7 +489,7 @@ async function execute(request, socket) {
489
489
 
490
490
  if (request.command === "stage-owner-replacement") {
491
491
  requireReplacement(socket, request)
492
- if (replacementRevision !== ownerRevision) throw new Error("Guardian owner state changed after prepare; abort the stale candidate before preparing again")
492
+ if (replacementRevision !== ownerRevision) throw new Error(`Guardian owner state changed after prepare (prepared revision ${replacementRevision}, current revision ${ownerRevision}); abort the stale candidate before preparing again`)
493
493
  if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
494
494
  throw new Error("Prepared replacement state does not match the requested authority")
495
495
  }
@@ -915,7 +915,8 @@ export default class ReleaseGroup extends EventEmitter {
915
915
  return
916
916
  }
917
917
 
918
- await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop()))
918
+ const retirementTimeoutMs = config.proxy.nonBlockingDrainTimeoutMs
919
+ await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
919
920
  const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
920
921
  nonBlockingResults.forEach((result, index) => {
921
922
  if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
@@ -928,7 +929,7 @@ export default class ReleaseGroup extends EventEmitter {
928
929
  })
929
930
  }
930
931
  })
931
- await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
932
+ await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
932
933
 
933
934
  // Last act of the retirement transition, before the release is recorded stopped: whatever
934
935
  // is still alive here is the residue, and the bracket pair above can now say whether it is
@@ -963,7 +964,14 @@ export default class ReleaseGroup extends EventEmitter {
963
964
  this.drainStartedAt = new Date().toISOString()
964
965
  }
965
966
 
967
+ // Refresh the live process definitions with the current retirement policy,
968
+ // but retain this generation's exact config identity for owner recovery.
969
+ // Replacing the stored generation config here would make a still-running
970
+ // singleton from the previous release fail the guardian provenance fence.
971
+ const generationConfig = this.config
972
+
966
973
  this.refreshProcessDefinitions(config)
974
+ this.config = generationConfig
967
975
  const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
968
976
  const results = await Promise.allSettled([...this.processes.entries()]
969
977
  .filter(([id]) => generationIds.has(id))
@@ -956,7 +956,7 @@ test("replacement staging rejects owner state published after prepare", async ()
956
956
  const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
957
957
 
958
958
  await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v2"}})
959
- await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare/i)
959
+ await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare \(prepared revision 1, current revision 2\)/i)
960
960
  await candidate.abortOwnerReplacement(prepared.replacementId)
961
961
  const fresh = await candidate.prepareOwnerReplacement(authority, nextAuthority)
962
962
 
@@ -174,7 +174,7 @@ test("exact bootstrap restores the committed generation after external owner ret
174
174
  expect(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
175
175
  // candidate activation must precede post-commit singleton completion
176
176
  expect(recoveryOrder).toEqual(["service", "activate", "singleton"])
177
- expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
177
+ expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])
178
178
  } finally {
179
179
  if (recovered) {
180
180
  const activeRecovery = recovered.status().activeReleaseId === "v2"
@@ -310,7 +310,7 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
310
310
  expect(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
311
311
  expect(releaseProcessPid(active, "v1", "worker")).toBe(v1WorkerPid)
312
312
  expect(isProcessRunning(v1WorkerPid)).toBe(true)
313
- expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
313
+ expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])
314
314
  } finally {
315
315
  if (recovered) {
316
316
  const shutdown = recovered.shutdown()
@@ -710,7 +710,7 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
710
710
 
711
711
  expect(recovering.daemonPid).toBe(recoveredDaemonPid)
712
712
  expect(recovering.ownerRecovery?.ready).toBe(true)
713
- expect(recovering.generationTransition?.phase).toBe("retiring_previous")
713
+ expect(recovering.generationTransition?.phase).toBe("committed_pending")
714
714
  expect(typeof guardianPid).toBe("number")
715
715
  await expect(() => sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: fixture.socketPath})).toThrow(/Another owner mutation/)
716
716
  await expect(() => sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})).toThrow(/Cannot shut down while generation transition recovery is in progress/)
@@ -720,7 +720,7 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
720
720
  await fs.writeFile(retirementGatePath, "release retirement\n")
721
721
  await waitForProcessExit(recoveredDaemonPid, 5000)
722
722
  await waitForProcessExit(guardianPid, 5000)
723
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v1", "activate:v2", "retire:v2"])
723
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v2", "retire:v1"])
724
724
  } finally {
725
725
  await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
726
726
  await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
@@ -742,7 +742,7 @@ test("owner recovery preserves a completed activation compensation without repla
742
742
 
743
743
  await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})
744
744
  await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})).toThrow(/activate command exited non-zero/)
745
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
745
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "activate:v1"])
746
746
 
747
747
  owner.kill("SIGKILL")
748
748
  await once(owner, "exit")
@@ -754,7 +754,7 @@ test("owner recovery preserves a completed activation compensation without repla
754
754
  expect(recovered.activeReleaseId).toBe("v1")
755
755
  expect(recovered.generationTransition).toBe(undefined)
756
756
  // owner recovery must not replay completed compensation hooks
757
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
757
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "activate:v1"])
758
758
 
759
759
  const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
760
760
  await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
@@ -854,7 +854,7 @@ test("owner recovery preserves complete private transition authority across a ca
854
854
 
855
855
  expect(recovered.status().activeReleaseId).toBe("v2")
856
856
  expect(recovered.status().generationTransition?.phase).toBe("committed")
857
- expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2"])
857
+ expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "activate:v2", "retire:v1"])).toEqual(["activate:v1", "activate:v2", "retire:v1"])
858
858
  } finally {
859
859
  if (recovered) await recovered.shutdown().catch(() => {})
860
860
  await stopFixtureGuardian(fixture.statePath)
@@ -1104,8 +1104,8 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
1104
1104
  await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed", AbortSignal.timeout(5000))
1105
1105
  await recoverySettled
1106
1106
  // ambiguous retirement must replay exactly once
1107
- expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(2)
1108
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v1", "activate:v2"])
1107
+ expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(1)
1108
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
1109
1109
  expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v2")
1110
1110
 
1111
1111
  const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
@@ -1784,7 +1784,7 @@ test("same-authority owner replacement preserves completed activation compensati
1784
1784
  await waitForLog(owner, "control socket listening")
1785
1785
  await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: oldSocketPath})
1786
1786
  await expect(sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: oldSocketPath})).rejects.toThrow(/activate command exited non-zero/)
1787
- expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v1\nretire:v2\nactivate:v1\n")
1787
+ expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v2\nactivate:v1\n")
1788
1788
 
1789
1789
  await writeConfig(configPath, failedConfig(oldSocketPath, false))
1790
1790
  candidate = spawn(process.execPath, [binPath, "daemon", "--config", configPath, "--replace-owner"], {stdio: ["ignore", "pipe", "pipe"]})
@@ -1793,7 +1793,7 @@ test("same-authority owner replacement preserves completed activation compensati
1793
1793
  expect(status.activeReleaseId).toBe("v1")
1794
1794
  expect(status.generationTransition).toBe(undefined)
1795
1795
  // Replacement must not replay completed compensation hooks.
1796
- expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v1\nretire:v2\nactivate:v1\n")
1796
+ expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v2\nactivate:v1\n")
1797
1797
 
1798
1798
  const shutdown = sendControlCommand({command: {command: "shutdown"}, path: oldSocketPath})
1799
1799
  await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
@@ -1900,7 +1900,7 @@ test("config-changing owner replacement proceeds after activation compensation c
1900
1900
 
1901
1901
  expect(status.activeReleaseId).toBe("v1")
1902
1902
  expect(status.generationTransition).toBe(undefined)
1903
- expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v1\nretire:v2\nactivate:v1\n")
1903
+ expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v2\nactivate:v1\n")
1904
1904
  const shutdown = sendControlCommand({command: {command: "shutdown"}, path: newSocketPath})
1905
1905
 
1906
1906
  await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
@@ -35,7 +35,6 @@ test("a nonBlockingDrain worker stops immediately while its release is still dra
35
35
 
36
36
  try {
37
37
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
38
- await waitForFile(fixture.workerReadyPath)
39
38
 
40
39
  // An open WebSocket keeps v1's connection drain pending after v2 takes over.
41
40
  socket = await openWebSocket(daemon)
@@ -80,7 +79,6 @@ test("an explicit stop returns within its retirement budget instead of waiting o
80
79
  try {
81
80
  await daemon.start()
82
81
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
83
- await waitForFile(fixture.workerReadyPath)
84
82
 
85
83
  workerPid = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.pid
86
84
 
@@ -104,8 +102,9 @@ test("an explicit stop returns within its retirement budget instead of waiting o
104
102
 
105
103
  expect(stopped.state).toBe("stopped")
106
104
  expect(stoppedWorker?.retirementTimedOut).toBe(true)
107
- // The bounded stop records the retirement timeout; the shutdown regression separately proves
108
- // that a non-blocking worker can remain alive after the acknowledgement.
105
+ // The tracked record is the shell wrapper, which is already gone; the worker it launched is
106
+ // still draining inside the same process group, and that is the residue the stop must report.
107
+ expect(stoppedWorker?.heldProcessGroup?.liveMemberCount).toBeGreaterThan(0)
109
108
  expect(logs.filter((entry) => entry.message === "release stop gave up waiting for a retiring process")).toMatchObject([
110
109
  {data: {processId: "worker", releaseId: "v1", timeoutMs: 100}}
111
110
  ])
@@ -708,7 +707,7 @@ test("candidate activation retires jobs-main with its workers without waiting fo
708
707
  }
709
708
  })
710
709
 
711
- test("opt-in generation lifecycle acknowledges old retirement before activating the candidate", async () => {
710
+ test("opt-in generation lifecycle activates the candidate before retiring the old generation", async () => {
712
711
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, nonBlockingDrainWorker: true, webDependsOnService: true})
713
712
  const daemon = await startDaemon(fixture.config)
714
713
 
@@ -719,7 +718,8 @@ test("opt-in generation lifecycle acknowledges old retirement before activating
719
718
 
720
719
  await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
721
720
 
722
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
721
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
722
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
723
723
  expect(daemon.status().activeReleaseId).toBe("v2")
724
724
  expect(daemon.status().generationTransition?.phase).toBe("committed")
725
725
  } finally {
@@ -813,7 +813,8 @@ test("exact committed retry finishes pending singleton replacement before succes
813
813
 
814
814
  expect(daemon.status().generationTransition?.phase).toBe("committed")
815
815
  expect(daemon.status().singletons[0]?.process.state).toBe("running")
816
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
816
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
817
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
817
818
  } finally {
818
819
  await daemon.shutdown()
819
820
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -837,27 +838,21 @@ test("first generation is not committed when its activation acknowledgement fail
837
838
  }
838
839
  })
839
840
 
840
- test("retirement acknowledgement failure retains the exact transition, blocks other deploys, and exact resume continues it", async () => {
841
+ test("retirement acknowledgement failure is reported after traffic switches", async () => {
841
842
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceQuietFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
842
843
  const daemon = await startDaemon(fixture.config)
843
844
 
844
845
  try {
845
846
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
846
- await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/retirement quiescence failed/)
847
+ const result = await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
848
+ await waitFor(() => Boolean(statusRelease(daemon, "v1").retirementError))
847
849
 
848
850
  const failed = daemon.status()
849
851
 
850
- expect(failed.activeReleaseId).toBe("v1")
851
- expect(failed.generationTransition?.phase).toBe("retiring_previous")
852
- expect(String(failed.generationTransition?.error)).toMatch(/quiet command exited non-zero/)
853
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1"])
854
- await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
855
-
856
- await fs.writeFile(fixture.retirementGatePath, "allow\n")
857
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
858
-
859
- expect(daemon.status().activeReleaseId).toBe("v2")
860
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
852
+ expect(failed.activeReleaseId).toBe("v2")
853
+ expect(failed.generationTransition?.phase).toBe("committed")
854
+ expect(String(statusRelease(daemon, "v1").retirementError)).toMatch(/quiet command exited non-zero/)
855
+ expect(result.activeReleaseId).toBe("v2")
861
856
  } finally {
862
857
  await daemon.shutdown()
863
858
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -894,7 +889,7 @@ test("candidate activation failure reports restoration failure and exact recover
894
889
  expect(String(activationEvent?.data.error)).toMatch(/activate command exited non-zero/)
895
890
  expect(String(restorationEvent?.data.activationError)).toMatch(/activate command exited non-zero/)
896
891
  expect(String(restorationEvent?.data.error)).toMatch(/incumbent restoration rejected/)
897
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2"])
892
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2"])
898
893
  await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
899
894
  const failedCandidate = daemon.releases.get("v2")
900
895
 
@@ -923,7 +918,8 @@ test("candidate activation failure reports restoration failure and exact recover
923
918
  const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
924
919
 
925
920
  expect(persisted?.generationTransition).toBe(undefined)
926
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
921
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 4)
922
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "retire:v1", "activate:v1"])
927
923
 
928
924
  const idempotent = await sendControlCommand({
929
925
  command: {
@@ -1224,7 +1220,7 @@ test("candidate activation failure compensates to the incumbent and admits a dif
1224
1220
  expect(compensated.generationTransition).toBe(undefined)
1225
1221
  expect(await fetchText(daemon, "/release")).toBe("v1")
1226
1222
  expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("running")
1227
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
1223
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "activate:v1"])
1228
1224
 
1229
1225
  await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
1230
1226
 
@@ -1246,7 +1242,6 @@ test("ambiguous candidate activation retires the candidate before reactivating t
1246
1242
 
1247
1243
  expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual([
1248
1244
  "activate:v1",
1249
- "retire:v1",
1250
1245
  "activate:v2",
1251
1246
  "retire:v2",
1252
1247
  "activate:v1"
@@ -1300,7 +1295,7 @@ test("candidate activation recovery keeps the fence when a worker-specific resum
1300
1295
  expect(status.generationTransition?.phase).toBe("restoring_previous")
1301
1296
  expect(String(status.generationTransition?.activationError)).toMatch(/activate command exited non-zero/)
1302
1297
  expect(String(status.generationTransition?.compensationError)).toMatch(/reactivate command exited non-zero/)
1303
- expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("quiesced")
1298
+ expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("running")
1304
1299
  expect(String(restorationEvent?.data.activationError)).toMatch(/activate command exited non-zero/)
1305
1300
  expect(String(restorationEvent?.data.error)).toMatch(/reactivate command exited non-zero/)
1306
1301
  await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
@@ -1461,7 +1456,8 @@ test("retired generation coordinator remains fenced after exit", async () => {
1461
1456
  expect(exit.code).toBe(null)
1462
1457
  expect(exit.id).toBe("beacon")
1463
1458
  expect(exit.signal).toBe("SIGKILL")
1464
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
1459
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
1460
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
1465
1461
  expect(stopped.lifecycleRole).toBe("retired")
1466
1462
  expect(stopped.pid).toBe(undefined)
1467
1463
  expect(stopped.restarts).toBe(0)
@@ -1547,11 +1543,12 @@ test("handoff-service quiescence failure is visible and leaves the generation al
1547
1543
  const retired = statusRelease(daemon, "v1")
1548
1544
 
1549
1545
  expect(retired.state).toBe("draining")
1550
- expect(String(retired.retirementError)).toMatch(/quiet command exited non-zero.*23/)
1551
- expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).toBe("stopping")
1546
+ await waitFor(() => Boolean(statusRelease(daemon, "v1").retirementError))
1547
+ expect(String(statusRelease(daemon, "v1").retirementError)).toMatch(/quiet command exited non-zero.*23/)
1548
+ expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).not.toBe("running")
1552
1549
  expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state).not.toBe("stopped")
1553
1550
  expect(logs.some((entry) => entry.message === "release retirement quiescence failed" && entry.data?.releaseId === "v1")).toBeTruthy()
1554
- expect(result.retirement).toEqual({error: retired.retirementError, releaseId: "v1", status: "quiescence_failed"})
1551
+ expect(result).toEqual({activeReleaseId: "v2", previousReleaseId: "v1"})
1555
1552
  } finally {
1556
1553
  await daemon.shutdown()
1557
1554
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -2571,14 +2568,13 @@ test("deploy can ensure the daemon before sending the release command", async ()
2571
2568
 
2572
2569
  /**
2573
2570
  * @param {{companionReplicas?: number, gatedWeb?: boolean, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
2574
- * @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string, workerReadyPath: string}>} Fixture data.
2571
+ * @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
2575
2572
  */
2576
2573
  async function createFixture(options = {}) {
2577
2574
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-test-"))
2578
2575
  const serviceLogPath = path.join(root, "service.log")
2579
2576
  const serviceQuietPath = path.join(root, "service.quiet")
2580
2577
  const lifecycleLogPath = path.join(root, "service.lifecycle")
2581
- const workerReadyPath = path.join(root, "worker.ready")
2582
2578
  const activationGatePath = path.join(root, "activation.allow")
2583
2579
  const retirementGatePath = path.join(root, "retirement.allow")
2584
2580
  const singletonLogPath = path.join(root, "singleton.log")
@@ -2629,7 +2625,7 @@ async function createFixture(options = {}) {
2629
2625
 
2630
2626
  if (options.nonBlockingDrainWorker) {
2631
2627
  processes.push({
2632
- command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0})); require("node:fs").writeFileSync(${JSON.stringify(workerReadyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
2628
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0})); setInterval(() => {}, 1000)`)}`,
2633
2629
  id: "worker",
2634
2630
  ...(options.workerReactivationLifecycle ? {lifecycle: {
2635
2631
  quietCommand: `printf 'worker-retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`,
@@ -2697,7 +2693,7 @@ async function createFixture(options = {}) {
2697
2693
  ...((options.persistState || options.handoffServiceActivate) ? {ownerRecovery: {reconnectGraceMs: 30000}, statePath} : {})
2698
2694
  })
2699
2695
 
2700
- return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath, workerReadyPath}
2696
+ return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
2701
2697
  }
2702
2698
 
2703
2699
  /**
@@ -2908,21 +2904,6 @@ async function waitFor(callback, timeoutMs = 3000) {
2908
2904
  throw new Error("Timed out waiting for condition")
2909
2905
  }
2910
2906
 
2911
- /**
2912
- * @param {string} filePath - File to await.
2913
- * @returns {Promise<void>} Resolves once the file exists.
2914
- */
2915
- async function waitForFile(filePath) {
2916
- await waitFor(async () => {
2917
- try {
2918
- await fs.access(filePath)
2919
- return true
2920
- } catch {
2921
- return false
2922
- }
2923
- })
2924
- }
2925
-
2926
2907
  /**
2927
2908
  * Resolves from retained output or the exact event that records it, and rejects if the
2928
2909
  * supervised process exits first.