rollbridge 0.1.69 → 0.1.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -9
- package/TODO.md +1 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/package.json +1 -1
- package/src/daemon.js +40 -19
- package/src/process-guardian.js +1 -1
- package/src/release-group.js +10 -2
- package/test/guardian-client.test.js +1 -1
- package/test/owner-recovery.test.js +9 -9
- package/test/owner-replacement.test.js +3 -3
- package/test/rollbridge.test.js +29 -48
package/README.md
CHANGED
|
@@ -183,8 +183,9 @@ handoffs and exits only after the worker pool has drained.
|
|
|
183
183
|
|
|
184
184
|
For a coordinator that starts quiescent, give the handoff jobs-main paired
|
|
185
185
|
`lifecycle.activateCommand` and `quietCommand` hooks. After candidate health,
|
|
186
|
-
Rollbridge
|
|
187
|
-
|
|
186
|
+
Rollbridge activates the candidate, commits the active proxy target, and only
|
|
187
|
+
then starts the old generation's bounded retirement asynchronously. The durable
|
|
188
|
+
transition is
|
|
188
189
|
generation-scoped and resumable; failures remain visible and block unrelated
|
|
189
190
|
deploys. Post-commit singleton replacement is also journaled and must complete
|
|
190
191
|
before an exact retry reports success. If the active coordinator restarts,
|
|
@@ -465,14 +466,14 @@ On `rollbridge deploy`, the required ordering is:
|
|
|
465
466
|
1. starts any missing persistent service and the candidate's handoff services;
|
|
466
467
|
2. starts the new release's `companion`s, then its `proxied` process, and
|
|
467
468
|
health-checks the proxied process;
|
|
468
|
-
3. when `activateCommand` is configured,
|
|
469
|
-
|
|
470
|
-
4. synchronously switches new traffic to the new release
|
|
471
|
-
the existing switch-then-retire behavior);
|
|
469
|
+
3. when `activateCommand` is configured, activates and acknowledges the
|
|
470
|
+
candidate;
|
|
471
|
+
4. synchronously switches new traffic to the new release;
|
|
472
472
|
5. replaces `singleton`s (stops the old one, then starts the new one);
|
|
473
|
-
6.
|
|
474
|
-
|
|
475
|
-
|
|
473
|
+
6. starts the previous generation's bounded retirement asynchronously after the
|
|
474
|
+
traffic switch; deploy success never waits for quiescence or drain, and
|
|
475
|
+
Rollbridge supervises all retained drains in the background and reaps each
|
|
476
|
+
generation only after its handoffs and workers end.
|
|
476
477
|
|
|
477
478
|
If the new release fails to start or health-check, the previous release stays
|
|
478
479
|
active and any service started during this deploy is rolled back.
|
package/TODO.md
CHANGED
|
@@ -39,6 +39,7 @@ This roadmap tracks planned Rollbridge features and documentation. Rollbridge sh
|
|
|
39
39
|
- [x] Implement opt-in durable guardian recovery for exact same-authority daemon process replacement without stopping retained generations.
|
|
40
40
|
- [x] Implement guardian-fenced atomic incompatible owner/config/control-socket/package/runtime replacement without stopping retained generations.
|
|
41
41
|
- [x] Add the authenticated one-time disruptive bridge for pre-replacement guardians; all protocol-capable replacements remain atomic.
|
|
42
|
+
- [x] Activate and traffic-switch a healthy candidate before asynchronously starting bounded old-generation retirement.
|
|
42
43
|
- [x] Replicas and stable worker indexes. (Supported on port-less `companion` processes; `proxied`/`singleton`/ported processes stay single.)
|
|
43
44
|
- [x] Allow one process config to start multiple replicas (`replicas`, companion-only for now).
|
|
44
45
|
- [x] Expose `ROLLBRIDGE_REPLICA_INDEX`, replica count, and per-replica template context (`{{replicaIndex}}`/`{{replicaCount}}`).
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
- Activate and switch traffic to a ready replacement generation before quiescing the incumbent. A stalled retirement can no longer stop the active web or jobs generation before its successor is serving traffic; retirement failures remain observable on the retired release after the handoff.
|
package/package.json
CHANGED
package/src/daemon.js
CHANGED
|
@@ -506,8 +506,29 @@ export default class RollbridgeDaemon {
|
|
|
506
506
|
/**
|
|
507
507
|
* Prepares from private guardian state, starts listeners, then asks the committed owner
|
|
508
508
|
* to atomically fence itself and transfer authority.
|
|
509
|
+
* @returns {Promise<void>} Resolves after the replacement is committed.
|
|
509
510
|
*/
|
|
510
511
|
async replaceIncompatibleOwner() {
|
|
512
|
+
const maxAttempts = 3
|
|
513
|
+
let lastError
|
|
514
|
+
|
|
515
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
516
|
+
try {
|
|
517
|
+
return await this.replaceIncompatibleOwnerAttempt()
|
|
518
|
+
} catch (error) {
|
|
519
|
+
lastError = error
|
|
520
|
+
const diagnostic = error instanceof Error ? error.message : String(error)
|
|
521
|
+
|
|
522
|
+
if (!/^Guardian owner state changed after prepare \(prepared revision \d+, current revision \d+\); abort the stale candidate before preparing again$/.test(diagnostic) || attempt === maxAttempts) {
|
|
523
|
+
throw error
|
|
524
|
+
}
|
|
525
|
+
this.logger("owner replacement candidate became stale; re-preparing", {attempt, maxAttempts, error: diagnostic})
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
throw lastError
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
async replaceIncompatibleOwnerAttempt() {
|
|
511
532
|
if (!this.statePath || !this.config.ownerRecovery) throw new Error("Atomic owner replacement requires ownerRecovery and the same statePath transaction anchor")
|
|
512
533
|
this.controlCommandsReady = false
|
|
513
534
|
const state = await readState(this.statePath)
|
|
@@ -1908,8 +1929,10 @@ export default class RollbridgeDaemon {
|
|
|
1908
1929
|
let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
|
|
1909
1930
|
|
|
1910
1931
|
if (transition.phase === "candidate_ready") {
|
|
1911
|
-
|
|
1912
|
-
|
|
1932
|
+
// The candidate has already started and passed its health gates. Keep the
|
|
1933
|
+
// incumbent authoritative until the candidate is activated and traffic is
|
|
1934
|
+
// switched; retirement is deliberately launched from committed_pending.
|
|
1935
|
+
await this.updateGenerationTransition("activating_candidate")
|
|
1913
1936
|
}
|
|
1914
1937
|
|
|
1915
1938
|
if (transition.phase === "retiring_previous") {
|
|
@@ -1983,22 +2006,20 @@ export default class RollbridgeDaemon {
|
|
|
1983
2006
|
|
|
1984
2007
|
if (transition.phase === "committed_pending") {
|
|
1985
2008
|
await this.refreshServiceDefinitions(release)
|
|
1986
|
-
if (
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
2009
|
+
if (previousRelease?.state === "active") {
|
|
2010
|
+
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
2011
|
+
|
|
2012
|
+
// Traffic is already on the healthy candidate. Retirement must not hold
|
|
2013
|
+
// the deploy mutation or its owner fence; its own bounded drain reports
|
|
2014
|
+
// failures and escalates any process that misses the retirement budget.
|
|
2015
|
+
void previousRelease.beginRetirement(retirementConfig)
|
|
2016
|
+
.catch((error) => {
|
|
2017
|
+
retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
|
|
2018
|
+
this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
|
|
2019
|
+
})
|
|
2020
|
+
.then(() => this.drainAndPrune(previousRelease, retirementConfig))
|
|
1995
2021
|
}
|
|
1996
2022
|
await this.checkpointGenerationTransition()
|
|
1997
|
-
if (previousRelease) {
|
|
1998
|
-
const retirementConfig = previousRelease.config
|
|
1999
|
-
|
|
2000
|
-
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
2001
|
-
}
|
|
2002
2023
|
try {
|
|
2003
2024
|
await this.replaceSingletons(release)
|
|
2004
2025
|
} catch (error) {
|
|
@@ -3129,13 +3150,13 @@ export default class RollbridgeDaemon {
|
|
|
3129
3150
|
await captureShutdownError(cleanupErrors, "proxy close", async () => this.proxy.close())
|
|
3130
3151
|
const dependentStopResults = await Promise.allSettled([
|
|
3131
3152
|
...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
|
|
3132
|
-
...[...this.startingReleases].map((release) => release.stop(
|
|
3133
|
-
...[...this.releases.values()].map((release) => release.stop(
|
|
3153
|
+
...[...this.startingReleases].map((release) => release.stop()),
|
|
3154
|
+
...[...this.releases.values()].map((release) => release.stop())
|
|
3134
3155
|
])
|
|
3135
3156
|
const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
|
|
3136
3157
|
const stopResults = [...dependentStopResults, ...serviceStopResults]
|
|
3137
3158
|
const guardian = this.guardian
|
|
3138
|
-
if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown(
|
|
3159
|
+
if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
|
|
3139
3160
|
await captureShutdownError(cleanupErrors, "proxy server close", () => this.closeServer(this.proxyServer))
|
|
3140
3161
|
|
|
3141
3162
|
// Wait for any in-flight write first so it can't recreate or overwrite the final state (no
|
package/src/process-guardian.js
CHANGED
|
@@ -489,7 +489,7 @@ async function execute(request, socket) {
|
|
|
489
489
|
|
|
490
490
|
if (request.command === "stage-owner-replacement") {
|
|
491
491
|
requireReplacement(socket, request)
|
|
492
|
-
if (replacementRevision !== ownerRevision) throw new Error(
|
|
492
|
+
if (replacementRevision !== ownerRevision) throw new Error(`Guardian owner state changed after prepare (prepared revision ${replacementRevision}, current revision ${ownerRevision}); abort the stale candidate before preparing again`)
|
|
493
493
|
if (request.ownerState === undefined || !isDeepStrictEqual(ownerAuthority(request.ownerState), replacementAuthority)) {
|
|
494
494
|
throw new Error("Prepared replacement state does not match the requested authority")
|
|
495
495
|
}
|
package/src/release-group.js
CHANGED
|
@@ -915,7 +915,8 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
915
915
|
return
|
|
916
916
|
}
|
|
917
917
|
|
|
918
|
-
|
|
918
|
+
const retirementTimeoutMs = config.proxy.nonBlockingDrainTimeoutMs
|
|
919
|
+
await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
|
|
919
920
|
const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
|
|
920
921
|
nonBlockingResults.forEach((result, index) => {
|
|
921
922
|
if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
|
|
@@ -928,7 +929,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
928
929
|
})
|
|
929
930
|
}
|
|
930
931
|
})
|
|
931
|
-
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
|
|
932
|
+
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
|
|
932
933
|
|
|
933
934
|
// Last act of the retirement transition, before the release is recorded stopped: whatever
|
|
934
935
|
// is still alive here is the residue, and the bracket pair above can now say whether it is
|
|
@@ -963,7 +964,14 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
963
964
|
this.drainStartedAt = new Date().toISOString()
|
|
964
965
|
}
|
|
965
966
|
|
|
967
|
+
// Refresh the live process definitions with the current retirement policy,
|
|
968
|
+
// but retain this generation's exact config identity for owner recovery.
|
|
969
|
+
// Replacing the stored generation config here would make a still-running
|
|
970
|
+
// singleton from the previous release fail the guardian provenance fence.
|
|
971
|
+
const generationConfig = this.config
|
|
972
|
+
|
|
966
973
|
this.refreshProcessDefinitions(config)
|
|
974
|
+
this.config = generationConfig
|
|
967
975
|
const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
|
|
968
976
|
const results = await Promise.allSettled([...this.processes.entries()]
|
|
969
977
|
.filter(([id]) => generationIds.has(id))
|
|
@@ -956,7 +956,7 @@ test("replacement staging rejects owner state published after prepare", async ()
|
|
|
956
956
|
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
957
957
|
|
|
958
958
|
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v2"}})
|
|
959
|
-
await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare/i)
|
|
959
|
+
await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare \(prepared revision 1, current revision 2\)/i)
|
|
960
960
|
await candidate.abortOwnerReplacement(prepared.replacementId)
|
|
961
961
|
const fresh = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
962
962
|
|
|
@@ -174,7 +174,7 @@ test("exact bootstrap restores the committed generation after external owner ret
|
|
|
174
174
|
expect(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
|
|
175
175
|
// candidate activation must precede post-commit singleton completion
|
|
176
176
|
expect(recoveryOrder).toEqual(["service", "activate", "singleton"])
|
|
177
|
-
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "
|
|
177
|
+
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])
|
|
178
178
|
} finally {
|
|
179
179
|
if (recovered) {
|
|
180
180
|
const activeRecovery = recovered.status().activeReleaseId === "v2"
|
|
@@ -310,7 +310,7 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
|
|
|
310
310
|
expect(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
|
|
311
311
|
expect(releaseProcessPid(active, "v1", "worker")).toBe(v1WorkerPid)
|
|
312
312
|
expect(isProcessRunning(v1WorkerPid)).toBe(true)
|
|
313
|
-
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "
|
|
313
|
+
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])
|
|
314
314
|
} finally {
|
|
315
315
|
if (recovered) {
|
|
316
316
|
const shutdown = recovered.shutdown()
|
|
@@ -710,7 +710,7 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
|
|
|
710
710
|
|
|
711
711
|
expect(recovering.daemonPid).toBe(recoveredDaemonPid)
|
|
712
712
|
expect(recovering.ownerRecovery?.ready).toBe(true)
|
|
713
|
-
expect(recovering.generationTransition?.phase).toBe("
|
|
713
|
+
expect(recovering.generationTransition?.phase).toBe("committed_pending")
|
|
714
714
|
expect(typeof guardianPid).toBe("number")
|
|
715
715
|
await expect(() => sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: fixture.socketPath})).toThrow(/Another owner mutation/)
|
|
716
716
|
await expect(() => sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})).toThrow(/Cannot shut down while generation transition recovery is in progress/)
|
|
@@ -720,7 +720,7 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
|
|
|
720
720
|
await fs.writeFile(retirementGatePath, "release retirement\n")
|
|
721
721
|
await waitForProcessExit(recoveredDaemonPid, 5000)
|
|
722
722
|
await waitForProcessExit(guardianPid, 5000)
|
|
723
|
-
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "
|
|
723
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v2", "retire:v1"])
|
|
724
724
|
} finally {
|
|
725
725
|
await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
|
|
726
726
|
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
|
|
@@ -742,7 +742,7 @@ test("owner recovery preserves a completed activation compensation without repla
|
|
|
742
742
|
|
|
743
743
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})
|
|
744
744
|
await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})).toThrow(/activate command exited non-zero/)
|
|
745
|
-
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:
|
|
745
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "activate:v1"])
|
|
746
746
|
|
|
747
747
|
owner.kill("SIGKILL")
|
|
748
748
|
await once(owner, "exit")
|
|
@@ -754,7 +754,7 @@ test("owner recovery preserves a completed activation compensation without repla
|
|
|
754
754
|
expect(recovered.activeReleaseId).toBe("v1")
|
|
755
755
|
expect(recovered.generationTransition).toBe(undefined)
|
|
756
756
|
// owner recovery must not replay completed compensation hooks
|
|
757
|
-
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:
|
|
757
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "activate:v1"])
|
|
758
758
|
|
|
759
759
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
760
760
|
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
|
|
@@ -854,7 +854,7 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
854
854
|
|
|
855
855
|
expect(recovered.status().activeReleaseId).toBe("v2")
|
|
856
856
|
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
857
|
-
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "
|
|
857
|
+
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "activate:v2", "retire:v1"])).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
858
858
|
} finally {
|
|
859
859
|
if (recovered) await recovered.shutdown().catch(() => {})
|
|
860
860
|
await stopFixtureGuardian(fixture.statePath)
|
|
@@ -1104,8 +1104,8 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
|
|
|
1104
1104
|
await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed", AbortSignal.timeout(5000))
|
|
1105
1105
|
await recoverySettled
|
|
1106
1106
|
// ambiguous retirement must replay exactly once
|
|
1107
|
-
expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(
|
|
1108
|
-
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "
|
|
1107
|
+
expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(1)
|
|
1108
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
1109
1109
|
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v2")
|
|
1110
1110
|
|
|
1111
1111
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
@@ -1784,7 +1784,7 @@ test("same-authority owner replacement preserves completed activation compensati
|
|
|
1784
1784
|
await waitForLog(owner, "control socket listening")
|
|
1785
1785
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: oldSocketPath})
|
|
1786
1786
|
await expect(sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: oldSocketPath})).rejects.toThrow(/activate command exited non-zero/)
|
|
1787
|
-
expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:
|
|
1787
|
+
expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v2\nactivate:v1\n")
|
|
1788
1788
|
|
|
1789
1789
|
await writeConfig(configPath, failedConfig(oldSocketPath, false))
|
|
1790
1790
|
candidate = spawn(process.execPath, [binPath, "daemon", "--config", configPath, "--replace-owner"], {stdio: ["ignore", "pipe", "pipe"]})
|
|
@@ -1793,7 +1793,7 @@ test("same-authority owner replacement preserves completed activation compensati
|
|
|
1793
1793
|
expect(status.activeReleaseId).toBe("v1")
|
|
1794
1794
|
expect(status.generationTransition).toBe(undefined)
|
|
1795
1795
|
// Replacement must not replay completed compensation hooks.
|
|
1796
|
-
expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:
|
|
1796
|
+
expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v2\nactivate:v1\n")
|
|
1797
1797
|
|
|
1798
1798
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: oldSocketPath})
|
|
1799
1799
|
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
|
|
@@ -1900,7 +1900,7 @@ test("config-changing owner replacement proceeds after activation compensation c
|
|
|
1900
1900
|
|
|
1901
1901
|
expect(status.activeReleaseId).toBe("v1")
|
|
1902
1902
|
expect(status.generationTransition).toBe(undefined)
|
|
1903
|
-
expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:
|
|
1903
|
+
expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v2\nactivate:v1\n")
|
|
1904
1904
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: newSocketPath})
|
|
1905
1905
|
|
|
1906
1906
|
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
|
package/test/rollbridge.test.js
CHANGED
|
@@ -35,7 +35,6 @@ test("a nonBlockingDrain worker stops immediately while its release is still dra
|
|
|
35
35
|
|
|
36
36
|
try {
|
|
37
37
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
38
|
-
await waitForFile(fixture.workerReadyPath)
|
|
39
38
|
|
|
40
39
|
// An open WebSocket keeps v1's connection drain pending after v2 takes over.
|
|
41
40
|
socket = await openWebSocket(daemon)
|
|
@@ -80,7 +79,6 @@ test("an explicit stop returns within its retirement budget instead of waiting o
|
|
|
80
79
|
try {
|
|
81
80
|
await daemon.start()
|
|
82
81
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
83
|
-
await waitForFile(fixture.workerReadyPath)
|
|
84
82
|
|
|
85
83
|
workerPid = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
86
84
|
|
|
@@ -104,8 +102,9 @@ test("an explicit stop returns within its retirement budget instead of waiting o
|
|
|
104
102
|
|
|
105
103
|
expect(stopped.state).toBe("stopped")
|
|
106
104
|
expect(stoppedWorker?.retirementTimedOut).toBe(true)
|
|
107
|
-
// The
|
|
108
|
-
//
|
|
105
|
+
// The tracked record is the shell wrapper, which is already gone; the worker it launched is
|
|
106
|
+
// still draining inside the same process group, and that is the residue the stop must report.
|
|
107
|
+
expect(stoppedWorker?.heldProcessGroup?.liveMemberCount).toBeGreaterThan(0)
|
|
109
108
|
expect(logs.filter((entry) => entry.message === "release stop gave up waiting for a retiring process")).toMatchObject([
|
|
110
109
|
{data: {processId: "worker", releaseId: "v1", timeoutMs: 100}}
|
|
111
110
|
])
|
|
@@ -708,7 +707,7 @@ test("candidate activation retires jobs-main with its workers without waiting fo
|
|
|
708
707
|
}
|
|
709
708
|
})
|
|
710
709
|
|
|
711
|
-
test("opt-in generation lifecycle
|
|
710
|
+
test("opt-in generation lifecycle activates the candidate before retiring the old generation", async () => {
|
|
712
711
|
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
713
712
|
const daemon = await startDaemon(fixture.config)
|
|
714
713
|
|
|
@@ -719,7 +718,8 @@ test("opt-in generation lifecycle acknowledges old retirement before activating
|
|
|
719
718
|
|
|
720
719
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
721
720
|
|
|
722
|
-
|
|
721
|
+
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
|
|
722
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
723
723
|
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
724
724
|
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
725
725
|
} finally {
|
|
@@ -813,7 +813,8 @@ test("exact committed retry finishes pending singleton replacement before succes
|
|
|
813
813
|
|
|
814
814
|
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
815
815
|
expect(daemon.status().singletons[0]?.process.state).toBe("running")
|
|
816
|
-
|
|
816
|
+
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
|
|
817
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
817
818
|
} finally {
|
|
818
819
|
await daemon.shutdown()
|
|
819
820
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -837,27 +838,21 @@ test("first generation is not committed when its activation acknowledgement fail
|
|
|
837
838
|
}
|
|
838
839
|
})
|
|
839
840
|
|
|
840
|
-
test("retirement acknowledgement failure
|
|
841
|
+
test("retirement acknowledgement failure is reported after traffic switches", async () => {
|
|
841
842
|
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceQuietFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
842
843
|
const daemon = await startDaemon(fixture.config)
|
|
843
844
|
|
|
844
845
|
try {
|
|
845
846
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
846
|
-
await
|
|
847
|
+
const result = await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
848
|
+
await waitFor(() => Boolean(statusRelease(daemon, "v1").retirementError))
|
|
847
849
|
|
|
848
850
|
const failed = daemon.status()
|
|
849
851
|
|
|
850
|
-
expect(failed.activeReleaseId).toBe("
|
|
851
|
-
expect(failed.generationTransition?.phase).toBe("
|
|
852
|
-
expect(String(
|
|
853
|
-
expect(
|
|
854
|
-
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
|
|
855
|
-
|
|
856
|
-
await fs.writeFile(fixture.retirementGatePath, "allow\n")
|
|
857
|
-
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
858
|
-
|
|
859
|
-
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
860
|
-
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
|
|
852
|
+
expect(failed.activeReleaseId).toBe("v2")
|
|
853
|
+
expect(failed.generationTransition?.phase).toBe("committed")
|
|
854
|
+
expect(String(statusRelease(daemon, "v1").retirementError)).toMatch(/quiet command exited non-zero/)
|
|
855
|
+
expect(result.activeReleaseId).toBe("v2")
|
|
861
856
|
} finally {
|
|
862
857
|
await daemon.shutdown()
|
|
863
858
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -894,7 +889,7 @@ test("candidate activation failure reports restoration failure and exact recover
|
|
|
894
889
|
expect(String(activationEvent?.data.error)).toMatch(/activate command exited non-zero/)
|
|
895
890
|
expect(String(restorationEvent?.data.activationError)).toMatch(/activate command exited non-zero/)
|
|
896
891
|
expect(String(restorationEvent?.data.error)).toMatch(/incumbent restoration rejected/)
|
|
897
|
-
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:
|
|
892
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2"])
|
|
898
893
|
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
|
|
899
894
|
const failedCandidate = daemon.releases.get("v2")
|
|
900
895
|
|
|
@@ -923,7 +918,8 @@ test("candidate activation failure reports restoration failure and exact recover
|
|
|
923
918
|
const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
|
|
924
919
|
|
|
925
920
|
expect(persisted?.generationTransition).toBe(undefined)
|
|
926
|
-
|
|
921
|
+
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 4)
|
|
922
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "retire:v1", "activate:v1"])
|
|
927
923
|
|
|
928
924
|
const idempotent = await sendControlCommand({
|
|
929
925
|
command: {
|
|
@@ -1224,7 +1220,7 @@ test("candidate activation failure compensates to the incumbent and admits a dif
|
|
|
1224
1220
|
expect(compensated.generationTransition).toBe(undefined)
|
|
1225
1221
|
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
1226
1222
|
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("running")
|
|
1227
|
-
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:
|
|
1223
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "activate:v1"])
|
|
1228
1224
|
|
|
1229
1225
|
await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
|
|
1230
1226
|
|
|
@@ -1246,7 +1242,6 @@ test("ambiguous candidate activation retires the candidate before reactivating t
|
|
|
1246
1242
|
|
|
1247
1243
|
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual([
|
|
1248
1244
|
"activate:v1",
|
|
1249
|
-
"retire:v1",
|
|
1250
1245
|
"activate:v2",
|
|
1251
1246
|
"retire:v2",
|
|
1252
1247
|
"activate:v1"
|
|
@@ -1300,7 +1295,7 @@ test("candidate activation recovery keeps the fence when a worker-specific resum
|
|
|
1300
1295
|
expect(status.generationTransition?.phase).toBe("restoring_previous")
|
|
1301
1296
|
expect(String(status.generationTransition?.activationError)).toMatch(/activate command exited non-zero/)
|
|
1302
1297
|
expect(String(status.generationTransition?.compensationError)).toMatch(/reactivate command exited non-zero/)
|
|
1303
|
-
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("
|
|
1298
|
+
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("running")
|
|
1304
1299
|
expect(String(restorationEvent?.data.activationError)).toMatch(/activate command exited non-zero/)
|
|
1305
1300
|
expect(String(restorationEvent?.data.error)).toMatch(/reactivate command exited non-zero/)
|
|
1306
1301
|
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
|
|
@@ -1461,7 +1456,8 @@ test("retired generation coordinator remains fenced after exit", async () => {
|
|
|
1461
1456
|
expect(exit.code).toBe(null)
|
|
1462
1457
|
expect(exit.id).toBe("beacon")
|
|
1463
1458
|
expect(exit.signal).toBe("SIGKILL")
|
|
1464
|
-
|
|
1459
|
+
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
|
|
1460
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
|
|
1465
1461
|
expect(stopped.lifecycleRole).toBe("retired")
|
|
1466
1462
|
expect(stopped.pid).toBe(undefined)
|
|
1467
1463
|
expect(stopped.restarts).toBe(0)
|
|
@@ -1547,11 +1543,12 @@ test("handoff-service quiescence failure is visible and leaves the generation al
|
|
|
1547
1543
|
const retired = statusRelease(daemon, "v1")
|
|
1548
1544
|
|
|
1549
1545
|
expect(retired.state).toBe("draining")
|
|
1550
|
-
|
|
1551
|
-
expect(
|
|
1546
|
+
await waitFor(() => Boolean(statusRelease(daemon, "v1").retirementError))
|
|
1547
|
+
expect(String(statusRelease(daemon, "v1").retirementError)).toMatch(/quiet command exited non-zero.*23/)
|
|
1548
|
+
expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).not.toBe("running")
|
|
1552
1549
|
expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state).not.toBe("stopped")
|
|
1553
1550
|
expect(logs.some((entry) => entry.message === "release retirement quiescence failed" && entry.data?.releaseId === "v1")).toBeTruthy()
|
|
1554
|
-
expect(result
|
|
1551
|
+
expect(result).toEqual({activeReleaseId: "v2", previousReleaseId: "v1"})
|
|
1555
1552
|
} finally {
|
|
1556
1553
|
await daemon.shutdown()
|
|
1557
1554
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -2571,14 +2568,13 @@ test("deploy can ensure the daemon before sending the release command", async ()
|
|
|
2571
2568
|
|
|
2572
2569
|
/**
|
|
2573
2570
|
* @param {{companionReplicas?: number, gatedWeb?: boolean, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
|
|
2574
|
-
* @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string
|
|
2571
|
+
* @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
|
|
2575
2572
|
*/
|
|
2576
2573
|
async function createFixture(options = {}) {
|
|
2577
2574
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-test-"))
|
|
2578
2575
|
const serviceLogPath = path.join(root, "service.log")
|
|
2579
2576
|
const serviceQuietPath = path.join(root, "service.quiet")
|
|
2580
2577
|
const lifecycleLogPath = path.join(root, "service.lifecycle")
|
|
2581
|
-
const workerReadyPath = path.join(root, "worker.ready")
|
|
2582
2578
|
const activationGatePath = path.join(root, "activation.allow")
|
|
2583
2579
|
const retirementGatePath = path.join(root, "retirement.allow")
|
|
2584
2580
|
const singletonLogPath = path.join(root, "singleton.log")
|
|
@@ -2629,7 +2625,7 @@ async function createFixture(options = {}) {
|
|
|
2629
2625
|
|
|
2630
2626
|
if (options.nonBlockingDrainWorker) {
|
|
2631
2627
|
processes.push({
|
|
2632
|
-
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0}));
|
|
2628
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0})); setInterval(() => {}, 1000)`)}`,
|
|
2633
2629
|
id: "worker",
|
|
2634
2630
|
...(options.workerReactivationLifecycle ? {lifecycle: {
|
|
2635
2631
|
quietCommand: `printf 'worker-retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`,
|
|
@@ -2697,7 +2693,7 @@ async function createFixture(options = {}) {
|
|
|
2697
2693
|
...((options.persistState || options.handoffServiceActivate) ? {ownerRecovery: {reconnectGraceMs: 30000}, statePath} : {})
|
|
2698
2694
|
})
|
|
2699
2695
|
|
|
2700
|
-
return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath
|
|
2696
|
+
return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
|
|
2701
2697
|
}
|
|
2702
2698
|
|
|
2703
2699
|
/**
|
|
@@ -2908,21 +2904,6 @@ async function waitFor(callback, timeoutMs = 3000) {
|
|
|
2908
2904
|
throw new Error("Timed out waiting for condition")
|
|
2909
2905
|
}
|
|
2910
2906
|
|
|
2911
|
-
/**
|
|
2912
|
-
* @param {string} filePath - File to await.
|
|
2913
|
-
* @returns {Promise<void>} Resolves once the file exists.
|
|
2914
|
-
*/
|
|
2915
|
-
async function waitForFile(filePath) {
|
|
2916
|
-
await waitFor(async () => {
|
|
2917
|
-
try {
|
|
2918
|
-
await fs.access(filePath)
|
|
2919
|
-
return true
|
|
2920
|
-
} catch {
|
|
2921
|
-
return false
|
|
2922
|
-
}
|
|
2923
|
-
})
|
|
2924
|
-
}
|
|
2925
|
-
|
|
2926
2907
|
/**
|
|
2927
2908
|
* Resolves from retained output or the exact event that records it, and rejects if the
|
|
2928
2909
|
* supervised process exits first.
|