rollbridge 0.1.29 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/daemon.js +20 -10
- package/src/guardian-client.js +5 -0
- package/src/process-guardian.js +13 -0
- package/test/owner-recovery.test.js +94 -2
package/package.json
CHANGED
package/src/daemon.js
CHANGED
|
@@ -1437,11 +1437,11 @@ export default class RollbridgeDaemon {
|
|
|
1437
1437
|
/**
|
|
1438
1438
|
* Persists a state snapshot (status plus recent events) to statePath, atomically and
|
|
1439
1439
|
* fire-and-forget unless the caller awaits the returned write. A failed write is logged.
|
|
1440
|
-
* @param {{throwOnError?: boolean}} [options] -
|
|
1440
|
+
* @param {{allowStopping?: boolean, throwOnError?: boolean}} [options] - Write behavior.
|
|
1441
1441
|
* @returns {Promise<void> | undefined} The queued write, or undefined when persistence is disabled.
|
|
1442
1442
|
*/
|
|
1443
|
-
persistState({throwOnError = false} = {}) {
|
|
1444
|
-
if (!this.statePath || !this.persistenceEnabled || this.stopping) return
|
|
1443
|
+
persistState({allowStopping = false, throwOnError = false} = {}) {
|
|
1444
|
+
if (!this.statePath || !this.persistenceEnabled || (this.stopping && !allowStopping)) return
|
|
1445
1445
|
|
|
1446
1446
|
const statePath = this.statePath
|
|
1447
1447
|
const status = /** @type {Record<string, JsonValue>} */ (secretSafeStateValue(this.status()))
|
|
@@ -1533,25 +1533,35 @@ export default class RollbridgeDaemon {
|
|
|
1533
1533
|
clearInterval(this.persistTimer)
|
|
1534
1534
|
this.persistTimer = undefined
|
|
1535
1535
|
}
|
|
1536
|
-
this.persistenceEnabled = false
|
|
1537
1536
|
if (this.pendingWrite) await this.pendingWrite
|
|
1538
1537
|
this.stateCleanupEnabled = false
|
|
1539
1538
|
this.controlClosePromise = this.closeServer(this.controlServer)
|
|
1540
1539
|
for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
|
|
1540
|
+
if (this.activeRelease) {
|
|
1541
|
+
await this.activeRelease.beginRetirement(this.activeRelease.config)
|
|
1542
|
+
this.activeRelease = undefined
|
|
1543
|
+
}
|
|
1541
1544
|
await Promise.all([
|
|
1542
1545
|
...[...this.services.values()].map((processInstance) => processInstance.quiesce()),
|
|
1543
1546
|
...[...this.singletons.values()].map((processInstance) => processInstance.quiesce()),
|
|
1544
1547
|
...[...this.startingReleases].map((release) => release.quiesce()),
|
|
1545
1548
|
...[...this.releases.values()].map((release) => release.quiesce())
|
|
1546
1549
|
])
|
|
1550
|
+
await this.persistState({allowStopping: true, throwOnError: true})
|
|
1551
|
+
this.persistenceEnabled = false
|
|
1547
1552
|
await this.removeControlSocket()
|
|
1548
1553
|
void this.closeServer(this.proxyServer)
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1554
|
+
if (this.guardian) {
|
|
1555
|
+
await this.guardian.retireOwner()
|
|
1556
|
+
this.guardian.disconnect()
|
|
1557
|
+
} else {
|
|
1558
|
+
void Promise.allSettled([
|
|
1559
|
+
...[...this.services.values()].map((processInstance) => processInstance.stop()),
|
|
1560
|
+
...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
|
|
1561
|
+
...[...this.startingReleases].map((release) => release.stop()),
|
|
1562
|
+
...[...this.releases.values()].map((release) => release.stop())
|
|
1563
|
+
])
|
|
1564
|
+
}
|
|
1555
1565
|
this.logger("external owner retired", {attestation, status: "draining"})
|
|
1556
1566
|
}
|
|
1557
1567
|
|
package/src/guardian-client.js
CHANGED
|
@@ -156,6 +156,11 @@ export default class GuardianClient {
|
|
|
156
156
|
await this.request({authority, command: "claim-owner", graceMs})
|
|
157
157
|
}
|
|
158
158
|
|
|
159
|
+
/** Starts graceful process retirement and relinquishes committed owner authority. */
|
|
160
|
+
async retireOwner() {
|
|
161
|
+
await this.request({command: "retire-owner"})
|
|
162
|
+
}
|
|
163
|
+
|
|
159
164
|
/** @param {import("./json.js").JsonValue} ownerState - Private transferable owner state. */
|
|
160
165
|
async publishOwnerState(ownerState) {
|
|
161
166
|
await this.request({command: "publish-owner-state", ownerState})
|
package/src/process-guardian.js
CHANGED
|
@@ -200,6 +200,19 @@ async function execute(request, socket) {
|
|
|
200
200
|
})
|
|
201
201
|
}
|
|
202
202
|
|
|
203
|
+
if (request.command === "retire-owner") {
|
|
204
|
+
requireOwner(socket, request.command)
|
|
205
|
+
if (replacementClient) throw new Error("Committed owner cannot retire while an owner replacement is prepared")
|
|
206
|
+
for (const entry of processes.values()) entry.desired = false
|
|
207
|
+
void Promise.allSettled([...processes.values()].map((entry) => entry.process.stop()))
|
|
208
|
+
ownerClient = undefined
|
|
209
|
+
ownerMutationClient = undefined
|
|
210
|
+
ownerMutationId = undefined
|
|
211
|
+
ownerRevision += 1
|
|
212
|
+
grantNextOwner()
|
|
213
|
+
return {retired: true}
|
|
214
|
+
}
|
|
215
|
+
|
|
203
216
|
if (request.command === "abandon-legacy-upgrade") {
|
|
204
217
|
if (!legacyGuardian) throw new Error("Guardian is not a legacy upgrade coordinator")
|
|
205
218
|
if (ownerClient || committedReplacementId) throw new Error("Committed guardian authority cannot abandon its legacy backend")
|
|
@@ -9,6 +9,7 @@ import os from "node:os"
|
|
|
9
9
|
import path from "node:path"
|
|
10
10
|
import test from "node:test"
|
|
11
11
|
import {fileURLToPath} from "node:url"
|
|
12
|
+
import {normalizeConfig} from "../src/config.js"
|
|
12
13
|
import {sendControlCommand} from "../src/control-client.js"
|
|
13
14
|
import RollbridgeDaemon from "../src/daemon.js"
|
|
14
15
|
import GuardianClient from "../src/guardian-client.js"
|
|
@@ -21,6 +22,79 @@ const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
|
|
|
21
22
|
/** @typedef {import("../src/daemon.js").DaemonStatus} DaemonStatus */
|
|
22
23
|
/** @typedef {DaemonStatus & {recovery: {configDigest: string}}} RecoveryState */
|
|
23
24
|
|
|
25
|
+
test("external owner retirement releases guardian authority without losing its generation", async () => {
|
|
26
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-retirement-recovery-"))
|
|
27
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
28
|
+
const statePath = path.join(root, "rollbridge.state.json")
|
|
29
|
+
const v1Path = path.join(root, "v1")
|
|
30
|
+
const v2Path = path.join(root, "v2")
|
|
31
|
+
const config = normalizeConfig({
|
|
32
|
+
application: "owner-retirement-recovery-test",
|
|
33
|
+
control: {path: socketPath},
|
|
34
|
+
ownerRecovery: {reconnectGraceMs: 50},
|
|
35
|
+
processes: [
|
|
36
|
+
{
|
|
37
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
|
|
38
|
+
id: "worker",
|
|
39
|
+
lifecycle: {drainCommand: "printf started > \"$ROLLBRIDGE_RELEASE_PATH/drain-started\"; while [ ! -f \"$ROLLBRIDGE_RELEASE_PATH/drained\" ]; do sleep 0.01; done", drainTimeoutMs: 3000},
|
|
40
|
+
nonBlockingDrain: true,
|
|
41
|
+
policy: "companion"
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
command: `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`,
|
|
45
|
+
health: {intervalMs: 25, path: "/ping", timeoutMs: 3000},
|
|
46
|
+
id: "web",
|
|
47
|
+
policy: "proxied",
|
|
48
|
+
port: {from: 0, to: 0}
|
|
49
|
+
}
|
|
50
|
+
],
|
|
51
|
+
proxy: {forceStopTimeoutMs: 500, healthPath: "/ping", healthTimeoutMs: 3000, host: "127.0.0.1", port: 0},
|
|
52
|
+
statePath
|
|
53
|
+
})
|
|
54
|
+
const retired = new RollbridgeDaemon({config, logger: () => {}})
|
|
55
|
+
let replacement
|
|
56
|
+
|
|
57
|
+
try {
|
|
58
|
+
await Promise.all([fs.mkdir(v1Path), fs.mkdir(v2Path)])
|
|
59
|
+
await retired.start()
|
|
60
|
+
await retired.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
|
|
61
|
+
const before = retired.status()
|
|
62
|
+
const v1WorkerPid = releaseProcessPid(before, "v1", "worker")
|
|
63
|
+
|
|
64
|
+
replacement = new RollbridgeDaemon({config, logger: () => {}})
|
|
65
|
+
await replacement.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
66
|
+
const v2WorkerPid = releaseProcessPid(replacement.status(), "v2", "worker")
|
|
67
|
+
|
|
68
|
+
await retired.retireOwner({attestation: `sha256:${"a".repeat(64)}`})
|
|
69
|
+
await replacement.start({reportOrphans: false})
|
|
70
|
+
const recovered = replacement.status()
|
|
71
|
+
const v1 = recovered.releases.find(({releaseId}) => releaseId === "v1")
|
|
72
|
+
const v2 = recovered.releases.find(({releaseId}) => releaseId === "v2")
|
|
73
|
+
|
|
74
|
+
assert.equal(recovered.activeReleaseId, "v2", "the prestarted candidate must remain active")
|
|
75
|
+
assert.deepEqual(recovered.releaseReferences.sort((a, b) => a.releaseId.localeCompare(b.releaseId)), [
|
|
76
|
+
{releaseId: "v1", releasePath: v1Path},
|
|
77
|
+
{releaseId: "v2", releasePath: v2Path}
|
|
78
|
+
])
|
|
79
|
+
assert.equal(v1?.state, "draining")
|
|
80
|
+
assert.equal(v1?.processes.find(({id}) => id === "worker")?.pid, v1WorkerPid)
|
|
81
|
+
assert.equal(v1?.processes.find(({id}) => id === "worker")?.state, "quiesced")
|
|
82
|
+
assert.equal(v2?.state, "active")
|
|
83
|
+
assert.equal(v2?.processes.find(({id}) => id === "worker")?.pid, v2WorkerPid)
|
|
84
|
+
assert.equal(v2?.processes.find(({id}) => id === "worker")?.state, "running")
|
|
85
|
+
await waitForFile(path.join(v1Path, "drain-started"), 1000)
|
|
86
|
+
await fs.writeFile(path.join(v1Path, "drained"), "done\n")
|
|
87
|
+
await waitForProcessExit(v1WorkerPid, 1000)
|
|
88
|
+
assert.equal(isAlive(v2WorkerPid), true, "the active candidate worker must remain usable while the old generation drains")
|
|
89
|
+
} finally {
|
|
90
|
+
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "drained"), "done\n").catch(() => {})))
|
|
91
|
+
await replacement?.shutdown().catch(() => {})
|
|
92
|
+
retired.guardian?.disconnect()
|
|
93
|
+
await stopFixtureGuardian(statePath)
|
|
94
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
95
|
+
}
|
|
96
|
+
})
|
|
97
|
+
|
|
24
98
|
test("replacement owner reconstructs one active and two draining generations after abrupt daemon exit", async () => {
|
|
25
99
|
const fixture = await createFixture()
|
|
26
100
|
let owner = spawnDaemon(fixture.configPath)
|
|
@@ -687,16 +761,19 @@ async function waitForState(statePath, predicate) {
|
|
|
687
761
|
|
|
688
762
|
/**
|
|
689
763
|
* @param {string} filePath - File whose creation is the transaction-boundary signal.
|
|
764
|
+
* @param {number} [timeoutMs] - Optional bounded wait.
|
|
690
765
|
* @returns {Promise<void>} Resolves when the file exists.
|
|
691
766
|
*/
|
|
692
|
-
async function waitForFile(filePath) {
|
|
767
|
+
async function waitForFile(filePath, timeoutMs) {
|
|
693
768
|
try {
|
|
694
769
|
await fs.access(filePath)
|
|
695
770
|
return
|
|
696
771
|
} catch (error) {
|
|
697
772
|
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
698
773
|
}
|
|
699
|
-
const
|
|
774
|
+
const controller = timeoutMs === undefined ? undefined : new AbortController()
|
|
775
|
+
const timer = timeoutMs === undefined ? undefined : setTimeout(() => controller?.abort(), timeoutMs)
|
|
776
|
+
const watcher = fs.watch(path.dirname(filePath), {signal: controller?.signal})
|
|
700
777
|
|
|
701
778
|
try {
|
|
702
779
|
for await (const change of watcher) {
|
|
@@ -708,11 +785,26 @@ async function waitForFile(filePath) {
|
|
|
708
785
|
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
709
786
|
}
|
|
710
787
|
}
|
|
788
|
+
} catch (error) {
|
|
789
|
+
if (error && typeof error === "object" && "name" in error && error.name === "AbortError") throw new Error(`Timed out waiting for ${filePath}`, {cause: error})
|
|
790
|
+
throw error
|
|
711
791
|
} finally {
|
|
792
|
+
clearTimeout(timer)
|
|
712
793
|
await watcher.return?.()
|
|
713
794
|
}
|
|
714
795
|
}
|
|
715
796
|
|
|
797
|
+
/**
|
|
798
|
+
* @param {number} pid - Exact fixture process.
|
|
799
|
+
* @param {number} timeoutMs - Bounded exit wait.
|
|
800
|
+
*/
|
|
801
|
+
async function waitForProcessExit(pid, timeoutMs) {
|
|
802
|
+
const deadline = Date.now() + timeoutMs
|
|
803
|
+
|
|
804
|
+
while (isAlive(pid) && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 10))
|
|
805
|
+
assert.equal(isAlive(pid), false, `process ${pid} did not exit within ${timeoutMs}ms`)
|
|
806
|
+
}
|
|
807
|
+
|
|
716
808
|
/**
|
|
717
809
|
* Opens a live WebSocket through the fixture proxy.
|
|
718
810
|
* @param {number} port - Proxy port.
|