rollbridge 0.1.39 → 0.1.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +12 -1
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +16 -2
- package/changelog.d/20260830055159-guardian-daemon-restart.md +2 -0
- package/docs/cli.md +23 -9
- package/docs/config.md +37 -13
- package/docs/logging.md +4 -3
- package/docs/troubleshooting.md +7 -5
- package/package.json +1 -1
- package/src/cli.js +93 -24
- package/src/config.js +14 -6
- package/src/daemon.js +643 -150
- package/src/guardian-client.js +68 -16
- package/src/managed-process.js +55 -8
- package/src/process-guardian.js +731 -43
- package/src/release-group.js +23 -7
- package/test/config-validation.test.js +4 -0
- package/test/fixtures/guardian-recovery-owner.js +86 -0
- package/test/fixtures/pre-split3-process-guardian.js +14 -0
- package/test/guardian-client.test.js +1339 -62
- package/test/managed-process.test.js +136 -7
- package/test/owner-recovery.test.js +564 -55
- package/test/owner-replacement.test.js +526 -23
- package/test/release-group.test.js +19 -2
- package/test/release-runtime-retention.test.js +121 -4
- package/test/rollbridge.test.js +21 -0
- package/test/support/process.js +41 -0
package/src/release-group.js
CHANGED
|
@@ -141,14 +141,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
141
141
|
.filter((processConfig) => processConfig.port && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
|
|
142
142
|
.map((processConfig) => snapshot.ports[processConfig.id])
|
|
143
143
|
const distinctGenerationPorts = new Set(generationPorts)
|
|
144
|
+
const ownsGenerationPorts = snapshot.state === "starting" || snapshot.state === "active" || snapshot.state === "draining"
|
|
144
145
|
|
|
145
146
|
if (distinctGenerationPorts.size !== generationPorts.length) throw new Error(`Persisted release ${this.releaseId} reuses a port within one live generation`)
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
147
|
+
if (ownsGenerationPorts) {
|
|
148
|
+
for (const port of distinctGenerationPorts) {
|
|
149
|
+
if (this.portReservations.has(port)) throw new Error(`Persisted release ${this.releaseId} port ${port} is already reserved by another live generation`)
|
|
150
|
+
}
|
|
151
|
+
for (const port of distinctGenerationPorts) {
|
|
152
|
+
this.portReservations.add(port)
|
|
153
|
+
this.ownedPortReservations.add(port)
|
|
154
|
+
}
|
|
152
155
|
}
|
|
153
156
|
|
|
154
157
|
this.ports = {...snapshot.ports}
|
|
@@ -281,6 +284,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
281
284
|
* The caller must prove the durable transition identity before using this path.
|
|
282
285
|
*/
|
|
283
286
|
async restartCommittedGeneration() {
|
|
287
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
284
288
|
this.assertCommittedGenerationRecoverable()
|
|
285
289
|
this.state = "starting"
|
|
286
290
|
try {
|
|
@@ -288,7 +292,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
288
292
|
const instances = this.getProcesses(processConfig.id)
|
|
289
293
|
|
|
290
294
|
if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
|
|
291
|
-
for (const {process} of instances)
|
|
295
|
+
for (const {process} of instances) {
|
|
296
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
297
|
+
await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
|
|
298
|
+
}
|
|
292
299
|
|
|
293
300
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
294
301
|
await waitForHealth({
|
|
@@ -296,6 +303,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
296
303
|
host: this.config.proxy.upstreamHost,
|
|
297
304
|
port: this.ports[processConfig.id]
|
|
298
305
|
})
|
|
306
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
299
307
|
}
|
|
300
308
|
}
|
|
301
309
|
} catch (error) {
|
|
@@ -564,6 +572,14 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
564
572
|
return this.transferredConnections.http + this.transferredConnections.websocket > 0
|
|
565
573
|
}
|
|
566
574
|
|
|
575
|
+
/** @returns {ReleaseConnections} Connections physically owned by this daemon listener. */
|
|
576
|
+
localConnections() {
|
|
577
|
+
return {
|
|
578
|
+
http: this.connections.http - this.transferredConnections.http,
|
|
579
|
+
websocket: this.connections.websocket - this.transferredConnections.websocket
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
|
|
567
583
|
/** Pauses only daemon-local connection-dependent retirement at owner handoff. */
|
|
568
584
|
pauseDrainForOwnerHandoff() {
|
|
569
585
|
if (this.state !== "draining") return
|
|
@@ -231,6 +231,10 @@ test("validateConfig accepts one durable handoff activation lifecycle and reject
|
|
|
231
231
|
const invalidType = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], lifecycle: {activateCommand: 5, quietCommand: "jobs retire"}}]})
|
|
232
232
|
assert.ok(invalidType.issues.some((issue) => issue.message === "processes[1].lifecycle.activateCommand must be a string"))
|
|
233
233
|
|
|
234
|
+
const emptyCommands = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], lifecycle: {activateCommand: " ", quietCommand: ""}}]})
|
|
235
|
+
assert.ok(emptyCommands.issues.some((issue) => issue.message === "processes[1].lifecycle.activateCommand must not be empty"))
|
|
236
|
+
assert.ok(emptyCommands.issues.some((issue) => issue.message === "processes[1].lifecycle.quietCommand must not be empty"))
|
|
237
|
+
|
|
234
238
|
const missingRetirement = validateConfig({...base, processes: [base.processes[0], {...base.processes[1], lifecycle: {activateCommand: "jobs activate"}}]})
|
|
235
239
|
assert.ok(missingRetirement.issues.some((issue) => /requires lifecycle\.quietCommand/.test(issue.message)))
|
|
236
240
|
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs"
|
|
4
|
+
import net from "node:net"
|
|
5
|
+
import {spawn} from "node:child_process"
|
|
6
|
+
|
|
7
|
+
const authorityText = process.env.GUARDIAN_AUTHORITY
|
|
8
|
+
const claimDelayMs = Number(process.env.GUARDIAN_CLAIM_DELAY_MS || 0)
|
|
9
|
+
const descendantPath = process.env.GUARDIAN_DESCENDANT_PATH
|
|
10
|
+
const exitAfterClaim = process.env.GUARDIAN_EXIT_AFTER_CLAIM === "1"
|
|
11
|
+
const markerPath = process.env.GUARDIAN_MARKER_PATH
|
|
12
|
+
const replacementCommittedPath = process.env.GUARDIAN_REPLACEMENT_COMMITTED_PATH
|
|
13
|
+
const replacementPreparedPath = process.env.GUARDIAN_REPLACEMENT_PREPARED_PATH
|
|
14
|
+
const skipReady = process.env.GUARDIAN_SKIP_READY === "1"
|
|
15
|
+
const socketPath = process.env.GUARDIAN_SOCKET_PATH
|
|
16
|
+
const startedLogPath = process.env.GUARDIAN_STARTED_LOG_PATH
|
|
17
|
+
const startedPath = process.env.GUARDIAN_STARTED_PATH
|
|
18
|
+
const token = process.env.GUARDIAN_TOKEN
|
|
19
|
+
|
|
20
|
+
if (!authorityText || !markerPath || !socketPath || !token || !Number.isInteger(claimDelayMs) || claimDelayMs < 0) {
|
|
21
|
+
throw new Error("Guardian recovery owner fixture requires authority, marker, socket, token, and a valid claim delay")
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
let buffer = ""
|
|
25
|
+
/** @type {string | undefined} */
|
|
26
|
+
let replacementId
|
|
27
|
+
let replacementCommitted = false
|
|
28
|
+
|
|
29
|
+
if (startedPath) fs.writeFileSync(startedPath, `${process.pid}\n`)
|
|
30
|
+
if (startedLogPath) fs.appendFileSync(startedLogPath, `${JSON.stringify({at: Date.now(), pid: process.pid})}\n`)
|
|
31
|
+
if (descendantPath) {
|
|
32
|
+
const descendant = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {stdio: "ignore"})
|
|
33
|
+
|
|
34
|
+
if (!descendant.pid) throw new Error("Guardian recovery owner fixture descendant did not start")
|
|
35
|
+
fs.writeFileSync(descendantPath, `${descendant.pid}\n`)
|
|
36
|
+
descendant.unref()
|
|
37
|
+
}
|
|
38
|
+
const timer = setTimeout(() => {
|
|
39
|
+
const socket = net.createConnection(socketPath)
|
|
40
|
+
|
|
41
|
+
socket.setEncoding("utf8")
|
|
42
|
+
socket.once("connect", () => {
|
|
43
|
+
socket.write(`${JSON.stringify({authority: JSON.parse(authorityText), command: "claim-owner", graceMs: 0, id: 1, ownerPid: process.pid, token})}\n`)
|
|
44
|
+
})
|
|
45
|
+
socket.on("data", (chunk) => {
|
|
46
|
+
buffer += chunk
|
|
47
|
+
let newline = buffer.indexOf("\n")
|
|
48
|
+
|
|
49
|
+
while (newline >= 0) {
|
|
50
|
+
const response = JSON.parse(buffer.slice(0, newline))
|
|
51
|
+
|
|
52
|
+
buffer = buffer.slice(newline + 1)
|
|
53
|
+
if (response.error) throw new Error(String(response.error))
|
|
54
|
+
if (response.id === 1) {
|
|
55
|
+
fs.writeFileSync(markerPath, `${process.pid}\n`)
|
|
56
|
+
if (exitAfterClaim) process.exit(47)
|
|
57
|
+
if (!skipReady) socket.write(`${JSON.stringify({command: "owner-ready", id: 2, ownerPid: process.pid, token})}\n`)
|
|
58
|
+
}
|
|
59
|
+
if (response.event === "replacement-prepared") {
|
|
60
|
+
replacementId = response.replacementId
|
|
61
|
+
if (replacementPreparedPath) fs.writeFileSync(replacementPreparedPath, `${replacementId}\n`)
|
|
62
|
+
}
|
|
63
|
+
if (response.id === 3) {
|
|
64
|
+
replacementCommitted = true
|
|
65
|
+
if (replacementCommittedPath) fs.writeFileSync(replacementCommittedPath, `${replacementId}\n`)
|
|
66
|
+
}
|
|
67
|
+
if (response.id === 4) socket.write(`${JSON.stringify({command: "finalize-owner-replacement", id: 5, replacementId, token})}\n`)
|
|
68
|
+
if (response.id === 5) socket.destroy()
|
|
69
|
+
newline = buffer.indexOf("\n")
|
|
70
|
+
}
|
|
71
|
+
})
|
|
72
|
+
process.once("SIGUSR1", () => {
|
|
73
|
+
if (!replacementCommitted) {
|
|
74
|
+
socket.destroy()
|
|
75
|
+
return
|
|
76
|
+
}
|
|
77
|
+
socket.write(`${JSON.stringify({command: "complete-owner-listener-retirement", id: 4, replacementId, token})}\n`)
|
|
78
|
+
})
|
|
79
|
+
process.once("SIGUSR2", () => {
|
|
80
|
+
if (!replacementId) throw new Error("Guardian recovery owner fixture has no prepared replacement")
|
|
81
|
+
socket.write(`${JSON.stringify({command: "commit-owner-replacement", id: 3, replacementId, token})}\n`)
|
|
82
|
+
})
|
|
83
|
+
}, claimDelayMs)
|
|
84
|
+
|
|
85
|
+
timer.unref()
|
|
86
|
+
setInterval(() => {}, 1000)
|
|
@@ -212,6 +212,20 @@ async function execute(request, socket) {
|
|
|
212
212
|
} else if (request.command === "update") {
|
|
213
213
|
if (!request.definition || !request.provenance) throw new Error("Guardian update requires definition and provenance")
|
|
214
214
|
if (record.provenance !== request.previousProvenance) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
215
|
+
const updateGatePath = request.definition.env?.ROLLBRIDGE_TEST_UPDATE_GATE
|
|
216
|
+
|
|
217
|
+
if (updateGatePath) {
|
|
218
|
+
await fs.writeFile(`${updateGatePath}.waiting`, "waiting\n")
|
|
219
|
+
while (true) {
|
|
220
|
+
try {
|
|
221
|
+
await fs.access(updateGatePath)
|
|
222
|
+
break
|
|
223
|
+
} catch (error) {
|
|
224
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
225
|
+
}
|
|
226
|
+
await new Promise((resolve) => setTimeout(resolve, 10))
|
|
227
|
+
}
|
|
228
|
+
}
|
|
215
229
|
record.process.updateDefinition({
|
|
216
230
|
...request.definition,
|
|
217
231
|
lifecycle: request.definition.lifecycle || {drainTimeoutMs: 0},
|