rollbridge 0.1.38 → 0.1.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/cli.md +10 -0
- package/package.json +1 -1
- package/src/cli.js +7 -2
- package/src/daemon.js +99 -4
- package/src/release-group.js +55 -0
- package/test/owner-recovery.test.js +253 -0
package/docs/cli.md
CHANGED
|
@@ -64,6 +64,16 @@ identity. Rollbridge does not calculate or interpret the digest.
|
|
|
64
64
|
With no release options, daemon behavior is unchanged: it starts listener-only
|
|
65
65
|
and waits for control-socket deployments.
|
|
66
66
|
|
|
67
|
+
If an external owner retirement has already journaled a committed generation but
|
|
68
|
+
cleared its active role, only a foreground bootstrap with that exact release id,
|
|
69
|
+
path, revision, and config authority may restore it. Rollbridge waits for the
|
|
70
|
+
retiring candidate processes to stop, journals `restoring_committed`, reconnects
|
|
71
|
+
their existing guardian registrations, restarts that candidate, health-checks it,
|
|
72
|
+
and restores its generation activation before completing singletons and exposing
|
|
73
|
+
control. A later exact bootstrap resumes the journaled restart without duplicating
|
|
74
|
+
processes. A mismatched tuple or a candidate that is still retiring fails closed
|
|
75
|
+
without stopping other retained generations.
|
|
76
|
+
|
|
67
77
|
`--takeover-owner` requires the complete bootstrap tuple. It bootstraps and
|
|
68
78
|
health-checks the replacement before sending the current daemon the private
|
|
69
79
|
retirement command. The current `performOwnerRetirement` path quiesces every
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -102,8 +102,13 @@ export async function runCli(argv) {
|
|
|
102
102
|
|
|
103
103
|
daemon.logger("bootstrap activation failed", {releaseId: bootstrap.releaseId, status: "error", ...errorLogData(failure)})
|
|
104
104
|
|
|
105
|
-
if (config.ownerRecovery && daemon.
|
|
106
|
-
|
|
105
|
+
if (config.ownerRecovery && daemon.releases.size > 0) {
|
|
106
|
+
if (daemon.activeRelease) {
|
|
107
|
+
await daemon.exposeControl()
|
|
108
|
+
return
|
|
109
|
+
}
|
|
110
|
+
await daemon.abandonOwnerRecoveryAttempt()
|
|
111
|
+
process.exitCode = 1
|
|
107
112
|
return
|
|
108
113
|
}
|
|
109
114
|
|
package/src/daemon.js
CHANGED
|
@@ -23,7 +23,7 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
23
23
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
24
24
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
25
25
|
* @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
26
|
-
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed"} GenerationTransitionPhase
|
|
26
|
+
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
27
27
|
* @typedef {{candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, configDigest: string, error?: string, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
28
28
|
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
29
29
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
@@ -228,7 +228,7 @@ export default class RollbridgeDaemon {
|
|
|
228
228
|
}
|
|
229
229
|
this.generationTransition = snapshot.generationTransition ? {...snapshot.generationTransition} : undefined
|
|
230
230
|
if (snapshot.activeReleaseId === null && snapshot.releases.length === 0) return
|
|
231
|
-
this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
|
|
231
|
+
if (!this.bootstrap) this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
|
|
232
232
|
this.ownerTransition = snapshot.ownerTransition ? {...snapshot.ownerTransition} : undefined
|
|
233
233
|
const singletonOwnerReleaseIds = new Set(Object.values(singletonReleaseIds))
|
|
234
234
|
|
|
@@ -255,9 +255,10 @@ export default class RollbridgeDaemon {
|
|
|
255
255
|
}
|
|
256
256
|
|
|
257
257
|
if (snapshot.activeReleaseId !== null && !this.activeRelease) throw new Error(`Owner recovery state does not contain active release ${snapshot.activeReleaseId}.`)
|
|
258
|
-
const
|
|
258
|
+
const committedBootstrapRelease = this.committedBootstrapRelease()
|
|
259
|
+
const definitionRelease = this.activeRelease || committedBootstrapRelease || [...this.releases.values()].at(-1)
|
|
259
260
|
if (!definitionRelease) throw new Error("Owner recovery state has no release definition for owned processes.")
|
|
260
|
-
if (!this.activeRelease && snapshot.singletons.length > 0) throw new Error("Owner recovery state has release-owned singletons without an active release identity.")
|
|
261
|
+
if (!this.activeRelease && !committedBootstrapRelease && snapshot.singletons.length > 0) throw new Error("Owner recovery state has release-owned singletons without an active release identity.")
|
|
261
262
|
for (const serviceStatus of snapshot.services) {
|
|
262
263
|
const processConfig = config.processes.find((candidate) => candidate.id === serviceStatus.id && candidate.policy === "service" && candidate.deployStrategy !== "handoff")
|
|
263
264
|
|
|
@@ -1242,6 +1243,13 @@ export default class RollbridgeDaemon {
|
|
|
1242
1243
|
this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
|
|
1243
1244
|
return {activeReleaseId: newReleaseId, previousReleaseId: transition.previousReleaseId}
|
|
1244
1245
|
}
|
|
1246
|
+
if (transition?.phase === "committed" && !this.activeRelease && this.bootstrap && this.releases.get(transition.candidateReleaseId)?.state === "draining") {
|
|
1247
|
+
this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
|
|
1248
|
+
this.assertCommittedBootstrapRecoveryReady()
|
|
1249
|
+
this.config = nextConfig
|
|
1250
|
+
await this.updateGenerationTransition("restoring_committed")
|
|
1251
|
+
return await this.resumeGenerationTransition()
|
|
1252
|
+
}
|
|
1245
1253
|
const release = new ReleaseGroup({
|
|
1246
1254
|
config: nextConfig,
|
|
1247
1255
|
logger: this.logger,
|
|
@@ -1387,6 +1395,16 @@ export default class RollbridgeDaemon {
|
|
|
1387
1395
|
this.logger("traffic switched", {previousReleaseId: previousRelease?.releaseId ?? null, releaseId: release.releaseId})
|
|
1388
1396
|
}
|
|
1389
1397
|
|
|
1398
|
+
if (transition.phase === "restoring_committed") {
|
|
1399
|
+
await this.resumeCommittedBootstrapGeneration(release)
|
|
1400
|
+
release.activate()
|
|
1401
|
+
this.activeRelease = release
|
|
1402
|
+
transition.phase = "committed_pending"
|
|
1403
|
+
transition.error = undefined
|
|
1404
|
+
transition.updatedAt = new Date().toISOString()
|
|
1405
|
+
this.logger("committed bootstrap generation restored", {releaseId: release.releaseId})
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1390
1408
|
if (transition.phase === "committed_pending") {
|
|
1391
1409
|
await this.checkpointGenerationTransition()
|
|
1392
1410
|
this.refreshServiceDefinitions(release)
|
|
@@ -1420,6 +1438,82 @@ export default class RollbridgeDaemon {
|
|
|
1420
1438
|
}
|
|
1421
1439
|
}
|
|
1422
1440
|
|
|
1441
|
+
/**
|
|
1442
|
+
* Returns the retained candidate only when the foreground bootstrap exactly proves the
|
|
1443
|
+
* committed transition that an external owner retirement left without an active role.
|
|
1444
|
+
* @returns {ReleaseGroup | undefined} Exact committed bootstrap candidate.
|
|
1445
|
+
*/
|
|
1446
|
+
committedBootstrapRelease() {
|
|
1447
|
+
const bootstrap = this.bootstrap
|
|
1448
|
+
const transition = this.generationTransition
|
|
1449
|
+
|
|
1450
|
+
if (!bootstrap || !transition || (transition.phase !== "committed" && transition.phase !== "restoring_committed") || transition.candidateReleaseId !== bootstrap.releaseId || transition.candidateReleasePath !== bootstrap.releasePath || transition.candidateRevision !== bootstrap.revision || transition.configDigest !== ownerConfigDigest(this.config)) return undefined
|
|
1451
|
+
const release = this.releases.get(bootstrap.releaseId)
|
|
1452
|
+
|
|
1453
|
+
if (!release || release.releasePath !== bootstrap.releasePath || release.revision !== bootstrap.revision) return undefined
|
|
1454
|
+
return release
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
/**
|
|
1458
|
+
* Fails closed before journaling recovery unless external retirement has fully stopped
|
|
1459
|
+
* the exact candidate and daemon-owned processes.
|
|
1460
|
+
* @returns {void}
|
|
1461
|
+
*/
|
|
1462
|
+
assertCommittedBootstrapRecoveryReady() {
|
|
1463
|
+
const release = this.committedBootstrapRelease()
|
|
1464
|
+
|
|
1465
|
+
if (!release) throw new Error("Committed generation has no exact foreground bootstrap recovery proof")
|
|
1466
|
+
release.assertCommittedGenerationStopped()
|
|
1467
|
+
const ownedProcesses = [
|
|
1468
|
+
...this.services.values(),
|
|
1469
|
+
...this.singletons.values()
|
|
1470
|
+
]
|
|
1471
|
+
const stillRetiring = ownedProcesses.find((processInstance) => {
|
|
1472
|
+
const {pid, state} = processInstance.status()
|
|
1473
|
+
return pid !== undefined || (state !== "stopped" && state !== "failed")
|
|
1474
|
+
})
|
|
1475
|
+
|
|
1476
|
+
if (stillRetiring) throw new Error(`Committed generation daemon process ${stillRetiring.id} is still retiring; exact bootstrap recovery will retry after it stops`)
|
|
1477
|
+
for (const [singletonId, singletonReleaseId] of this.singletonReleaseIds) {
|
|
1478
|
+
if (singletonReleaseId !== release.releaseId) throw new Error(`Committed generation singleton ${singletonId} belongs to retained release ${singletonReleaseId}`)
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
/**
|
|
1483
|
+
* Resumes a durably journaled committed-candidate restart. Running processes are
|
|
1484
|
+
* necessarily owned by this exact recovery phase; stopped processes are restarted.
|
|
1485
|
+
* Singleton completion remains in the established committed_pending phase.
|
|
1486
|
+
* @param {ReleaseGroup} release - Exact committed candidate.
|
|
1487
|
+
* @returns {Promise<void>}
|
|
1488
|
+
*/
|
|
1489
|
+
async resumeCommittedBootstrapGeneration(release) {
|
|
1490
|
+
if (this.generationTransition?.phase !== "restoring_committed" || release !== this.committedBootstrapRelease()) {
|
|
1491
|
+
throw new Error("Committed bootstrap recovery is not durably journaled for this exact candidate")
|
|
1492
|
+
}
|
|
1493
|
+
const resumableStates = new Set(["failed", "running", "stopped"])
|
|
1494
|
+
const invalidProcess = [...this.services.values(), ...this.singletons.values()].find((processInstance) => {
|
|
1495
|
+
const {pid, state} = processInstance.status()
|
|
1496
|
+
return !resumableStates.has(state) || (state === "running") !== (pid !== undefined)
|
|
1497
|
+
})
|
|
1498
|
+
|
|
1499
|
+
if (invalidProcess) throw new Error(`Committed bootstrap recovery found daemon process ${invalidProcess.id} outside its journaled restart states`)
|
|
1500
|
+
release.assertCommittedGenerationRecoverable()
|
|
1501
|
+
|
|
1502
|
+
try {
|
|
1503
|
+
for (const processInstance of this.services.values()) {
|
|
1504
|
+
await processInstance.start("deploy")
|
|
1505
|
+
}
|
|
1506
|
+
await release.restartCommittedGeneration()
|
|
1507
|
+
await release.activateGeneration()
|
|
1508
|
+
} catch (error) {
|
|
1509
|
+
await Promise.allSettled([
|
|
1510
|
+
release.abortCommittedGenerationRestart(),
|
|
1511
|
+
...[...this.services.values()].map((processInstance) => processInstance.stop())
|
|
1512
|
+
])
|
|
1513
|
+
throw error
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1423
1517
|
/** @param {GenerationTransitionPhase} phase - Durable phase to enter. */
|
|
1424
1518
|
async updateGenerationTransition(phase) {
|
|
1425
1519
|
if (!this.generationTransition) throw new Error("No release generation transition to update")
|
|
@@ -1455,6 +1549,7 @@ export default class RollbridgeDaemon {
|
|
|
1455
1549
|
shouldResumeDrain(release) {
|
|
1456
1550
|
const transition = this.generationTransition
|
|
1457
1551
|
|
|
1552
|
+
if (release === this.committedBootstrapRelease()) return false
|
|
1458
1553
|
return !transition || transition.phase === "committed_pending" || transition.phase === "committed" || transition.previousReleaseId !== release.releaseId
|
|
1459
1554
|
}
|
|
1460
1555
|
|
package/src/release-group.js
CHANGED
|
@@ -276,6 +276,61 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
276
276
|
await instance.process.activateStrict()
|
|
277
277
|
}
|
|
278
278
|
|
|
279
|
+
/**
|
|
280
|
+
* Restarts only the exact processes reconstructed for a committed generation.
|
|
281
|
+
* The caller must prove the durable transition identity before using this path.
|
|
282
|
+
*/
|
|
283
|
+
async restartCommittedGeneration() {
|
|
284
|
+
this.assertCommittedGenerationRecoverable()
|
|
285
|
+
this.state = "starting"
|
|
286
|
+
try {
|
|
287
|
+
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
288
|
+
const instances = this.getProcesses(processConfig.id)
|
|
289
|
+
|
|
290
|
+
if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
|
|
291
|
+
for (const {process} of instances) await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
|
|
292
|
+
|
|
293
|
+
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
294
|
+
await waitForHealth({
|
|
295
|
+
health: processConfig.health,
|
|
296
|
+
host: this.config.proxy.upstreamHost,
|
|
297
|
+
port: this.ports[processConfig.id]
|
|
298
|
+
})
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
} catch (error) {
|
|
302
|
+
await this.abortCommittedGenerationRestart()
|
|
303
|
+
throw error
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/** Fails closed unless every exact candidate process has finished external retirement. */
|
|
308
|
+
assertCommittedGenerationStopped() {
|
|
309
|
+
if (this.state !== "draining") throw new Error(`Committed generation ${this.releaseId} is not retained as draining`)
|
|
310
|
+
const statuses = [...this.processes.values()].map((processInstance) => processInstance.status())
|
|
311
|
+
|
|
312
|
+
if (statuses.some(({pid, state}) => pid !== undefined || (state !== "stopped" && state !== "failed"))) {
|
|
313
|
+
throw new Error(`Committed generation ${this.releaseId} is still retiring; exact bootstrap recovery will retry after its processes stop`)
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** Accepts only process states produced after the exact recovery phase was journaled. */
|
|
318
|
+
assertCommittedGenerationRecoverable() {
|
|
319
|
+
if (this.state !== "draining" && this.state !== "starting") throw new Error(`Committed generation ${this.releaseId} is not retained in its journaled recovery state`)
|
|
320
|
+
const statuses = [...this.processes.values()].map((processInstance) => processInstance.status())
|
|
321
|
+
const resumableStates = new Set(["failed", "running", "stopped"])
|
|
322
|
+
|
|
323
|
+
if (statuses.some(({pid, state}) => !resumableStates.has(state) || (state === "running") !== (pid !== undefined))) {
|
|
324
|
+
throw new Error(`Committed generation ${this.releaseId} has a process outside its journaled restart states`)
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Stops only a failed committed-bootstrap restart while retaining its exact identity. */
|
|
329
|
+
async abortCommittedGenerationRestart() {
|
|
330
|
+
await Promise.allSettled([...this.processes.values()].map((processInstance) => processInstance.stop()))
|
|
331
|
+
this.state = "draining"
|
|
332
|
+
}
|
|
333
|
+
|
|
279
334
|
/** @returns {Promise<void>} Allocates all configured per-process ports. */
|
|
280
335
|
async allocatePorts() {
|
|
281
336
|
if (this.portsAllocated) return
|
|
@@ -95,6 +95,259 @@ test("external owner retirement releases guardian authority without losing its g
|
|
|
95
95
|
}
|
|
96
96
|
})
|
|
97
97
|
|
|
98
|
+
test("exact bootstrap restores the committed generation after external owner retirement", async () => {
|
|
99
|
+
const fixture = await createFixture()
|
|
100
|
+
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
101
|
+
const retired = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
102
|
+
/** @type {RollbridgeDaemon | undefined} */
|
|
103
|
+
let recovered
|
|
104
|
+
const v1Path = await prepareRelease(fixture.root, "v1")
|
|
105
|
+
const v2Path = await prepareRelease(fixture.root, "v2")
|
|
106
|
+
|
|
107
|
+
try {
|
|
108
|
+
await retired.start()
|
|
109
|
+
await retired.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
|
|
110
|
+
await retired.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
111
|
+
const committed = retired.status()
|
|
112
|
+
const v1WorkerPid = releaseProcessPid(committed, "v1", "worker")
|
|
113
|
+
const retiredPids = [
|
|
114
|
+
...(committed.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid) || []),
|
|
115
|
+
...committed.services.map(({process}) => process.pid),
|
|
116
|
+
...committed.singletons.map(({process}) => process.pid)
|
|
117
|
+
].filter((pid) => typeof pid === "number")
|
|
118
|
+
|
|
119
|
+
assert.equal(committed.activeReleaseId, "v2")
|
|
120
|
+
assert.equal(committed.generationTransition?.phase, "committed")
|
|
121
|
+
await retired.retireOwner({attestation: `sha256:${"a".repeat(64)}`})
|
|
122
|
+
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
123
|
+
await Promise.all(retiredPids.map((pid) => waitForProcessExit(pid, 3000)))
|
|
124
|
+
|
|
125
|
+
recovered = new RollbridgeDaemon({
|
|
126
|
+
bootstrap: {releaseId: "v2", releasePath: v2Path, revision: "v2"},
|
|
127
|
+
config,
|
|
128
|
+
configPath: fixture.configPath,
|
|
129
|
+
logger: () => {}
|
|
130
|
+
})
|
|
131
|
+
await recovered.start({exposeControl: false})
|
|
132
|
+
const recoveryOrder = /** @type {string[]} */ ([])
|
|
133
|
+
const recoveredCandidate = recovered.releases.get("v2")
|
|
134
|
+
const recoveredService = recovered.services.get("beacon")
|
|
135
|
+
|
|
136
|
+
assert.ok(recoveredCandidate && recoveredService && recovered.singletons.get("singleton"))
|
|
137
|
+
const activateGeneration = recoveredCandidate.activateGeneration.bind(recoveredCandidate)
|
|
138
|
+
const startService = recoveredService.start.bind(recoveredService)
|
|
139
|
+
const replaceSingletons = recovered.replaceSingletons.bind(recovered)
|
|
140
|
+
|
|
141
|
+
recoveredCandidate.activateGeneration = async () => {
|
|
142
|
+
recoveryOrder.push("activate")
|
|
143
|
+
await activateGeneration()
|
|
144
|
+
}
|
|
145
|
+
recoveredService.start = async (...args) => {
|
|
146
|
+
assert.equal(recovered?.generationTransition?.phase, "restoring_committed")
|
|
147
|
+
recoveryOrder.push("service")
|
|
148
|
+
await startService(...args)
|
|
149
|
+
}
|
|
150
|
+
recovered.replaceSingletons = async (...args) => {
|
|
151
|
+
recoveryOrder.push("singleton")
|
|
152
|
+
await replaceSingletons(...args)
|
|
153
|
+
}
|
|
154
|
+
await recovered.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
155
|
+
const active = recovered.status()
|
|
156
|
+
|
|
157
|
+
assert.equal(active.activeReleaseId, "v2")
|
|
158
|
+
assert.equal(active.generationTransition?.phase, "committed")
|
|
159
|
+
assert.equal(active.releases.find(({releaseId}) => releaseId === "v1")?.state, "draining")
|
|
160
|
+
assert.equal(releaseProcessPid(active, "v1", "worker"), v1WorkerPid)
|
|
161
|
+
assert.equal(isAlive(v1WorkerPid), true, "the retained previous generation must keep draining")
|
|
162
|
+
assert.equal(active.releases.find(({releaseId}) => releaseId === "v2")?.state, "active")
|
|
163
|
+
assert.ok(active.releases.find(({releaseId}) => releaseId === "v2")?.processes.every(({pid, state}) => typeof pid === "number" && state === "running"))
|
|
164
|
+
assert.ok(active.services.every(({process}) => typeof process.pid === "number" && process.state === "running"))
|
|
165
|
+
assert.ok(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running"))
|
|
166
|
+
assert.deepEqual(recoveryOrder, ["service", "activate", "singleton"], "candidate activation must precede post-commit singleton completion")
|
|
167
|
+
assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
|
|
168
|
+
} finally {
|
|
169
|
+
if (recovered) {
|
|
170
|
+
const activeRecovery = recovered.status().activeReleaseId === "v2"
|
|
171
|
+
const shutdown = recovered.shutdown()
|
|
172
|
+
|
|
173
|
+
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
174
|
+
if (activeRecovery) await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
175
|
+
await shutdown.catch(() => undefined)
|
|
176
|
+
}
|
|
177
|
+
retired.guardian?.disconnect()
|
|
178
|
+
await stopFixtureGuardian(fixture.statePath)
|
|
179
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
180
|
+
}
|
|
181
|
+
})
|
|
182
|
+
|
|
183
|
+
test("journaled committed bootstrap recovery resumes after a restart begins", async () => {
|
|
184
|
+
const fixture = await createFixture()
|
|
185
|
+
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
186
|
+
const retired = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
187
|
+
/** @type {RollbridgeDaemon | undefined} */
|
|
188
|
+
let interrupted
|
|
189
|
+
/** @type {RollbridgeDaemon | undefined} */
|
|
190
|
+
let recovered
|
|
191
|
+
const v1Path = await prepareRelease(fixture.root, "v1")
|
|
192
|
+
const v2Path = await prepareRelease(fixture.root, "v2")
|
|
193
|
+
|
|
194
|
+
try {
|
|
195
|
+
await retired.start()
|
|
196
|
+
await retired.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
|
|
197
|
+
await retired.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
198
|
+
const committed = retired.status()
|
|
199
|
+
const v1WorkerPid = releaseProcessPid(committed, "v1", "worker")
|
|
200
|
+
const retiringPids = [
|
|
201
|
+
...(committed.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid) || []),
|
|
202
|
+
...committed.services.map(({process}) => process.pid),
|
|
203
|
+
...committed.singletons.map(({process}) => process.pid)
|
|
204
|
+
].filter((pid) => typeof pid === "number")
|
|
205
|
+
|
|
206
|
+
await retired.retireOwner({attestation: `sha256:${"b".repeat(64)}`})
|
|
207
|
+
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
208
|
+
await Promise.all(retiringPids.map((pid) => waitForProcessExit(pid, 3000)))
|
|
209
|
+
|
|
210
|
+
const bootstrap = {releaseId: "v2", releasePath: v2Path, revision: "v2"}
|
|
211
|
+
interrupted = new RollbridgeDaemon({bootstrap, config, configPath: fixture.configPath, logger: () => {}})
|
|
212
|
+
await interrupted.start({exposeControl: false})
|
|
213
|
+
interrupted.assertCommittedBootstrapRecoveryReady()
|
|
214
|
+
await interrupted.updateGenerationTransition("restoring_committed")
|
|
215
|
+
const candidate = interrupted.releases.get("v2")
|
|
216
|
+
|
|
217
|
+
assert.ok(candidate)
|
|
218
|
+
for (const processInstance of interrupted.services.values()) await processInstance.start("deploy")
|
|
219
|
+
await candidate.restartCommittedGeneration()
|
|
220
|
+
await interrupted.checkpointGenerationTransition()
|
|
221
|
+
const restarted = interrupted.status()
|
|
222
|
+
const restartedCandidatePids = restarted.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid)
|
|
223
|
+
const restartedServicePids = restarted.services.map(({process}) => process.pid)
|
|
224
|
+
|
|
225
|
+
assert.equal(restarted.generationTransition?.phase, "restoring_committed")
|
|
226
|
+
assert.ok(restartedCandidatePids?.every((pid) => typeof pid === "number"))
|
|
227
|
+
await interrupted.retireCommittedOwner(undefined)
|
|
228
|
+
interrupted.guardian?.disconnect()
|
|
229
|
+
|
|
230
|
+
recovered = new RollbridgeDaemon({bootstrap, config, configPath: fixture.configPath, logger: () => {}})
|
|
231
|
+
await recovered.start({exposeControl: false})
|
|
232
|
+
const active = recovered.status()
|
|
233
|
+
|
|
234
|
+
assert.equal(active.activeReleaseId, "v2")
|
|
235
|
+
assert.equal(active.generationTransition?.phase, "committed")
|
|
236
|
+
assert.deepEqual(active.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid), restartedCandidatePids)
|
|
237
|
+
assert.deepEqual(active.services.map(({process}) => process.pid), restartedServicePids)
|
|
238
|
+
assert.ok(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running"))
|
|
239
|
+
assert.equal(releaseProcessPid(active, "v1", "worker"), v1WorkerPid)
|
|
240
|
+
assert.equal(isAlive(v1WorkerPid), true)
|
|
241
|
+
assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
|
|
242
|
+
} finally {
|
|
243
|
+
if (recovered) {
|
|
244
|
+
const shutdown = recovered.shutdown()
|
|
245
|
+
|
|
246
|
+
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n").catch(() => undefined)))
|
|
247
|
+
await shutdown.catch(() => undefined)
|
|
248
|
+
}
|
|
249
|
+
interrupted?.guardian?.disconnect()
|
|
250
|
+
retired.guardian?.disconnect()
|
|
251
|
+
await stopFixtureGuardian(fixture.statePath)
|
|
252
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
253
|
+
}
|
|
254
|
+
})
|
|
255
|
+
|
|
256
|
+
test("committed bootstrap tuple mismatches fail closed without singletons", async () => {
|
|
257
|
+
const fixture = await createFixture()
|
|
258
|
+
fixture.config.processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
259
|
+
.filter((processConfig) => processConfig.id !== "singleton")
|
|
260
|
+
await writeConfig(fixture.configPath, fixture.config)
|
|
261
|
+
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
262
|
+
const retired = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
263
|
+
/** @type {RollbridgeDaemon | undefined} */
|
|
264
|
+
let recovered
|
|
265
|
+
const v1Path = await prepareRelease(fixture.root, "v1")
|
|
266
|
+
const v2Path = await prepareRelease(fixture.root, "v2")
|
|
267
|
+
const wrongPath = await prepareRelease(fixture.root, "wrong")
|
|
268
|
+
|
|
269
|
+
try {
|
|
270
|
+
await retired.start()
|
|
271
|
+
await retired.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
|
|
272
|
+
await retired.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
273
|
+
const committed = retired.status()
|
|
274
|
+
const v1WorkerPid = releaseProcessPid(committed, "v1", "worker")
|
|
275
|
+
const retiringPids = [
|
|
276
|
+
...(committed.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid) || []),
|
|
277
|
+
...committed.services.map(({process}) => process.pid)
|
|
278
|
+
].filter((pid) => typeof pid === "number")
|
|
279
|
+
|
|
280
|
+
await retired.retireOwner({attestation: `sha256:${"c".repeat(64)}`})
|
|
281
|
+
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
282
|
+
await Promise.all(retiringPids.map((pid) => waitForProcessExit(pid, 3000)))
|
|
283
|
+
recovered = new RollbridgeDaemon({
|
|
284
|
+
bootstrap: {releaseId: "v2", releasePath: v2Path, revision: "v2"},
|
|
285
|
+
config,
|
|
286
|
+
configPath: fixture.configPath,
|
|
287
|
+
logger: () => {}
|
|
288
|
+
})
|
|
289
|
+
await recovered.start({exposeControl: false})
|
|
290
|
+
const owner = recovered
|
|
291
|
+
|
|
292
|
+
await assert.rejects(
|
|
293
|
+
() => owner.deploy({releaseId: "v2", releasePath: wrongPath, revision: "v2"}),
|
|
294
|
+
/only the exact same release, path, revision, and config authority/u
|
|
295
|
+
)
|
|
296
|
+
await assert.rejects(
|
|
297
|
+
() => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "wrong"}),
|
|
298
|
+
/only the exact same release, path, revision, and config authority/u
|
|
299
|
+
)
|
|
300
|
+
const changedConfig = structuredClone(fixture.config)
|
|
301
|
+
const jobs = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (changedConfig.processes)
|
|
302
|
+
.find((processConfig) => processConfig.id === "jobs")
|
|
303
|
+
|
|
304
|
+
assert.ok(jobs)
|
|
305
|
+
jobs.env = {.../** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobs.env), MISMATCHED_AUTHORITY: "true"}
|
|
306
|
+
await writeConfig(fixture.configPath, changedConfig)
|
|
307
|
+
await assert.rejects(
|
|
308
|
+
() => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"}),
|
|
309
|
+
/only the exact same release, path, revision, and config authority/u
|
|
310
|
+
)
|
|
311
|
+
await writeConfig(fixture.configPath, fixture.config)
|
|
312
|
+
await assert.rejects(
|
|
313
|
+
() => owner.deploy({releaseId: "wrong", releasePath: wrongPath, revision: "wrong"}),
|
|
314
|
+
/only the exact same release, path, revision, and config authority/u
|
|
315
|
+
)
|
|
316
|
+
const preserved = owner.status()
|
|
317
|
+
|
|
318
|
+
assert.equal(preserved.activeReleaseId, null)
|
|
319
|
+
assert.equal(preserved.generationTransition?.candidateReleaseId, "v2")
|
|
320
|
+
assert.equal(preserved.generationTransition?.phase, "committed")
|
|
321
|
+
assert.equal(preserved.releases.find(({releaseId}) => releaseId === "v2")?.state, "draining")
|
|
322
|
+
assert.equal(releaseProcessPid(preserved, "v1", "worker"), v1WorkerPid)
|
|
323
|
+
assert.equal(isAlive(v1WorkerPid), true)
|
|
324
|
+
assert.equal(preserved.releases.some(({releaseId}) => releaseId === "wrong"), false)
|
|
325
|
+
|
|
326
|
+
await owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
327
|
+
await Promise.all([
|
|
328
|
+
owner.stopRelease("v2"),
|
|
329
|
+
fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
330
|
+
])
|
|
331
|
+
const afterIntentionalStop = await owner.deploy({releaseId: "wrong", releasePath: wrongPath, revision: "wrong"})
|
|
332
|
+
|
|
333
|
+
assert.equal(afterIntentionalStop.activeReleaseId, "wrong")
|
|
334
|
+
assert.equal(owner.status().activeReleaseId, "wrong")
|
|
335
|
+
} finally {
|
|
336
|
+
if (recovered) {
|
|
337
|
+
const activeReleaseId = recovered.status().activeReleaseId
|
|
338
|
+
const shutdown = recovered.shutdown()
|
|
339
|
+
|
|
340
|
+
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
341
|
+
if (activeReleaseId === "v2") await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
342
|
+
if (activeReleaseId === "wrong") await fs.writeFile(path.join(wrongPath, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
343
|
+
await shutdown.catch(() => undefined)
|
|
344
|
+
}
|
|
345
|
+
retired.guardian?.disconnect()
|
|
346
|
+
await stopFixtureGuardian(fixture.statePath)
|
|
347
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
348
|
+
}
|
|
349
|
+
})
|
|
350
|
+
|
|
98
351
|
test("replacement owner reconstructs one active and two draining generations after abrupt daemon exit", async () => {
|
|
99
352
|
const fixture = await createFixture()
|
|
100
353
|
let owner = spawnDaemon(fixture.configPath)
|