rollbridge 0.1.34 → 0.1.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +4 -3
- package/package.json +1 -1
- package/src/daemon.js +23 -7
- package/src/guardian-client.js +33 -0
- package/src/process-guardian.js +19 -7
- package/test/guardian-client.test.js +70 -1
- package/test/owner-replacement.test.js +164 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
### Fixed
|
|
2
2
|
|
|
3
|
-
-
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
- Select retired-owner commit proof from the guardian-published committed owner
|
|
4
|
+
snapshot rather than candidate-local process order, reserve its incumbent-owned
|
|
5
|
+
registration through commit, then attach it to the committed candidate while
|
|
6
|
+
retaining the replacement transaction, authority, and control-path fences.
|
|
6
7
|
- Fail closed when an older retained guardian cannot commit that replacement
|
|
7
8
|
atomically after the incumbent control socket disappears, preserving the
|
|
8
9
|
incumbent owner, retained connections, and guardian-managed release processes.
|
package/package.json
CHANGED
package/src/daemon.js
CHANGED
|
@@ -342,6 +342,11 @@ export default class RollbridgeDaemon {
|
|
|
342
342
|
const transfer = /** @type {PrivateOwnerState} */ (prepared.ownerState)
|
|
343
343
|
|
|
344
344
|
if (!transfer?.config || !transfer.snapshot) throw new Error("Committed owner published incomplete replacement state")
|
|
345
|
+
const registeredProcesses = new Map((await this.guardian.inventory()).map(({key, provenance}) => [key, provenance]))
|
|
346
|
+
const reservedProcessKey = legacyBridge ? undefined : ownerSnapshotProcessKeys(transfer.snapshot, transfer.singletonReleaseIds)
|
|
347
|
+
.find((key) => registeredProcesses.has(key))
|
|
348
|
+
|
|
349
|
+
if (reservedProcessKey) this.guardian.reserveProcessRecovery(reservedProcessKey, /** @type {string} */ (registeredProcesses.get(reservedProcessKey)))
|
|
345
350
|
await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, releaseConfigs: transfer.releaseConfigs, resumeDrains: false, singletonReleaseIds: transfer.singletonReleaseIds, synchronizeLifecycleRoles: false})
|
|
346
351
|
for (const release of this.releases.values()) release.preserveConfigOnRetirement = true
|
|
347
352
|
this.logger("owner replacement candidate prepared", {activeReleaseId: this.activeRelease?.releaseId ?? null, replacementId: prepared.replacementId})
|
|
@@ -386,6 +391,7 @@ export default class RollbridgeDaemon {
|
|
|
386
391
|
if (incumbentControl) {
|
|
387
392
|
const listenerSession = incumbentControl
|
|
388
393
|
|
|
394
|
+
if (reservedProcessKey) await this.guardian.recoverReservedProcess(reservedProcessKey)
|
|
389
395
|
incumbentControl.onEvent((event) => this.handleIncumbentListenerEvent(event, listenerSession))
|
|
390
396
|
await incumbentControl.request({
|
|
391
397
|
command: "yield-owner-listeners",
|
|
@@ -414,13 +420,20 @@ export default class RollbridgeDaemon {
|
|
|
414
420
|
snapshot: this.status()
|
|
415
421
|
})
|
|
416
422
|
|
|
423
|
+
if (staged.committed && reservedProcessKey) {
|
|
424
|
+
committedAuthority = true
|
|
425
|
+
await this.guardian.recoverReservedProcess(reservedProcessKey)
|
|
426
|
+
}
|
|
427
|
+
|
|
417
428
|
if (!staged.committed) {
|
|
418
429
|
if (retiredIncumbentControl) {
|
|
419
|
-
const processKey =
|
|
430
|
+
const processKey = reservedProcessKey
|
|
420
431
|
|
|
421
|
-
if (!processKey) throw new Error("Retired owner replacement requires an exact
|
|
432
|
+
if (!processKey || !this.guardian.processes.has(processKey)) throw new Error("Retired owner replacement requires an exact reserved process from committed owner state")
|
|
422
433
|
try {
|
|
423
434
|
await this.guardian.commitRetiredOwnerReplacement(prepared.replacementId, processKey)
|
|
435
|
+
committedAuthority = true
|
|
436
|
+
await this.guardian.recoverReservedProcess(processKey)
|
|
424
437
|
} catch (error) {
|
|
425
438
|
if (!(error instanceof Error) || error.message !== "Guardian commit-retired-owner-replacement requires the committed owner") throw error
|
|
426
439
|
throw new Error(
|
|
@@ -544,7 +557,7 @@ export default class RollbridgeDaemon {
|
|
|
544
557
|
*/
|
|
545
558
|
async prepareLegacyOwnerReplacement({error, persisted, persistedAuthority}) {
|
|
546
559
|
const diagnostic = error instanceof Error ? error.message : String(error)
|
|
547
|
-
const legacyProcessKey =
|
|
560
|
+
const legacyProcessKey = ownerSnapshotProcessKeys(persisted, persisted.singletonReleaseIds)[0]
|
|
548
561
|
|
|
549
562
|
if (!isLegacyGuardianPrepareDiagnostic(diagnostic)) throw error
|
|
550
563
|
if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state", {cause: error})
|
|
@@ -1996,18 +2009,21 @@ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
|
|
|
1996
2009
|
}
|
|
1997
2010
|
|
|
1998
2011
|
/**
|
|
1999
|
-
* @param {OwnerRecoverySnapshot} snapshot - Durable
|
|
2012
|
+
* @param {OwnerRecoverySnapshot} snapshot - Durable committed owner snapshot.
|
|
2013
|
+
* @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
|
|
2000
2014
|
* @returns {string[]} Exact guardian registration keys present in the snapshot.
|
|
2001
2015
|
*/
|
|
2002
|
-
function
|
|
2016
|
+
function ownerSnapshotProcessKeys(snapshot, singletonReleaseIds = {}) {
|
|
2003
2017
|
const keys = []
|
|
2004
2018
|
|
|
2005
2019
|
for (const release of snapshot.releases) {
|
|
2006
2020
|
for (const processStatus of release.processes) keys.push(`release:${release.releaseId}:${processStatus.id}`)
|
|
2007
2021
|
}
|
|
2008
2022
|
for (const service of snapshot.services) keys.push(`service:${service.id}`)
|
|
2009
|
-
|
|
2010
|
-
|
|
2023
|
+
for (const singleton of snapshot.singletons) {
|
|
2024
|
+
const releaseId = singletonReleaseIds[singleton.id] || snapshot.activeReleaseId
|
|
2025
|
+
|
|
2026
|
+
if (releaseId) keys.push(`singleton:${releaseId}:${singleton.id}`)
|
|
2011
2027
|
}
|
|
2012
2028
|
return keys
|
|
2013
2029
|
}
|
package/src/guardian-client.js
CHANGED
|
@@ -21,6 +21,8 @@ export default class GuardianClient {
|
|
|
21
21
|
this.idleWaiters = /** @type {(() => void)[]} */ ([])
|
|
22
22
|
this.guardianExitPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
23
23
|
this.processes = /** @type {Map<string, GuardianProcess>} */ (new Map())
|
|
24
|
+
this.reservedProcessKey = /** @type {string | undefined} */ (undefined)
|
|
25
|
+
this.reservedProcessProvenance = /** @type {string | undefined} */ (undefined)
|
|
24
26
|
this.events = /** @type {Map<string, {reject: (error: Error) => void, resolve: (value: Record<string, import("./json.js").JsonValue>) => void}[]>} */ (new Map())
|
|
25
27
|
this.eventHandlers = /** @type {Map<string, ((event: Record<string, import("./json.js").JsonValue>) => void)[]>} */ (new Map())
|
|
26
28
|
}
|
|
@@ -111,6 +113,27 @@ export default class GuardianClient {
|
|
|
111
113
|
return processInstance
|
|
112
114
|
}
|
|
113
115
|
|
|
116
|
+
/**
|
|
117
|
+
* @param {string} key - Exact committed-owner registration reserved until replacement commit.
|
|
118
|
+
* @param {string} provenance - Guardian-inventoried definition fence.
|
|
119
|
+
*/
|
|
120
|
+
reserveProcessRecovery(key, provenance) {
|
|
121
|
+
if (this.reservedProcessKey) throw new Error(`Guardian process recovery ${this.reservedProcessKey} is already reserved`)
|
|
122
|
+
this.reservedProcessKey = key
|
|
123
|
+
this.reservedProcessProvenance = provenance
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** @param {string} key - Exact reserved registration to attach after authority commits. */
|
|
127
|
+
async recoverReservedProcess(key) {
|
|
128
|
+
if (this.reservedProcessKey !== key) throw new Error(`Guardian process recovery ${key} is not reserved`)
|
|
129
|
+
const processInstance = this.processes.get(key)
|
|
130
|
+
|
|
131
|
+
if (!processInstance) throw new Error(`Reserved guardian process ${key} was not reconstructed`)
|
|
132
|
+
await processInstance.attachReserved()
|
|
133
|
+
this.reservedProcessKey = undefined
|
|
134
|
+
this.reservedProcessProvenance = undefined
|
|
135
|
+
}
|
|
136
|
+
|
|
114
137
|
/**
|
|
115
138
|
* @param {Record<string, import("./json.js").JsonValue>} command - Command.
|
|
116
139
|
* @returns {Promise<import("./json.js").JsonValue>} Guardian response.
|
|
@@ -354,6 +377,16 @@ class GuardianProcess extends ManagedProcess {
|
|
|
354
377
|
|
|
355
378
|
/** Reconnects to an already registered guardian process without changing its desired state. */
|
|
356
379
|
async recover() {
|
|
380
|
+
if (this.client.reservedProcessKey === this.key) {
|
|
381
|
+
if (this.client.reservedProcessProvenance !== this.provenance) throw new Error(`Guardian provenance mismatch for reserved process ${this.key}`)
|
|
382
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "status", key: this.key}))
|
|
383
|
+
return
|
|
384
|
+
}
|
|
385
|
+
await this.ensureRegistered()
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/** Attaches a reconstructed process whose incumbent-owned registration was reserved through commit. */
|
|
389
|
+
async attachReserved() {
|
|
357
390
|
await this.ensureRegistered()
|
|
358
391
|
}
|
|
359
392
|
|
package/src/process-guardian.js
CHANGED
|
@@ -68,7 +68,7 @@ let ownerState = bootstrap.ownerState
|
|
|
68
68
|
let ownerRevision = ownerState === undefined ? 0 : 1
|
|
69
69
|
/** @type {number | undefined} */
|
|
70
70
|
let replacementRevision
|
|
71
|
-
const legacyKeys = legacyGuardian ?
|
|
71
|
+
const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new Set()
|
|
72
72
|
/** @type {net.Socket | undefined} */
|
|
73
73
|
let ownerMutationClient
|
|
74
74
|
/** @type {string | undefined} */
|
|
@@ -301,6 +301,9 @@ async function execute(request, socket) {
|
|
|
301
301
|
if (request.command === "commit-retired-owner-replacement") {
|
|
302
302
|
requireReplacement(socket, request)
|
|
303
303
|
requireProcess(request)
|
|
304
|
+
if (!committedOwnerProcessKeys(ownerState).has(/** @type {string} */ (request.key))) {
|
|
305
|
+
throw new Error(`Guardian process ${request.key} does not belong to the committed owner`)
|
|
306
|
+
}
|
|
304
307
|
if (!replacementOwnerState) throw new Error("Retired owner replacement transaction is not staged")
|
|
305
308
|
if (!isDeepStrictEqual(ownerAuthority(ownerState), replacementAuthority)) throw new Error("Retired owner replacement requires unchanged owner authority")
|
|
306
309
|
const controlPath = ownerControlPath(ownerState)
|
|
@@ -622,11 +625,11 @@ function errorMessage(error) {
|
|
|
622
625
|
}
|
|
623
626
|
|
|
624
627
|
/**
|
|
625
|
-
* Derives only exact process keys
|
|
626
|
-
* @param {import("./json.js").JsonValue | undefined} state -
|
|
627
|
-
* @returns {Set<string>} Exact
|
|
628
|
+
* Derives only exact process keys serialized by committed private owner state.
|
|
629
|
+
* @param {import("./json.js").JsonValue | undefined} state - Committed owner state.
|
|
630
|
+
* @returns {Set<string>} Exact committed registrations eligible as owner proof.
|
|
628
631
|
*/
|
|
629
|
-
function
|
|
632
|
+
function committedOwnerProcessKeys(state) {
|
|
630
633
|
const keys = new Set()
|
|
631
634
|
|
|
632
635
|
if (!state || typeof state !== "object" || Array.isArray(state) || !("snapshot" in state)) return keys
|
|
@@ -646,9 +649,18 @@ function legacyOwnerKeys(state) {
|
|
|
646
649
|
if (service && typeof service === "object" && !Array.isArray(service) && typeof service.id === "string") keys.add(`service:${service.id}`)
|
|
647
650
|
}
|
|
648
651
|
}
|
|
649
|
-
|
|
652
|
+
const singletonReleaseIds = "singletonReleaseIds" in state && state.singletonReleaseIds && typeof state.singletonReleaseIds === "object" && !Array.isArray(state.singletonReleaseIds)
|
|
653
|
+
? state.singletonReleaseIds
|
|
654
|
+
: {}
|
|
655
|
+
|
|
656
|
+
if (Array.isArray(snapshot.singletons)) {
|
|
650
657
|
for (const singleton of snapshot.singletons) {
|
|
651
|
-
if (singleton
|
|
658
|
+
if (!singleton || typeof singleton !== "object" || Array.isArray(singleton) || typeof singleton.id !== "string") continue
|
|
659
|
+
const releaseId = singleton.id in singletonReleaseIds && typeof singletonReleaseIds[singleton.id] === "string"
|
|
660
|
+
? singletonReleaseIds[singleton.id]
|
|
661
|
+
: snapshot.activeReleaseId
|
|
662
|
+
|
|
663
|
+
if (typeof releaseId === "string") keys.add(`singleton:${releaseId}:${singleton.id}`)
|
|
652
664
|
}
|
|
653
665
|
}
|
|
654
666
|
return keys
|
|
@@ -224,6 +224,65 @@ test("retired owner replacement commit carries its exact recovered process key",
|
|
|
224
224
|
await client.commitRetiredOwnerReplacement(replacementId, processKey)
|
|
225
225
|
})
|
|
226
226
|
|
|
227
|
+
test("reserved process recovery rejects a reconstructed definition with different provenance", async () => {
|
|
228
|
+
const fixture = await createGuardian()
|
|
229
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
230
|
+
const processKey = "release:v1:worker"
|
|
231
|
+
|
|
232
|
+
try {
|
|
233
|
+
await fixture.client.process(processKey, definition("worker")).recover()
|
|
234
|
+
const [registration] = await fixture.client.inventory()
|
|
235
|
+
|
|
236
|
+
assert.ok(registration)
|
|
237
|
+
await candidate.connect()
|
|
238
|
+
candidate.reserveProcessRecovery(processKey, registration.provenance)
|
|
239
|
+
await assert.rejects(
|
|
240
|
+
() => candidate.process(processKey, definition("different-worker")).recover(),
|
|
241
|
+
/provenance mismatch for reserved process/
|
|
242
|
+
)
|
|
243
|
+
assert.deepEqual((await fixture.client.inventory()).map(({key}) => key), [processKey])
|
|
244
|
+
} finally {
|
|
245
|
+
candidate.disconnect()
|
|
246
|
+
await cleanupGuardian(fixture)
|
|
247
|
+
}
|
|
248
|
+
})
|
|
249
|
+
|
|
250
|
+
test("retired owner replacement rejects a registered process absent from committed owner state", async () => {
|
|
251
|
+
const fixture = await createGuardian()
|
|
252
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
253
|
+
const committedProcessKey = "release:v1:worker"
|
|
254
|
+
const candidateProcessKey = "release:candidate:worker"
|
|
255
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
256
|
+
const snapshot = {
|
|
257
|
+
activeReleaseId: "v1",
|
|
258
|
+
control: {path: path.join(fixture.root, "rollbridge.sock")},
|
|
259
|
+
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
260
|
+
services: [],
|
|
261
|
+
singletons: []
|
|
262
|
+
}
|
|
263
|
+
const candidateSnapshot = {
|
|
264
|
+
...snapshot,
|
|
265
|
+
releases: [...snapshot.releases, {processes: [{id: "worker"}], releaseId: "candidate"}]
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
try {
|
|
269
|
+
await fixture.client.process(committedProcessKey, definition("worker")).recover()
|
|
270
|
+
await fixture.client.process(candidateProcessKey, definition("candidate-worker")).recover()
|
|
271
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
272
|
+
await candidate.connect()
|
|
273
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
274
|
+
|
|
275
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot: candidateSnapshot})
|
|
276
|
+
await assert.rejects(
|
|
277
|
+
() => candidate.commitRetiredOwnerReplacement(prepared.replacementId, candidateProcessKey),
|
|
278
|
+
/process .* does not belong to the committed owner/
|
|
279
|
+
)
|
|
280
|
+
} finally {
|
|
281
|
+
candidate.disconnect()
|
|
282
|
+
await cleanupGuardian(fixture)
|
|
283
|
+
}
|
|
284
|
+
})
|
|
285
|
+
|
|
227
286
|
test("retired owner replacement requires unchanged authority and the exact control path absent", async () => {
|
|
228
287
|
const fixture = await createGuardian()
|
|
229
288
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
@@ -232,7 +291,13 @@ test("retired owner replacement requires unchanged authority and the exact contr
|
|
|
232
291
|
const processKey = "release:v1:worker"
|
|
233
292
|
const authority = {configDigest: "incumbent", runtime: null}
|
|
234
293
|
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
235
|
-
const snapshot = {
|
|
294
|
+
const snapshot = {
|
|
295
|
+
activeReleaseId: "v1",
|
|
296
|
+
control: {path: controlPath},
|
|
297
|
+
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
298
|
+
services: [],
|
|
299
|
+
singletons: []
|
|
300
|
+
}
|
|
236
301
|
|
|
237
302
|
try {
|
|
238
303
|
await fixture.client.process(processKey, definition("worker")).recover()
|
|
@@ -260,6 +325,10 @@ test("retired owner replacement requires unchanged authority and the exact contr
|
|
|
260
325
|
() => contender.request({command: "commit-retired-owner-replacement", key: processKey, replacementId: ready.replacementId}),
|
|
261
326
|
/not the prepared candidate/
|
|
262
327
|
)
|
|
328
|
+
await assert.rejects(
|
|
329
|
+
() => candidate.commitRetiredOwnerReplacement("stale-replacement", processKey),
|
|
330
|
+
/not the prepared candidate/
|
|
331
|
+
)
|
|
263
332
|
await assert.rejects(
|
|
264
333
|
() => candidate.commitRetiredOwnerReplacement(ready.replacementId, "release:v1:wrong"),
|
|
265
334
|
/process .* is not registered/
|
|
@@ -259,6 +259,7 @@ test("cross-version replacement fails closed without dropping a retained WebSock
|
|
|
259
259
|
const compatibilitySockets = new Set()
|
|
260
260
|
let compatibilityGuardian
|
|
261
261
|
let committedProcessKey
|
|
262
|
+
let retainedGuardianSocketPath = /** @type {string | undefined} */ (undefined)
|
|
262
263
|
let owner = /** @type {import("node:child_process").ChildProcess | undefined} */ (undefined)
|
|
263
264
|
let replacement = /** @type {RollbridgeDaemon | undefined} */ (undefined)
|
|
264
265
|
let retainedConnection
|
|
@@ -287,6 +288,9 @@ test("cross-version replacement fails closed without dropping a retained WebSock
|
|
|
287
288
|
const guardianSocketPath = state.recovery.guardian.socketPath
|
|
288
289
|
const expectedProcessKey = "release:v1:worker"
|
|
289
290
|
|
|
291
|
+
if (typeof guardianSocketPath !== "string") throw new Error("Retained guardian state is missing its socket path")
|
|
292
|
+
retainedGuardianSocketPath = guardianSocketPath
|
|
293
|
+
|
|
290
294
|
compatibilityGuardian = net.createServer((candidateSocket) => {
|
|
291
295
|
const guardianSocket = net.createConnection(guardianSocketPath)
|
|
292
296
|
let buffer = ""
|
|
@@ -378,6 +382,166 @@ test("cross-version replacement fails closed without dropping a retained WebSock
|
|
|
378
382
|
replacement?.guardian?.disconnect()
|
|
379
383
|
for (const socket of compatibilitySockets) socket.destroy()
|
|
380
384
|
if (compatibilityGuardian?.listening) await closeServer(compatibilityGuardian)
|
|
385
|
+
if (retainedGuardianSocketPath) {
|
|
386
|
+
const cleanupState = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
387
|
+
|
|
388
|
+
cleanupState.recovery.guardian.socketPath = retainedGuardianSocketPath
|
|
389
|
+
await fs.writeFile(statePath, `${JSON.stringify(cleanupState)}\n`)
|
|
390
|
+
}
|
|
391
|
+
await stopGuardian(statePath)
|
|
392
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
393
|
+
}
|
|
394
|
+
})
|
|
395
|
+
|
|
396
|
+
test("cross-version replacement preserves committed-owner proof until commit then recovers every process", async () => {
|
|
397
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-committed-proof-"))
|
|
398
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
399
|
+
const statePath = path.join(root, "state.json")
|
|
400
|
+
const compatibilitySocketPath = path.join(root, "retained-guardian.sock")
|
|
401
|
+
const releasePath = path.join(root, "v1")
|
|
402
|
+
const daemonConfig = normalizeConfig(config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
403
|
+
const owner = new RollbridgeDaemon({config: daemonConfig, logger: () => {}})
|
|
404
|
+
const compatibilitySockets = new Set()
|
|
405
|
+
const candidateProcessKey = "release:candidate:worker"
|
|
406
|
+
const committedOwnerProcessKey = "release:v1:worker"
|
|
407
|
+
const candidateRecoveredKeys = new Set()
|
|
408
|
+
let compatibilityGuardian
|
|
409
|
+
let committedProcessKey
|
|
410
|
+
let recoveredKeysAtCommit = /** @type {Set<string> | undefined} */ (undefined)
|
|
411
|
+
let retainedConnection
|
|
412
|
+
let retainedConnectionClosed = false
|
|
413
|
+
let retainedGuardianSocketPath = /** @type {string | undefined} */ (undefined)
|
|
414
|
+
/** @type {RollbridgeDaemon | undefined} */
|
|
415
|
+
let replacement
|
|
416
|
+
|
|
417
|
+
try {
|
|
418
|
+
await fs.mkdir(releasePath)
|
|
419
|
+
await makeFifo(path.join(releasePath, "worker.fifo"))
|
|
420
|
+
await owner.start()
|
|
421
|
+
await owner.deploy({releaseId: "v1", releasePath, revision: "v1"})
|
|
422
|
+
const ownerProcess = owner.guardian?.processes.values().next().value
|
|
423
|
+
|
|
424
|
+
assert.ok(owner.guardian)
|
|
425
|
+
assert.ok(ownerProcess)
|
|
426
|
+
await owner.guardian.request({
|
|
427
|
+
command: "register",
|
|
428
|
+
definition: ownerProcess.definition,
|
|
429
|
+
key: candidateProcessKey,
|
|
430
|
+
provenance: ownerProcess.provenance
|
|
431
|
+
})
|
|
432
|
+
const running = owner.status()
|
|
433
|
+
const processState = running.releases[0]?.processes.map(({id, pid, state}) => ({id, pid, state}))
|
|
434
|
+
const expectedProcessKeys = new Set(running.releases[0]?.processes.map(({id}) => `release:v1:${id}`))
|
|
435
|
+
const proxyPort = /** @type {{port?: number}} */ (running.proxy).port
|
|
436
|
+
|
|
437
|
+
assert.deepEqual(processState?.map(({state}) => state), ["running", "running"])
|
|
438
|
+
if (typeof proxyPort !== "number") throw new Error("Retained owner proxy is missing its port")
|
|
439
|
+
retainedConnection = await openWebSocket(proxyPort)
|
|
440
|
+
retainedConnection.once("close", () => { retainedConnectionClosed = true })
|
|
441
|
+
await owner.closeServer(owner.controlServer)
|
|
442
|
+
await owner.removeControlSocket()
|
|
443
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
444
|
+
const guardianSocketPath = state.recovery.guardian.socketPath
|
|
445
|
+
|
|
446
|
+
if (typeof guardianSocketPath !== "string") throw new Error("Retained guardian state is missing its socket path")
|
|
447
|
+
retainedGuardianSocketPath = guardianSocketPath
|
|
448
|
+
compatibilityGuardian = net.createServer((candidateSocket) => {
|
|
449
|
+
const guardianSocket = net.createConnection(guardianSocketPath)
|
|
450
|
+
let buffer = ""
|
|
451
|
+
|
|
452
|
+
compatibilitySockets.add(candidateSocket)
|
|
453
|
+
compatibilitySockets.add(guardianSocket)
|
|
454
|
+
candidateSocket.setEncoding("utf8")
|
|
455
|
+
candidateSocket.once("close", () => {
|
|
456
|
+
compatibilitySockets.delete(candidateSocket)
|
|
457
|
+
guardianSocket.destroy()
|
|
458
|
+
})
|
|
459
|
+
guardianSocket.once("close", () => {
|
|
460
|
+
compatibilitySockets.delete(guardianSocket)
|
|
461
|
+
candidateSocket.destroy()
|
|
462
|
+
})
|
|
463
|
+
guardianSocket.on("data", (chunk) => candidateSocket.write(chunk))
|
|
464
|
+
candidateSocket.on("data", (chunk) => {
|
|
465
|
+
buffer += chunk
|
|
466
|
+
let newline = buffer.indexOf("\n")
|
|
467
|
+
|
|
468
|
+
while (newline >= 0) {
|
|
469
|
+
const line = buffer.slice(0, newline)
|
|
470
|
+
const request = JSON.parse(line)
|
|
471
|
+
|
|
472
|
+
buffer = buffer.slice(newline + 1)
|
|
473
|
+
if (request.command === "register") candidateRecoveredKeys.add(request.key)
|
|
474
|
+
if (request.command === "commit-retired-owner-replacement") {
|
|
475
|
+
committedProcessKey = request.key
|
|
476
|
+
recoveredKeysAtCommit = new Set(candidateRecoveredKeys)
|
|
477
|
+
if (candidateRecoveredKeys.has(request.key)) {
|
|
478
|
+
candidateSocket.write(`${JSON.stringify({error: `Guardian ${request.command} requires the committed owner`, id: request.id})}\n`)
|
|
479
|
+
newline = buffer.indexOf("\n")
|
|
480
|
+
continue
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
guardianSocket.write(`${line}\n`)
|
|
484
|
+
newline = buffer.indexOf("\n")
|
|
485
|
+
}
|
|
486
|
+
})
|
|
487
|
+
})
|
|
488
|
+
await listenUnix(compatibilityGuardian, compatibilitySocketPath)
|
|
489
|
+
state.recovery.guardian.socketPath = compatibilitySocketPath
|
|
490
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
491
|
+
|
|
492
|
+
assert.equal((await owner.guardian?.replacementStatus())?.ownerClaimed, true)
|
|
493
|
+
await assert.rejects(fs.access(socketPath), {code: "ENOENT"})
|
|
494
|
+
|
|
495
|
+
replacement = new RollbridgeDaemon({
|
|
496
|
+
config: daemonConfig,
|
|
497
|
+
logger: (message) => {
|
|
498
|
+
if (message !== "owner replacement candidate prepared" || !replacement?.guardian) return
|
|
499
|
+
const guardian = replacement.guardian
|
|
500
|
+
const recoveredProcess = guardian.processes.values().next().value
|
|
501
|
+
|
|
502
|
+
if (!recoveredProcess) throw new Error("Replacement did not reconstruct a guardian process")
|
|
503
|
+
guardian.processes = new Map([[candidateProcessKey, recoveredProcess], ...guardian.processes])
|
|
504
|
+
const commitRetiredOwnerReplacement = guardian.commitRetiredOwnerReplacement.bind(guardian)
|
|
505
|
+
|
|
506
|
+
guardian.commitRetiredOwnerReplacement = async (replacementId, processKey) => {
|
|
507
|
+
committedProcessKey = processKey
|
|
508
|
+
await commitRetiredOwnerReplacement(replacementId, processKey)
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
})
|
|
512
|
+
await replacement.replaceIncompatibleOwner()
|
|
513
|
+
const recovered = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
514
|
+
const recoveredReleases = /** @type {{processes: {id: string, pid?: number, state: string}[]}[]} */ (recovered.releases)
|
|
515
|
+
|
|
516
|
+
assert.equal(recovered.activeReleaseId, "v1")
|
|
517
|
+
assert.equal(committedProcessKey, committedOwnerProcessKey)
|
|
518
|
+
assert.equal(recoveredKeysAtCommit?.has(committedOwnerProcessKey), false)
|
|
519
|
+
assert.deepEqual(candidateRecoveredKeys, expectedProcessKeys)
|
|
520
|
+
assert.equal([...replacement.guardian?.processes.keys() || []][0], candidateProcessKey)
|
|
521
|
+
assert.equal(retainedConnectionClosed, false, "successful compatibility handoff must preserve retained connections")
|
|
522
|
+
assert.equal(retainedConnection.destroyed, false, "successful compatibility handoff must leave the retained listener serving")
|
|
523
|
+
assert.deepEqual(recovered.releaseReferences, [{releaseId: "v1", releasePath}])
|
|
524
|
+
assert.deepEqual(recoveredReleases[0]?.processes.map(({id, pid, state}) => ({id, pid, state})), processState)
|
|
525
|
+
for (const {pid} of processState || []) {
|
|
526
|
+
if (typeof pid !== "number") throw new Error("Retained process is missing its PID")
|
|
527
|
+
assert.doesNotThrow(() => process.kill(pid, 0))
|
|
528
|
+
}
|
|
529
|
+
} finally {
|
|
530
|
+
if (retainedGuardianSocketPath) {
|
|
531
|
+
const cleanupState = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
532
|
+
|
|
533
|
+
cleanupState.recovery.guardian.socketPath = retainedGuardianSocketPath
|
|
534
|
+
await fs.writeFile(statePath, `${JSON.stringify(cleanupState)}\n`)
|
|
535
|
+
}
|
|
536
|
+
const shutdown = replacement?.controlCommandsReady ? replacement.shutdown().catch(() => {}) : owner.shutdown().catch(() => {})
|
|
537
|
+
|
|
538
|
+
retainedConnection?.destroy()
|
|
539
|
+
await owner.closeServer(owner.proxyServer)
|
|
540
|
+
await Promise.all([fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n").catch(() => {}), shutdown])
|
|
541
|
+
owner.guardian?.disconnect()
|
|
542
|
+
replacement?.guardian?.disconnect()
|
|
543
|
+
for (const socket of compatibilitySockets) socket.destroy()
|
|
544
|
+
if (compatibilityGuardian?.listening) await closeServer(compatibilityGuardian)
|
|
381
545
|
await stopGuardian(statePath)
|
|
382
546
|
await fs.rm(root, {force: true, recursive: true})
|
|
383
547
|
}
|