rollbridge 0.1.35 → 0.1.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +10 -3
- package/docs/cli.md +2 -1
- package/docs/config.md +4 -2
- package/docs/troubleshooting.md +3 -0
- package/package.json +1 -1
- package/src/daemon.js +226 -60
- package/src/guardian-client.js +99 -1
- package/src/process-guardian.js +31 -6
- package/test/fixtures/partial-owner-replacement-process-guardian.js +106 -0
- package/test/guardian-client.test.js +97 -0
- package/test/owner-replacement.test.js +604 -8
|
@@ -19,6 +19,476 @@ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."
|
|
|
19
19
|
const binPath = path.join(repoRoot, "bin", "rollbridge")
|
|
20
20
|
const dummyAppPath = path.join(repoRoot, "test", "fixtures", "dummy-app.js")
|
|
21
21
|
const legacyDaemonPath = path.join(repoRoot, "test", "fixtures", "pre-split3-daemon-runner.js")
|
|
22
|
+
const partialGuardianPath = path.join(repoRoot, "test", "fixtures", "partial-owner-replacement-process-guardian.js")
|
|
23
|
+
|
|
24
|
+
test("partial owner-replacement guardian crosses the authenticated legacy bridge before retired-owner commit", async () => {
|
|
25
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-"))
|
|
26
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
27
|
+
const statePath = path.join(root, "state.json")
|
|
28
|
+
const configPath = path.join(root, "rollbridge.cjs")
|
|
29
|
+
const daemonPidPath = path.join(root, "daemon.pid")
|
|
30
|
+
const releasePath = path.join(root, "v1")
|
|
31
|
+
const packagePath = path.join(root, "candidate-package")
|
|
32
|
+
const runtimePath = path.join(root, "runtime")
|
|
33
|
+
const partialSocketPath = path.join(root, "partial-guardian.sock")
|
|
34
|
+
let owner
|
|
35
|
+
let partialGuardian
|
|
36
|
+
let retainedConnection
|
|
37
|
+
let backendGuardianIdentity
|
|
38
|
+
|
|
39
|
+
try {
|
|
40
|
+
await fs.mkdir(releasePath)
|
|
41
|
+
await makeFifo(path.join(releasePath, "worker.fifo"))
|
|
42
|
+
await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
43
|
+
owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
|
|
44
|
+
await waitForLog(owner, "control socket listening")
|
|
45
|
+
assert.ok(owner.pid)
|
|
46
|
+
await fs.writeFile(daemonPidPath, `${owner.pid}\n`)
|
|
47
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: socketPath})
|
|
48
|
+
const before = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
49
|
+
const workerPid = releaseProcessPid(before, "v1", "worker")
|
|
50
|
+
const webPid = releaseProcessPid(before, "v1", "web")
|
|
51
|
+
const proxyPort = /** @type {{port?: number}} */ (before.proxy).port
|
|
52
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
53
|
+
|
|
54
|
+
if (typeof proxyPort !== "number") throw new Error("Partial guardian fixture proxy is missing its bound port")
|
|
55
|
+
|
|
56
|
+
backendGuardianIdentity = {...state.recovery.guardian}
|
|
57
|
+
partialGuardian = await startPartialGuardian({
|
|
58
|
+
backendPath: state.recovery.guardian.socketPath,
|
|
59
|
+
mode: "partial",
|
|
60
|
+
socketPath: partialSocketPath,
|
|
61
|
+
token: state.recovery.guardian.token
|
|
62
|
+
})
|
|
63
|
+
state.recovery.guardian = {...state.recovery.guardian, pid: partialGuardian.pid, socketPath: partialSocketPath}
|
|
64
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
65
|
+
retainedConnection = await openWebSocket(proxyPort)
|
|
66
|
+
const retainedClosed = once(retainedConnection, "close")
|
|
67
|
+
await fs.rm(socketPath)
|
|
68
|
+
await prepareCandidatePackage(packagePath)
|
|
69
|
+
const ownerExit = once(owner, "exit")
|
|
70
|
+
const ensured = await runEnsureDaemon({configPath, daemonPidPath, logPath: path.join(root, "candidate.log"), packagePath, runtimePath})
|
|
71
|
+
|
|
72
|
+
assert.equal(ensured.code, 0, `${ensured.stderr}\n${await fs.readFile(path.join(root, "candidate.log"), "utf8")}`)
|
|
73
|
+
assert.deepEqual(await ownerExit, [null, "SIGKILL"], "the exact authenticated incumbent boundary is crossed once")
|
|
74
|
+
await retainedClosed
|
|
75
|
+
const status = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
76
|
+
|
|
77
|
+
assert.equal(status.activeReleaseId, "v1")
|
|
78
|
+
assert.equal(releaseProcessPid(status, "v1", "worker"), workerPid)
|
|
79
|
+
assert.equal(releaseProcessPid(status, "v1", "web"), webPid)
|
|
80
|
+
assert.deepEqual(status.ownerTransition, {
|
|
81
|
+
disruptive: true,
|
|
82
|
+
mode: "legacy-first-upgrade",
|
|
83
|
+
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
84
|
+
})
|
|
85
|
+
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
|
|
86
|
+
|
|
87
|
+
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
88
|
+
await shutdown
|
|
89
|
+
} finally {
|
|
90
|
+
retainedConnection?.destroy()
|
|
91
|
+
if (owner && owner.exitCode === null && owner.signalCode === null) {
|
|
92
|
+
const exited = once(owner, "exit")
|
|
93
|
+
|
|
94
|
+
owner.kill("SIGKILL")
|
|
95
|
+
await exited
|
|
96
|
+
}
|
|
97
|
+
partialGuardian?.kill("SIGTERM")
|
|
98
|
+
if (partialGuardian && partialGuardian.exitCode === null && partialGuardian.signalCode === null) await once(partialGuardian, "exit")
|
|
99
|
+
if (backendGuardianIdentity) {
|
|
100
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8").catch(() => "null"))
|
|
101
|
+
|
|
102
|
+
if (state?.recovery?.guardian?.socketPath === partialSocketPath) {
|
|
103
|
+
state.recovery.guardian = backendGuardianIdentity
|
|
104
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
await stopGuardian(statePath)
|
|
108
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
109
|
+
}
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
test("partial guardian replacement remains fenced through coordinator reconstruction", async () => {
|
|
113
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-fence-"))
|
|
114
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
115
|
+
const statePath = path.join(root, "state.json")
|
|
116
|
+
const configPath = path.join(root, "rollbridge.cjs")
|
|
117
|
+
const releasePath = path.join(root, "v1")
|
|
118
|
+
const partialSocketPath = path.join(root, "partial-guardian.sock")
|
|
119
|
+
const restoreStarted = deferred()
|
|
120
|
+
const continueRestore = deferred()
|
|
121
|
+
let owner
|
|
122
|
+
let partialGuardian
|
|
123
|
+
let candidate = /** @type {RollbridgeDaemon | undefined} */ (undefined)
|
|
124
|
+
let contender = /** @type {GuardianClient | undefined} */ (undefined)
|
|
125
|
+
let backendGuardianIdentity
|
|
126
|
+
let replacementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
127
|
+
|
|
128
|
+
try {
|
|
129
|
+
await fs.mkdir(releasePath)
|
|
130
|
+
await makeFifo(path.join(releasePath, "worker.fifo"))
|
|
131
|
+
await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
132
|
+
owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
|
|
133
|
+
await waitForLog(owner, "control socket listening")
|
|
134
|
+
assert.ok(owner.pid)
|
|
135
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: socketPath})
|
|
136
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
137
|
+
|
|
138
|
+
backendGuardianIdentity = {...state.recovery.guardian}
|
|
139
|
+
partialGuardian = await startPartialGuardian({
|
|
140
|
+
backendPath: backendGuardianIdentity.socketPath,
|
|
141
|
+
mode: "partial",
|
|
142
|
+
socketPath: partialSocketPath,
|
|
143
|
+
token: backendGuardianIdentity.token
|
|
144
|
+
})
|
|
145
|
+
state.recovery.guardian = {...backendGuardianIdentity, pid: partialGuardian.pid, socketPath: partialSocketPath}
|
|
146
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
147
|
+
const replacement = new RollbridgeDaemon({
|
|
148
|
+
config: normalizeConfig(config({controlPath: socketPath, extraCompanion: false, statePath})),
|
|
149
|
+
configPath,
|
|
150
|
+
legacyIncumbentPid: owner.pid,
|
|
151
|
+
logger: () => {}
|
|
152
|
+
})
|
|
153
|
+
candidate = replacement
|
|
154
|
+
replacement.restoreOwnerState = async () => {
|
|
155
|
+
restoreStarted.resolve(undefined)
|
|
156
|
+
await continueRestore.promise
|
|
157
|
+
throw new Error("injected reconstruction stop after fence audit")
|
|
158
|
+
}
|
|
159
|
+
replacementPromise = replacement.replaceIncompatibleOwner()
|
|
160
|
+
void replacementPromise.catch(() => {})
|
|
161
|
+
await restoreStarted.promise
|
|
162
|
+
assert.equal(JSON.parse(await fs.readFile(statePath, "utf8")).recovery.guardian.socketPath, partialSocketPath)
|
|
163
|
+
|
|
164
|
+
let mutationError
|
|
165
|
+
|
|
166
|
+
try {
|
|
167
|
+
await sendControlCommand({command: {command: "restart", processId: "missing"}, path: socketPath})
|
|
168
|
+
} catch (error) {
|
|
169
|
+
mutationError = error
|
|
170
|
+
}
|
|
171
|
+
contender = new GuardianClient({pid: partialGuardian.pid, socketPath: partialSocketPath, token: backendGuardianIdentity.token})
|
|
172
|
+
await contender.connect()
|
|
173
|
+
let contenderError
|
|
174
|
+
let contenderPrepared
|
|
175
|
+
|
|
176
|
+
try {
|
|
177
|
+
contenderPrepared = await contender.prepareOwnerReplacement(
|
|
178
|
+
{configDigest: state.recovery.configDigest, runtime: state.daemonRuntime ?? null},
|
|
179
|
+
{configDigest: "contender", runtime: null}
|
|
180
|
+
)
|
|
181
|
+
} catch (error) {
|
|
182
|
+
contenderError = error
|
|
183
|
+
}
|
|
184
|
+
if (contenderPrepared) await contender.abortOwnerReplacement(contenderPrepared.replacementId)
|
|
185
|
+
contender.disconnect()
|
|
186
|
+
contender = undefined
|
|
187
|
+
continueRestore.resolve(undefined)
|
|
188
|
+
await assert.rejects(replacementPromise, /injected reconstruction stop after fence audit/)
|
|
189
|
+
replacementPromise = undefined
|
|
190
|
+
assert.match(mutationError instanceof Error ? mutationError.message : "", /fenced while an owner replacement is prepared/)
|
|
191
|
+
assert.match(contenderError instanceof Error ? contenderError.message : "", /another owner replacement candidate is already prepared/i)
|
|
192
|
+
assert.notEqual(JSON.parse(await fs.readFile(statePath, "utf8")).recovery.guardian.socketPath, `${statePath}.split3-guardian.sock`)
|
|
193
|
+
assert.equal((await sendControlCommand({command: {command: "status"}, path: socketPath})).daemonPid, owner.pid)
|
|
194
|
+
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
|
|
195
|
+
|
|
196
|
+
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
197
|
+
await shutdown
|
|
198
|
+
} finally {
|
|
199
|
+
continueRestore.resolve(undefined)
|
|
200
|
+
contender?.disconnect()
|
|
201
|
+
await replacementPromise?.catch(() => undefined)
|
|
202
|
+
candidate?.guardian?.disconnect()
|
|
203
|
+
if (owner && owner.exitCode === null && owner.signalCode === null) {
|
|
204
|
+
const exited = once(owner, "exit")
|
|
205
|
+
|
|
206
|
+
owner.kill("SIGKILL")
|
|
207
|
+
await exited
|
|
208
|
+
}
|
|
209
|
+
partialGuardian?.kill("SIGTERM")
|
|
210
|
+
if (partialGuardian && partialGuardian.exitCode === null && partialGuardian.signalCode === null) await once(partialGuardian, "exit")
|
|
211
|
+
if (backendGuardianIdentity) {
|
|
212
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8").catch(() => "null"))
|
|
213
|
+
|
|
214
|
+
if (state?.recovery?.guardian?.socketPath === partialSocketPath) {
|
|
215
|
+
state.recovery.guardian = backendGuardianIdentity
|
|
216
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
await stopGuardian(statePath)
|
|
220
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
221
|
+
}
|
|
222
|
+
})
|
|
223
|
+
|
|
224
|
+
test("partial guardian replacement persists the coordinator only after ownership confirmation", async () => {
|
|
225
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-persist-"))
|
|
226
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
227
|
+
const statePath = path.join(root, "state.json")
|
|
228
|
+
const configPath = path.join(root, "rollbridge.cjs")
|
|
229
|
+
const releasePath = path.join(root, "v1")
|
|
230
|
+
const partialSocketPath = path.join(root, "partial-guardian.sock")
|
|
231
|
+
const ownershipConfirmationStarted = deferred()
|
|
232
|
+
const continueOwnershipConfirmation = deferred()
|
|
233
|
+
let owner
|
|
234
|
+
let partialGuardian
|
|
235
|
+
let candidate = /** @type {RollbridgeDaemon | undefined} */ (undefined)
|
|
236
|
+
let backendGuardianIdentity
|
|
237
|
+
let replacementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
238
|
+
|
|
239
|
+
try {
|
|
240
|
+
await fs.mkdir(releasePath)
|
|
241
|
+
await makeFifo(path.join(releasePath, "worker.fifo"))
|
|
242
|
+
await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
243
|
+
owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
|
|
244
|
+
await waitForLog(owner, "control socket listening")
|
|
245
|
+
assert.ok(owner.pid)
|
|
246
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: socketPath})
|
|
247
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
248
|
+
|
|
249
|
+
backendGuardianIdentity = {...state.recovery.guardian}
|
|
250
|
+
partialGuardian = await startPartialGuardian({
|
|
251
|
+
backendPath: backendGuardianIdentity.socketPath,
|
|
252
|
+
mode: "partial",
|
|
253
|
+
socketPath: partialSocketPath,
|
|
254
|
+
token: backendGuardianIdentity.token
|
|
255
|
+
})
|
|
256
|
+
state.recovery.guardian = {...backendGuardianIdentity, pid: partialGuardian.pid, socketPath: partialSocketPath}
|
|
257
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
258
|
+
const replacement = new RollbridgeDaemon({
|
|
259
|
+
config: normalizeConfig(config({controlPath: socketPath, extraCompanion: false, statePath})),
|
|
260
|
+
configPath,
|
|
261
|
+
legacyIncumbentPid: owner.pid,
|
|
262
|
+
logger: () => {}
|
|
263
|
+
})
|
|
264
|
+
const restoreOwnerState = replacement.restoreOwnerState.bind(replacement)
|
|
265
|
+
|
|
266
|
+
candidate = replacement
|
|
267
|
+
replacement.restoreOwnerState = async (...args) => {
|
|
268
|
+
await restoreOwnerState(...args)
|
|
269
|
+
const coordinator = replacement.guardian
|
|
270
|
+
|
|
271
|
+
if (!coordinator) throw new Error("Partial guardian persistence fixture is missing its coordinator")
|
|
272
|
+
const completeLegacyOwnerClaim = coordinator.completeLegacyOwnerClaim.bind(coordinator)
|
|
273
|
+
|
|
274
|
+
coordinator.completeLegacyOwnerClaim = async (replacementId) => {
|
|
275
|
+
ownershipConfirmationStarted.resolve(undefined)
|
|
276
|
+
await continueOwnershipConfirmation.promise
|
|
277
|
+
await completeLegacyOwnerClaim(replacementId)
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
replacementPromise = replacement.replaceIncompatibleOwner()
|
|
281
|
+
void replacementPromise.catch(() => {})
|
|
282
|
+
await ownershipConfirmationStarted.promise
|
|
283
|
+
assert.notEqual(
|
|
284
|
+
JSON.parse(await fs.readFile(statePath, "utf8")).recovery.guardian.socketPath,
|
|
285
|
+
`${statePath}.split3-guardian.sock`,
|
|
286
|
+
"durable state must not name the coordinator before its legacy ownership claim is confirmed"
|
|
287
|
+
)
|
|
288
|
+
continueOwnershipConfirmation.resolve(undefined)
|
|
289
|
+
await replacementPromise
|
|
290
|
+
replacementPromise = undefined
|
|
291
|
+
const shutdown = replacement.shutdown()
|
|
292
|
+
|
|
293
|
+
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
294
|
+
await shutdown
|
|
295
|
+
candidate = undefined
|
|
296
|
+
} finally {
|
|
297
|
+
continueOwnershipConfirmation.resolve(undefined)
|
|
298
|
+
await replacementPromise?.catch(() => undefined)
|
|
299
|
+
if (candidate?.controlServer || candidate?.proxyServer) {
|
|
300
|
+
const shutdown = candidate.shutdown().catch(() => undefined)
|
|
301
|
+
|
|
302
|
+
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
303
|
+
await shutdown
|
|
304
|
+
}
|
|
305
|
+
candidate?.guardian?.disconnect()
|
|
306
|
+
if (owner && owner.exitCode === null && owner.signalCode === null) {
|
|
307
|
+
const exited = once(owner, "exit")
|
|
308
|
+
|
|
309
|
+
owner.kill("SIGKILL")
|
|
310
|
+
await exited
|
|
311
|
+
}
|
|
312
|
+
partialGuardian?.kill("SIGTERM")
|
|
313
|
+
if (partialGuardian && partialGuardian.exitCode === null && partialGuardian.signalCode === null) await once(partialGuardian, "exit")
|
|
314
|
+
if (backendGuardianIdentity) {
|
|
315
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8").catch(() => "null"))
|
|
316
|
+
|
|
317
|
+
if (state?.recovery?.guardian?.socketPath === partialSocketPath) {
|
|
318
|
+
state.recovery.guardian = backendGuardianIdentity
|
|
319
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
await stopGuardian(statePath)
|
|
323
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
324
|
+
}
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
test("failed partial upgrade resumes an incumbent retired release drain", {timeout: 5000}, async () => {
|
|
328
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-drain-"))
|
|
329
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
330
|
+
const statePath = path.join(root, "state.json")
|
|
331
|
+
const configPath = path.join(root, "rollbridge.cjs")
|
|
332
|
+
const v1Path = path.join(root, "v1")
|
|
333
|
+
const v2Path = path.join(root, "v2")
|
|
334
|
+
const partialSocketPath = path.join(root, "partial-guardian.sock")
|
|
335
|
+
let owner
|
|
336
|
+
let partialGuardian
|
|
337
|
+
let retainedConnection
|
|
338
|
+
let candidate = /** @type {RollbridgeDaemon | undefined} */ (undefined)
|
|
339
|
+
let backendGuardianIdentity
|
|
340
|
+
|
|
341
|
+
try {
|
|
342
|
+
await Promise.all([fs.mkdir(v1Path), fs.mkdir(v2Path)])
|
|
343
|
+
await Promise.all([makeFifo(path.join(v1Path, "worker.fifo")), makeFifo(path.join(v2Path, "worker.fifo"))])
|
|
344
|
+
await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
345
|
+
owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
|
|
346
|
+
await waitForLog(owner, "control socket listening")
|
|
347
|
+
assert.ok(owner.pid)
|
|
348
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: socketPath})
|
|
349
|
+
const v1Status = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
350
|
+
const proxyPort = /** @type {{port?: number}} */ (v1Status.proxy).port
|
|
351
|
+
|
|
352
|
+
if (typeof proxyPort !== "number") throw new Error("Partial guardian drain fixture proxy is missing its bound port")
|
|
353
|
+
retainedConnection = await openWebSocket(proxyPort)
|
|
354
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: socketPath})
|
|
355
|
+
const draining = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
356
|
+
|
|
357
|
+
assert.equal(releaseState(draining, "v1"), "draining")
|
|
358
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
359
|
+
|
|
360
|
+
backendGuardianIdentity = {...state.recovery.guardian}
|
|
361
|
+
partialGuardian = await startPartialGuardian({
|
|
362
|
+
backendPath: backendGuardianIdentity.socketPath,
|
|
363
|
+
mode: "wrong-provenance",
|
|
364
|
+
socketPath: partialSocketPath,
|
|
365
|
+
token: backendGuardianIdentity.token
|
|
366
|
+
})
|
|
367
|
+
state.recovery.guardian = {...backendGuardianIdentity, pid: partialGuardian.pid, socketPath: partialSocketPath}
|
|
368
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
369
|
+
const replacement = new RollbridgeDaemon({
|
|
370
|
+
config: normalizeConfig(config({controlPath: socketPath, extraCompanion: false, statePath})),
|
|
371
|
+
configPath,
|
|
372
|
+
legacyIncumbentPid: owner.pid,
|
|
373
|
+
logger: () => {}
|
|
374
|
+
})
|
|
375
|
+
|
|
376
|
+
candidate = replacement
|
|
377
|
+
await assert.rejects(() => replacement.replaceIncompatibleOwner(), /provenance mismatch/)
|
|
378
|
+
assert.equal(owner.exitCode, null)
|
|
379
|
+
assert.equal(owner.signalCode, null)
|
|
380
|
+
const releaseDrained = waitForLog(owner, "release drained")
|
|
381
|
+
|
|
382
|
+
retainedConnection.destroy()
|
|
383
|
+
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
384
|
+
await releaseDrained
|
|
385
|
+
assert.equal(releaseState(await sendControlCommand({command: {command: "status"}, path: socketPath}), "v1"), "stopped")
|
|
386
|
+
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
|
|
387
|
+
|
|
388
|
+
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
389
|
+
await shutdown
|
|
390
|
+
} finally {
|
|
391
|
+
retainedConnection?.destroy()
|
|
392
|
+
candidate?.guardian?.disconnect()
|
|
393
|
+
if (owner && owner.exitCode === null && owner.signalCode === null) owner.kill("SIGKILL")
|
|
394
|
+
partialGuardian?.kill("SIGTERM")
|
|
395
|
+
if (partialGuardian && partialGuardian.exitCode === null && partialGuardian.signalCode === null) await once(partialGuardian, "exit")
|
|
396
|
+
if (backendGuardianIdentity) {
|
|
397
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8").catch(() => "null"))
|
|
398
|
+
|
|
399
|
+
if (state?.recovery?.guardian?.socketPath === partialSocketPath) {
|
|
400
|
+
state.recovery.guardian = backendGuardianIdentity
|
|
401
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
await stopGuardian(statePath)
|
|
405
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
406
|
+
}
|
|
407
|
+
})
|
|
408
|
+
|
|
409
|
+
test("partial guardian classification failures preserve the incumbent, children, and retained stream", async (t) => {
|
|
410
|
+
for (const fault of ["malformed-capability", "wrong-pid", "wrong-provenance"]) {
|
|
411
|
+
await t.test(fault, async () => {
|
|
412
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), `rollbridge-owner-replacement-partial-${fault}-`))
|
|
413
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
414
|
+
const statePath = path.join(root, "state.json")
|
|
415
|
+
const configPath = path.join(root, "rollbridge.cjs")
|
|
416
|
+
const releasePath = path.join(root, "v1")
|
|
417
|
+
const partialSocketPath = path.join(root, "partial-guardian.sock")
|
|
418
|
+
let owner
|
|
419
|
+
let partialGuardian
|
|
420
|
+
let retainedConnection
|
|
421
|
+
let candidate = /** @type {RollbridgeDaemon | undefined} */ (undefined)
|
|
422
|
+
|
|
423
|
+
try {
|
|
424
|
+
await fs.mkdir(releasePath)
|
|
425
|
+
await makeFifo(path.join(releasePath, "worker.fifo"))
|
|
426
|
+
await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
427
|
+
owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
|
|
428
|
+
await waitForLog(owner, "control socket listening")
|
|
429
|
+
assert.ok(owner.pid)
|
|
430
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: socketPath})
|
|
431
|
+
const before = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
432
|
+
const processPids = [releaseProcessPid(before, "v1", "worker"), releaseProcessPid(before, "v1", "web")]
|
|
433
|
+
const proxyPort = /** @type {{port?: number}} */ (before.proxy).port
|
|
434
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
435
|
+
const backendIdentity = {...state.recovery.guardian}
|
|
436
|
+
|
|
437
|
+
if (typeof proxyPort !== "number") throw new Error("Partial guardian failure fixture proxy is missing its bound port")
|
|
438
|
+
partialGuardian = await startPartialGuardian({
|
|
439
|
+
backendPath: backendIdentity.socketPath,
|
|
440
|
+
mode: fault === "wrong-provenance" ? fault : fault === "malformed-capability" ? fault : "partial",
|
|
441
|
+
socketPath: partialSocketPath,
|
|
442
|
+
token: backendIdentity.token
|
|
443
|
+
})
|
|
444
|
+
state.recovery.guardian = {
|
|
445
|
+
...backendIdentity,
|
|
446
|
+
pid: fault === "wrong-pid" ? backendIdentity.pid : partialGuardian.pid,
|
|
447
|
+
socketPath: partialSocketPath
|
|
448
|
+
}
|
|
449
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
450
|
+
retainedConnection = await openWebSocket(proxyPort)
|
|
451
|
+
let retainedClosed = false
|
|
452
|
+
|
|
453
|
+
retainedConnection.once("close", () => { retainedClosed = true })
|
|
454
|
+
const replacement = new RollbridgeDaemon({
|
|
455
|
+
config: normalizeConfig(config({controlPath: socketPath, extraCompanion: false, statePath})),
|
|
456
|
+
configPath,
|
|
457
|
+
legacyIncumbentPid: owner.pid,
|
|
458
|
+
logger: () => {}
|
|
459
|
+
})
|
|
460
|
+
candidate = replacement
|
|
461
|
+
const expected = fault === "malformed-capability"
|
|
462
|
+
? /invalid owner-replacement capability response/
|
|
463
|
+
: fault === "wrong-pid"
|
|
464
|
+
? /does not own socket|does not match the retained guardian command and socket/
|
|
465
|
+
: /provenance mismatch/
|
|
466
|
+
|
|
467
|
+
await assert.rejects(() => replacement.replaceIncompatibleOwner(), expected)
|
|
468
|
+
assert.equal(owner.exitCode, null)
|
|
469
|
+
assert.equal(owner.signalCode, null)
|
|
470
|
+
assert.equal(retainedClosed, false)
|
|
471
|
+
assert.equal(retainedConnection.destroyed, false)
|
|
472
|
+
for (const pid of processPids) assert.doesNotThrow(() => process.kill(pid, 0))
|
|
473
|
+
assert.equal(releaseProcessPid(await sendControlCommand({command: {command: "status"}, path: socketPath}), "v1", "worker"), processPids[0])
|
|
474
|
+
await assert.rejects(fs.access(`${statePath}.split3-guardian.sock`), {code: "ENOENT"})
|
|
475
|
+
|
|
476
|
+
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
|
|
477
|
+
|
|
478
|
+
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
479
|
+
await shutdown
|
|
480
|
+
} finally {
|
|
481
|
+
retainedConnection?.destroy()
|
|
482
|
+
candidate?.guardian?.disconnect()
|
|
483
|
+
if (owner && owner.exitCode === null && owner.signalCode === null) owner.kill("SIGKILL")
|
|
484
|
+
partialGuardian?.kill("SIGTERM")
|
|
485
|
+
if (partialGuardian && partialGuardian.exitCode === null && partialGuardian.signalCode === null) await once(partialGuardian, "exit")
|
|
486
|
+
await stopGuardian(statePath)
|
|
487
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
488
|
+
}
|
|
489
|
+
})
|
|
490
|
+
}
|
|
491
|
+
})
|
|
22
492
|
|
|
23
493
|
test("first pre-split package upgrade is explicitly disruptive and later replacements are atomic", async () => {
|
|
24
494
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-legacy-"))
|
|
@@ -81,7 +551,7 @@ test("first pre-split package upgrade is explicitly disruptive and later replace
|
|
|
81
551
|
assert.deepEqual(JSON.parse(firstUpgrade.stdout).ownerTransition, {
|
|
82
552
|
disruptive: true,
|
|
83
553
|
mode: "legacy-first-upgrade",
|
|
84
|
-
reason: "
|
|
554
|
+
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
85
555
|
})
|
|
86
556
|
await interrupted
|
|
87
557
|
const bridged = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
@@ -89,7 +559,7 @@ test("first pre-split package upgrade is explicitly disruptive and later replace
|
|
|
89
559
|
assert.deepEqual(bridged.ownerTransition, {
|
|
90
560
|
disruptive: true,
|
|
91
561
|
mode: "legacy-first-upgrade",
|
|
92
|
-
reason: "
|
|
562
|
+
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
93
563
|
})
|
|
94
564
|
assert.equal(releaseProcessPid(bridged, "v1", "worker"), legacyWorkerPid)
|
|
95
565
|
assert.equal(bridged.activeReleaseId, "v1")
|
|
@@ -393,16 +863,24 @@ test("cross-version replacement fails closed without dropping a retained WebSock
|
|
|
393
863
|
}
|
|
394
864
|
})
|
|
395
865
|
|
|
396
|
-
test("
|
|
866
|
+
test("cross-version replacement preserves committed-owner proof until commit then recovers every process", async () => {
|
|
397
867
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-committed-proof-"))
|
|
398
868
|
const socketPath = path.join(root, "rollbridge.sock")
|
|
399
869
|
const statePath = path.join(root, "state.json")
|
|
870
|
+
const compatibilitySocketPath = path.join(root, "retained-guardian.sock")
|
|
400
871
|
const releasePath = path.join(root, "v1")
|
|
401
872
|
const daemonConfig = normalizeConfig(config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
402
873
|
const owner = new RollbridgeDaemon({config: daemonConfig, logger: () => {}})
|
|
874
|
+
const compatibilitySockets = new Set()
|
|
403
875
|
const candidateProcessKey = "release:candidate:worker"
|
|
404
876
|
const committedOwnerProcessKey = "release:v1:worker"
|
|
877
|
+
const candidateRecoveredKeys = new Set()
|
|
878
|
+
let compatibilityGuardian
|
|
405
879
|
let committedProcessKey
|
|
880
|
+
let recoveredKeysAtCommit = /** @type {Set<string> | undefined} */ (undefined)
|
|
881
|
+
let retainedConnection
|
|
882
|
+
let retainedConnectionClosed = false
|
|
883
|
+
let retainedGuardianSocketPath = /** @type {string | undefined} */ (undefined)
|
|
406
884
|
/** @type {RollbridgeDaemon | undefined} */
|
|
407
885
|
let replacement
|
|
408
886
|
|
|
@@ -421,14 +899,66 @@ test("same-authority replacement commits with committed-owner proof after the in
|
|
|
421
899
|
key: candidateProcessKey,
|
|
422
900
|
provenance: ownerProcess.provenance
|
|
423
901
|
})
|
|
424
|
-
|
|
902
|
+
const running = owner.status()
|
|
903
|
+
const processState = running.releases[0]?.processes.map(({id, pid, state}) => ({id, pid, state}))
|
|
904
|
+
const expectedProcessKeys = new Set(running.releases[0]?.processes.map(({id}) => `release:v1:${id}`))
|
|
905
|
+
const proxyPort = /** @type {{port?: number}} */ (running.proxy).port
|
|
906
|
+
|
|
907
|
+
assert.deepEqual(processState?.map(({state}) => state), ["running", "running"])
|
|
908
|
+
if (typeof proxyPort !== "number") throw new Error("Retained owner proxy is missing its port")
|
|
909
|
+
retainedConnection = await openWebSocket(proxyPort)
|
|
910
|
+
retainedConnection.once("close", () => { retainedConnectionClosed = true })
|
|
425
911
|
await owner.closeServer(owner.controlServer)
|
|
426
912
|
await owner.removeControlSocket()
|
|
427
|
-
await
|
|
428
|
-
const
|
|
429
|
-
|
|
913
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
914
|
+
const guardianSocketPath = state.recovery.guardian.socketPath
|
|
915
|
+
|
|
916
|
+
if (typeof guardianSocketPath !== "string") throw new Error("Retained guardian state is missing its socket path")
|
|
917
|
+
retainedGuardianSocketPath = guardianSocketPath
|
|
918
|
+
compatibilityGuardian = net.createServer((candidateSocket) => {
|
|
919
|
+
const guardianSocket = net.createConnection(guardianSocketPath)
|
|
920
|
+
let buffer = ""
|
|
921
|
+
|
|
922
|
+
compatibilitySockets.add(candidateSocket)
|
|
923
|
+
compatibilitySockets.add(guardianSocket)
|
|
924
|
+
candidateSocket.setEncoding("utf8")
|
|
925
|
+
candidateSocket.once("close", () => {
|
|
926
|
+
compatibilitySockets.delete(candidateSocket)
|
|
927
|
+
guardianSocket.destroy()
|
|
928
|
+
})
|
|
929
|
+
guardianSocket.once("close", () => {
|
|
930
|
+
compatibilitySockets.delete(guardianSocket)
|
|
931
|
+
candidateSocket.destroy()
|
|
932
|
+
})
|
|
933
|
+
guardianSocket.on("data", (chunk) => candidateSocket.write(chunk))
|
|
934
|
+
candidateSocket.on("data", (chunk) => {
|
|
935
|
+
buffer += chunk
|
|
936
|
+
let newline = buffer.indexOf("\n")
|
|
937
|
+
|
|
938
|
+
while (newline >= 0) {
|
|
939
|
+
const line = buffer.slice(0, newline)
|
|
940
|
+
const request = JSON.parse(line)
|
|
941
|
+
|
|
942
|
+
buffer = buffer.slice(newline + 1)
|
|
943
|
+
if (request.command === "register") candidateRecoveredKeys.add(request.key)
|
|
944
|
+
if (request.command === "commit-retired-owner-replacement") {
|
|
945
|
+
committedProcessKey = request.key
|
|
946
|
+
recoveredKeysAtCommit = new Set(candidateRecoveredKeys)
|
|
947
|
+
if (candidateRecoveredKeys.has(request.key)) {
|
|
948
|
+
candidateSocket.write(`${JSON.stringify({error: `Guardian ${request.command} requires the committed owner`, id: request.id})}\n`)
|
|
949
|
+
newline = buffer.indexOf("\n")
|
|
950
|
+
continue
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
guardianSocket.write(`${line}\n`)
|
|
954
|
+
newline = buffer.indexOf("\n")
|
|
955
|
+
}
|
|
956
|
+
})
|
|
957
|
+
})
|
|
958
|
+
await listenUnix(compatibilityGuardian, compatibilitySocketPath)
|
|
959
|
+
state.recovery.guardian.socketPath = compatibilitySocketPath
|
|
960
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
430
961
|
|
|
431
|
-
assert.deepEqual(processState?.map(({state}) => state), ["quiesced", "quiesced"])
|
|
432
962
|
assert.equal((await owner.guardian?.replacementStatus())?.ownerClaimed, true)
|
|
433
963
|
await assert.rejects(fs.access(socketPath), {code: "ENOENT"})
|
|
434
964
|
|
|
@@ -455,14 +985,33 @@ test("same-authority replacement commits with committed-owner proof after the in
|
|
|
455
985
|
|
|
456
986
|
assert.equal(recovered.activeReleaseId, "v1")
|
|
457
987
|
assert.equal(committedProcessKey, committedOwnerProcessKey)
|
|
988
|
+
assert.equal(recoveredKeysAtCommit?.has(committedOwnerProcessKey), false)
|
|
989
|
+
assert.deepEqual(candidateRecoveredKeys, expectedProcessKeys)
|
|
990
|
+
assert.equal([...replacement.guardian?.processes.keys() || []][0], candidateProcessKey)
|
|
991
|
+
assert.equal(retainedConnectionClosed, false, "successful compatibility handoff must preserve retained connections")
|
|
992
|
+
assert.equal(retainedConnection.destroyed, false, "successful compatibility handoff must leave the retained listener serving")
|
|
458
993
|
assert.deepEqual(recovered.releaseReferences, [{releaseId: "v1", releasePath}])
|
|
459
994
|
assert.deepEqual(recoveredReleases[0]?.processes.map(({id, pid, state}) => ({id, pid, state})), processState)
|
|
995
|
+
for (const {pid} of processState || []) {
|
|
996
|
+
if (typeof pid !== "number") throw new Error("Retained process is missing its PID")
|
|
997
|
+
assert.doesNotThrow(() => process.kill(pid, 0))
|
|
998
|
+
}
|
|
460
999
|
} finally {
|
|
1000
|
+
if (retainedGuardianSocketPath) {
|
|
1001
|
+
const cleanupState = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
1002
|
+
|
|
1003
|
+
cleanupState.recovery.guardian.socketPath = retainedGuardianSocketPath
|
|
1004
|
+
await fs.writeFile(statePath, `${JSON.stringify(cleanupState)}\n`)
|
|
1005
|
+
}
|
|
461
1006
|
const shutdown = replacement?.controlCommandsReady ? replacement.shutdown().catch(() => {}) : owner.shutdown().catch(() => {})
|
|
462
1007
|
|
|
1008
|
+
retainedConnection?.destroy()
|
|
1009
|
+
await owner.closeServer(owner.proxyServer)
|
|
463
1010
|
await Promise.all([fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n").catch(() => {}), shutdown])
|
|
464
1011
|
owner.guardian?.disconnect()
|
|
465
1012
|
replacement?.guardian?.disconnect()
|
|
1013
|
+
for (const socket of compatibilitySockets) socket.destroy()
|
|
1014
|
+
if (compatibilityGuardian?.listening) await closeServer(compatibilityGuardian)
|
|
466
1015
|
await stopGuardian(statePath)
|
|
467
1016
|
await fs.rm(root, {force: true, recursive: true})
|
|
468
1017
|
}
|
|
@@ -891,6 +1440,32 @@ async function runEnsureDaemon({configPath, daemonPidPath, logPath, packagePath,
|
|
|
891
1440
|
])
|
|
892
1441
|
}
|
|
893
1442
|
|
|
1443
|
+
/**
|
|
1444
|
+
* Starts an authenticated protocol proxy that exposes the exact partial guardian surface.
|
|
1445
|
+
* @param {{backendPath: string, mode: string, socketPath: string, token: string}} options - Fixture identity.
|
|
1446
|
+
* @returns {Promise<import("node:child_process").ChildProcess>} Ready guardian proxy.
|
|
1447
|
+
*/
|
|
1448
|
+
async function startPartialGuardian({backendPath, mode, socketPath, token}) {
|
|
1449
|
+
const child = spawn(process.execPath, [partialGuardianPath, socketPath, backendPath, mode], {
|
|
1450
|
+
stdio: ["ignore", "ignore", "ignore", "ipc"]
|
|
1451
|
+
})
|
|
1452
|
+
|
|
1453
|
+
await new Promise((resolve, reject) => {
|
|
1454
|
+
child.once("error", reject)
|
|
1455
|
+
child.once("exit", (code) => reject(new Error(`Partial guardian exited before readiness with status ${code}`)))
|
|
1456
|
+
child.once("message", (message) => {
|
|
1457
|
+
if (message && typeof message === "object" && "error" in message) reject(new Error(String(message.error)))
|
|
1458
|
+
else resolve(undefined)
|
|
1459
|
+
})
|
|
1460
|
+
child.send({token}, (error) => {
|
|
1461
|
+
if (error) reject(error)
|
|
1462
|
+
})
|
|
1463
|
+
})
|
|
1464
|
+
if (child.connected) await once(child, "disconnect")
|
|
1465
|
+
assert.ok(child.pid)
|
|
1466
|
+
return child
|
|
1467
|
+
}
|
|
1468
|
+
|
|
894
1469
|
/**
|
|
895
1470
|
* @param {net.Server} server - Unix server.
|
|
896
1471
|
* @param {string} socketPath - Unix socket path.
|
|
@@ -934,6 +1509,27 @@ function releaseProcessPid(status, releaseId, processId) {
|
|
|
934
1509
|
return pid
|
|
935
1510
|
}
|
|
936
1511
|
|
|
1512
|
+
/**
|
|
1513
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} status - Daemon status.
|
|
1514
|
+
* @param {string} releaseId - Release identity.
|
|
1515
|
+
* @returns {string} Release lifecycle state.
|
|
1516
|
+
*/
|
|
1517
|
+
function releaseState(status, releaseId) {
|
|
1518
|
+
const releases = /** @type {{releaseId: string, state: string}[]} */ (status.releases)
|
|
1519
|
+
const state = releases.find((release) => release.releaseId === releaseId)?.state
|
|
1520
|
+
|
|
1521
|
+
if (!state) throw new Error(`Missing release state for ${releaseId}`)
|
|
1522
|
+
return state
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
/** @returns {{promise: Promise<void>, resolve: (value: void) => void}} Controllable event barrier. */
|
|
1526
|
+
function deferred() {
|
|
1527
|
+
let resolve = /** @type {(value: void) => void} */ (() => {})
|
|
1528
|
+
const promise = new Promise((promiseResolve) => { resolve = promiseResolve })
|
|
1529
|
+
|
|
1530
|
+
return {promise, resolve}
|
|
1531
|
+
}
|
|
1532
|
+
|
|
937
1533
|
/**
|
|
938
1534
|
* @param {string} command - Executable.
|
|
939
1535
|
* @param {string[]} args - Arguments.
|