rollbridge 0.1.36 → 0.1.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +7 -0
- package/docs/cli.md +2 -1
- package/docs/config.md +4 -2
- package/docs/troubleshooting.md +3 -0
- package/package.json +1 -1
- package/src/daemon.js +216 -61
- package/src/guardian-client.js +66 -1
- package/src/process-guardian.js +31 -6
- package/test/fixtures/partial-owner-replacement-process-guardian.js +106 -0
- package/test/guardian-client.test.js +74 -0
- package/test/owner-replacement.test.js +519 -2
|
@@ -19,6 +19,476 @@ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."
|
|
|
19
19
|
const binPath = path.join(repoRoot, "bin", "rollbridge")
|
|
20
20
|
const dummyAppPath = path.join(repoRoot, "test", "fixtures", "dummy-app.js")
|
|
21
21
|
const legacyDaemonPath = path.join(repoRoot, "test", "fixtures", "pre-split3-daemon-runner.js")
|
|
22
|
+
const partialGuardianPath = path.join(repoRoot, "test", "fixtures", "partial-owner-replacement-process-guardian.js")
|
|
23
|
+
|
|
24
|
+
test("partial owner-replacement guardian crosses the authenticated legacy bridge before retired-owner commit", async () => {
|
|
25
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-"))
|
|
26
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
27
|
+
const statePath = path.join(root, "state.json")
|
|
28
|
+
const configPath = path.join(root, "rollbridge.cjs")
|
|
29
|
+
const daemonPidPath = path.join(root, "daemon.pid")
|
|
30
|
+
const releasePath = path.join(root, "v1")
|
|
31
|
+
const packagePath = path.join(root, "candidate-package")
|
|
32
|
+
const runtimePath = path.join(root, "runtime")
|
|
33
|
+
const partialSocketPath = path.join(root, "partial-guardian.sock")
|
|
34
|
+
let owner
|
|
35
|
+
let partialGuardian
|
|
36
|
+
let retainedConnection
|
|
37
|
+
let backendGuardianIdentity
|
|
38
|
+
|
|
39
|
+
try {
|
|
40
|
+
await fs.mkdir(releasePath)
|
|
41
|
+
await makeFifo(path.join(releasePath, "worker.fifo"))
|
|
42
|
+
await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
43
|
+
owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
|
|
44
|
+
await waitForLog(owner, "control socket listening")
|
|
45
|
+
assert.ok(owner.pid)
|
|
46
|
+
await fs.writeFile(daemonPidPath, `${owner.pid}\n`)
|
|
47
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: socketPath})
|
|
48
|
+
const before = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
49
|
+
const workerPid = releaseProcessPid(before, "v1", "worker")
|
|
50
|
+
const webPid = releaseProcessPid(before, "v1", "web")
|
|
51
|
+
const proxyPort = /** @type {{port?: number}} */ (before.proxy).port
|
|
52
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
53
|
+
|
|
54
|
+
if (typeof proxyPort !== "number") throw new Error("Partial guardian fixture proxy is missing its bound port")
|
|
55
|
+
|
|
56
|
+
backendGuardianIdentity = {...state.recovery.guardian}
|
|
57
|
+
partialGuardian = await startPartialGuardian({
|
|
58
|
+
backendPath: state.recovery.guardian.socketPath,
|
|
59
|
+
mode: "partial",
|
|
60
|
+
socketPath: partialSocketPath,
|
|
61
|
+
token: state.recovery.guardian.token
|
|
62
|
+
})
|
|
63
|
+
state.recovery.guardian = {...state.recovery.guardian, pid: partialGuardian.pid, socketPath: partialSocketPath}
|
|
64
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
65
|
+
retainedConnection = await openWebSocket(proxyPort)
|
|
66
|
+
const retainedClosed = once(retainedConnection, "close")
|
|
67
|
+
await fs.rm(socketPath)
|
|
68
|
+
await prepareCandidatePackage(packagePath)
|
|
69
|
+
const ownerExit = once(owner, "exit")
|
|
70
|
+
const ensured = await runEnsureDaemon({configPath, daemonPidPath, logPath: path.join(root, "candidate.log"), packagePath, runtimePath})
|
|
71
|
+
|
|
72
|
+
assert.equal(ensured.code, 0, `${ensured.stderr}\n${await fs.readFile(path.join(root, "candidate.log"), "utf8")}`)
|
|
73
|
+
assert.deepEqual(await ownerExit, [null, "SIGKILL"], "the exact authenticated incumbent boundary is crossed once")
|
|
74
|
+
await retainedClosed
|
|
75
|
+
const status = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
76
|
+
|
|
77
|
+
assert.equal(status.activeReleaseId, "v1")
|
|
78
|
+
assert.equal(releaseProcessPid(status, "v1", "worker"), workerPid)
|
|
79
|
+
assert.equal(releaseProcessPid(status, "v1", "web"), webPid)
|
|
80
|
+
assert.deepEqual(status.ownerTransition, {
|
|
81
|
+
disruptive: true,
|
|
82
|
+
mode: "legacy-first-upgrade",
|
|
83
|
+
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
84
|
+
})
|
|
85
|
+
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
|
|
86
|
+
|
|
87
|
+
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
88
|
+
await shutdown
|
|
89
|
+
} finally {
|
|
90
|
+
retainedConnection?.destroy()
|
|
91
|
+
if (owner && owner.exitCode === null && owner.signalCode === null) {
|
|
92
|
+
const exited = once(owner, "exit")
|
|
93
|
+
|
|
94
|
+
owner.kill("SIGKILL")
|
|
95
|
+
await exited
|
|
96
|
+
}
|
|
97
|
+
partialGuardian?.kill("SIGTERM")
|
|
98
|
+
if (partialGuardian && partialGuardian.exitCode === null && partialGuardian.signalCode === null) await once(partialGuardian, "exit")
|
|
99
|
+
if (backendGuardianIdentity) {
|
|
100
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8").catch(() => "null"))
|
|
101
|
+
|
|
102
|
+
if (state?.recovery?.guardian?.socketPath === partialSocketPath) {
|
|
103
|
+
state.recovery.guardian = backendGuardianIdentity
|
|
104
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
await stopGuardian(statePath)
|
|
108
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
109
|
+
}
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
test("partial guardian replacement remains fenced through coordinator reconstruction", async () => {
|
|
113
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-fence-"))
|
|
114
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
115
|
+
const statePath = path.join(root, "state.json")
|
|
116
|
+
const configPath = path.join(root, "rollbridge.cjs")
|
|
117
|
+
const releasePath = path.join(root, "v1")
|
|
118
|
+
const partialSocketPath = path.join(root, "partial-guardian.sock")
|
|
119
|
+
const restoreStarted = deferred()
|
|
120
|
+
const continueRestore = deferred()
|
|
121
|
+
let owner
|
|
122
|
+
let partialGuardian
|
|
123
|
+
let candidate = /** @type {RollbridgeDaemon | undefined} */ (undefined)
|
|
124
|
+
let contender = /** @type {GuardianClient | undefined} */ (undefined)
|
|
125
|
+
let backendGuardianIdentity
|
|
126
|
+
let replacementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
127
|
+
|
|
128
|
+
try {
|
|
129
|
+
await fs.mkdir(releasePath)
|
|
130
|
+
await makeFifo(path.join(releasePath, "worker.fifo"))
|
|
131
|
+
await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
132
|
+
owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
|
|
133
|
+
await waitForLog(owner, "control socket listening")
|
|
134
|
+
assert.ok(owner.pid)
|
|
135
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: socketPath})
|
|
136
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
137
|
+
|
|
138
|
+
backendGuardianIdentity = {...state.recovery.guardian}
|
|
139
|
+
partialGuardian = await startPartialGuardian({
|
|
140
|
+
backendPath: backendGuardianIdentity.socketPath,
|
|
141
|
+
mode: "partial",
|
|
142
|
+
socketPath: partialSocketPath,
|
|
143
|
+
token: backendGuardianIdentity.token
|
|
144
|
+
})
|
|
145
|
+
state.recovery.guardian = {...backendGuardianIdentity, pid: partialGuardian.pid, socketPath: partialSocketPath}
|
|
146
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
147
|
+
const replacement = new RollbridgeDaemon({
|
|
148
|
+
config: normalizeConfig(config({controlPath: socketPath, extraCompanion: false, statePath})),
|
|
149
|
+
configPath,
|
|
150
|
+
legacyIncumbentPid: owner.pid,
|
|
151
|
+
logger: () => {}
|
|
152
|
+
})
|
|
153
|
+
candidate = replacement
|
|
154
|
+
replacement.restoreOwnerState = async () => {
|
|
155
|
+
restoreStarted.resolve(undefined)
|
|
156
|
+
await continueRestore.promise
|
|
157
|
+
throw new Error("injected reconstruction stop after fence audit")
|
|
158
|
+
}
|
|
159
|
+
replacementPromise = replacement.replaceIncompatibleOwner()
|
|
160
|
+
void replacementPromise.catch(() => {})
|
|
161
|
+
await restoreStarted.promise
|
|
162
|
+
assert.equal(JSON.parse(await fs.readFile(statePath, "utf8")).recovery.guardian.socketPath, partialSocketPath)
|
|
163
|
+
|
|
164
|
+
let mutationError
|
|
165
|
+
|
|
166
|
+
try {
|
|
167
|
+
await sendControlCommand({command: {command: "restart", processId: "missing"}, path: socketPath})
|
|
168
|
+
} catch (error) {
|
|
169
|
+
mutationError = error
|
|
170
|
+
}
|
|
171
|
+
contender = new GuardianClient({pid: partialGuardian.pid, socketPath: partialSocketPath, token: backendGuardianIdentity.token})
|
|
172
|
+
await contender.connect()
|
|
173
|
+
let contenderError
|
|
174
|
+
let contenderPrepared
|
|
175
|
+
|
|
176
|
+
try {
|
|
177
|
+
contenderPrepared = await contender.prepareOwnerReplacement(
|
|
178
|
+
{configDigest: state.recovery.configDigest, runtime: state.daemonRuntime ?? null},
|
|
179
|
+
{configDigest: "contender", runtime: null}
|
|
180
|
+
)
|
|
181
|
+
} catch (error) {
|
|
182
|
+
contenderError = error
|
|
183
|
+
}
|
|
184
|
+
if (contenderPrepared) await contender.abortOwnerReplacement(contenderPrepared.replacementId)
|
|
185
|
+
contender.disconnect()
|
|
186
|
+
contender = undefined
|
|
187
|
+
continueRestore.resolve(undefined)
|
|
188
|
+
await assert.rejects(replacementPromise, /injected reconstruction stop after fence audit/)
|
|
189
|
+
replacementPromise = undefined
|
|
190
|
+
assert.match(mutationError instanceof Error ? mutationError.message : "", /fenced while an owner replacement is prepared/)
|
|
191
|
+
assert.match(contenderError instanceof Error ? contenderError.message : "", /another owner replacement candidate is already prepared/i)
|
|
192
|
+
assert.notEqual(JSON.parse(await fs.readFile(statePath, "utf8")).recovery.guardian.socketPath, `${statePath}.split3-guardian.sock`)
|
|
193
|
+
assert.equal((await sendControlCommand({command: {command: "status"}, path: socketPath})).daemonPid, owner.pid)
|
|
194
|
+
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
|
|
195
|
+
|
|
196
|
+
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
197
|
+
await shutdown
|
|
198
|
+
} finally {
|
|
199
|
+
continueRestore.resolve(undefined)
|
|
200
|
+
contender?.disconnect()
|
|
201
|
+
await replacementPromise?.catch(() => undefined)
|
|
202
|
+
candidate?.guardian?.disconnect()
|
|
203
|
+
if (owner && owner.exitCode === null && owner.signalCode === null) {
|
|
204
|
+
const exited = once(owner, "exit")
|
|
205
|
+
|
|
206
|
+
owner.kill("SIGKILL")
|
|
207
|
+
await exited
|
|
208
|
+
}
|
|
209
|
+
partialGuardian?.kill("SIGTERM")
|
|
210
|
+
if (partialGuardian && partialGuardian.exitCode === null && partialGuardian.signalCode === null) await once(partialGuardian, "exit")
|
|
211
|
+
if (backendGuardianIdentity) {
|
|
212
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8").catch(() => "null"))
|
|
213
|
+
|
|
214
|
+
if (state?.recovery?.guardian?.socketPath === partialSocketPath) {
|
|
215
|
+
state.recovery.guardian = backendGuardianIdentity
|
|
216
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
await stopGuardian(statePath)
|
|
220
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
221
|
+
}
|
|
222
|
+
})
|
|
223
|
+
|
|
224
|
+
test("partial guardian replacement persists the coordinator only after ownership confirmation", async () => {
|
|
225
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-persist-"))
|
|
226
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
227
|
+
const statePath = path.join(root, "state.json")
|
|
228
|
+
const configPath = path.join(root, "rollbridge.cjs")
|
|
229
|
+
const releasePath = path.join(root, "v1")
|
|
230
|
+
const partialSocketPath = path.join(root, "partial-guardian.sock")
|
|
231
|
+
const ownershipConfirmationStarted = deferred()
|
|
232
|
+
const continueOwnershipConfirmation = deferred()
|
|
233
|
+
let owner
|
|
234
|
+
let partialGuardian
|
|
235
|
+
let candidate = /** @type {RollbridgeDaemon | undefined} */ (undefined)
|
|
236
|
+
let backendGuardianIdentity
|
|
237
|
+
let replacementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
238
|
+
|
|
239
|
+
try {
|
|
240
|
+
await fs.mkdir(releasePath)
|
|
241
|
+
await makeFifo(path.join(releasePath, "worker.fifo"))
|
|
242
|
+
await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
243
|
+
owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
|
|
244
|
+
await waitForLog(owner, "control socket listening")
|
|
245
|
+
assert.ok(owner.pid)
|
|
246
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: socketPath})
|
|
247
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
248
|
+
|
|
249
|
+
backendGuardianIdentity = {...state.recovery.guardian}
|
|
250
|
+
partialGuardian = await startPartialGuardian({
|
|
251
|
+
backendPath: backendGuardianIdentity.socketPath,
|
|
252
|
+
mode: "partial",
|
|
253
|
+
socketPath: partialSocketPath,
|
|
254
|
+
token: backendGuardianIdentity.token
|
|
255
|
+
})
|
|
256
|
+
state.recovery.guardian = {...backendGuardianIdentity, pid: partialGuardian.pid, socketPath: partialSocketPath}
|
|
257
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
258
|
+
const replacement = new RollbridgeDaemon({
|
|
259
|
+
config: normalizeConfig(config({controlPath: socketPath, extraCompanion: false, statePath})),
|
|
260
|
+
configPath,
|
|
261
|
+
legacyIncumbentPid: owner.pid,
|
|
262
|
+
logger: () => {}
|
|
263
|
+
})
|
|
264
|
+
const restoreOwnerState = replacement.restoreOwnerState.bind(replacement)
|
|
265
|
+
|
|
266
|
+
candidate = replacement
|
|
267
|
+
replacement.restoreOwnerState = async (...args) => {
|
|
268
|
+
await restoreOwnerState(...args)
|
|
269
|
+
const coordinator = replacement.guardian
|
|
270
|
+
|
|
271
|
+
if (!coordinator) throw new Error("Partial guardian persistence fixture is missing its coordinator")
|
|
272
|
+
const completeLegacyOwnerClaim = coordinator.completeLegacyOwnerClaim.bind(coordinator)
|
|
273
|
+
|
|
274
|
+
coordinator.completeLegacyOwnerClaim = async (replacementId) => {
|
|
275
|
+
ownershipConfirmationStarted.resolve(undefined)
|
|
276
|
+
await continueOwnershipConfirmation.promise
|
|
277
|
+
await completeLegacyOwnerClaim(replacementId)
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
replacementPromise = replacement.replaceIncompatibleOwner()
|
|
281
|
+
void replacementPromise.catch(() => {})
|
|
282
|
+
await ownershipConfirmationStarted.promise
|
|
283
|
+
assert.notEqual(
|
|
284
|
+
JSON.parse(await fs.readFile(statePath, "utf8")).recovery.guardian.socketPath,
|
|
285
|
+
`${statePath}.split3-guardian.sock`,
|
|
286
|
+
"durable state must not name the coordinator before its legacy ownership claim is confirmed"
|
|
287
|
+
)
|
|
288
|
+
continueOwnershipConfirmation.resolve(undefined)
|
|
289
|
+
await replacementPromise
|
|
290
|
+
replacementPromise = undefined
|
|
291
|
+
const shutdown = replacement.shutdown()
|
|
292
|
+
|
|
293
|
+
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
294
|
+
await shutdown
|
|
295
|
+
candidate = undefined
|
|
296
|
+
} finally {
|
|
297
|
+
continueOwnershipConfirmation.resolve(undefined)
|
|
298
|
+
await replacementPromise?.catch(() => undefined)
|
|
299
|
+
if (candidate?.controlServer || candidate?.proxyServer) {
|
|
300
|
+
const shutdown = candidate.shutdown().catch(() => undefined)
|
|
301
|
+
|
|
302
|
+
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n").catch(() => undefined)
|
|
303
|
+
await shutdown
|
|
304
|
+
}
|
|
305
|
+
candidate?.guardian?.disconnect()
|
|
306
|
+
if (owner && owner.exitCode === null && owner.signalCode === null) {
|
|
307
|
+
const exited = once(owner, "exit")
|
|
308
|
+
|
|
309
|
+
owner.kill("SIGKILL")
|
|
310
|
+
await exited
|
|
311
|
+
}
|
|
312
|
+
partialGuardian?.kill("SIGTERM")
|
|
313
|
+
if (partialGuardian && partialGuardian.exitCode === null && partialGuardian.signalCode === null) await once(partialGuardian, "exit")
|
|
314
|
+
if (backendGuardianIdentity) {
|
|
315
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8").catch(() => "null"))
|
|
316
|
+
|
|
317
|
+
if (state?.recovery?.guardian?.socketPath === partialSocketPath) {
|
|
318
|
+
state.recovery.guardian = backendGuardianIdentity
|
|
319
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
await stopGuardian(statePath)
|
|
323
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
324
|
+
}
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
test("failed partial upgrade resumes an incumbent retired release drain", {timeout: 5000}, async () => {
|
|
328
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-drain-"))
|
|
329
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
330
|
+
const statePath = path.join(root, "state.json")
|
|
331
|
+
const configPath = path.join(root, "rollbridge.cjs")
|
|
332
|
+
const v1Path = path.join(root, "v1")
|
|
333
|
+
const v2Path = path.join(root, "v2")
|
|
334
|
+
const partialSocketPath = path.join(root, "partial-guardian.sock")
|
|
335
|
+
let owner
|
|
336
|
+
let partialGuardian
|
|
337
|
+
let retainedConnection
|
|
338
|
+
let candidate = /** @type {RollbridgeDaemon | undefined} */ (undefined)
|
|
339
|
+
let backendGuardianIdentity
|
|
340
|
+
|
|
341
|
+
try {
|
|
342
|
+
await Promise.all([fs.mkdir(v1Path), fs.mkdir(v2Path)])
|
|
343
|
+
await Promise.all([makeFifo(path.join(v1Path, "worker.fifo")), makeFifo(path.join(v2Path, "worker.fifo"))])
|
|
344
|
+
await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
345
|
+
owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
|
|
346
|
+
await waitForLog(owner, "control socket listening")
|
|
347
|
+
assert.ok(owner.pid)
|
|
348
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: socketPath})
|
|
349
|
+
const v1Status = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
350
|
+
const proxyPort = /** @type {{port?: number}} */ (v1Status.proxy).port
|
|
351
|
+
|
|
352
|
+
if (typeof proxyPort !== "number") throw new Error("Partial guardian drain fixture proxy is missing its bound port")
|
|
353
|
+
retainedConnection = await openWebSocket(proxyPort)
|
|
354
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: socketPath})
|
|
355
|
+
const draining = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
356
|
+
|
|
357
|
+
assert.equal(releaseState(draining, "v1"), "draining")
|
|
358
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
359
|
+
|
|
360
|
+
backendGuardianIdentity = {...state.recovery.guardian}
|
|
361
|
+
partialGuardian = await startPartialGuardian({
|
|
362
|
+
backendPath: backendGuardianIdentity.socketPath,
|
|
363
|
+
mode: "wrong-provenance",
|
|
364
|
+
socketPath: partialSocketPath,
|
|
365
|
+
token: backendGuardianIdentity.token
|
|
366
|
+
})
|
|
367
|
+
state.recovery.guardian = {...backendGuardianIdentity, pid: partialGuardian.pid, socketPath: partialSocketPath}
|
|
368
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
369
|
+
const replacement = new RollbridgeDaemon({
|
|
370
|
+
config: normalizeConfig(config({controlPath: socketPath, extraCompanion: false, statePath})),
|
|
371
|
+
configPath,
|
|
372
|
+
legacyIncumbentPid: owner.pid,
|
|
373
|
+
logger: () => {}
|
|
374
|
+
})
|
|
375
|
+
|
|
376
|
+
candidate = replacement
|
|
377
|
+
await assert.rejects(() => replacement.replaceIncompatibleOwner(), /provenance mismatch/)
|
|
378
|
+
assert.equal(owner.exitCode, null)
|
|
379
|
+
assert.equal(owner.signalCode, null)
|
|
380
|
+
const releaseDrained = waitForLog(owner, "release drained")
|
|
381
|
+
|
|
382
|
+
retainedConnection.destroy()
|
|
383
|
+
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
384
|
+
await releaseDrained
|
|
385
|
+
assert.equal(releaseState(await sendControlCommand({command: {command: "status"}, path: socketPath}), "v1"), "stopped")
|
|
386
|
+
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
|
|
387
|
+
|
|
388
|
+
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
389
|
+
await shutdown
|
|
390
|
+
} finally {
|
|
391
|
+
retainedConnection?.destroy()
|
|
392
|
+
candidate?.guardian?.disconnect()
|
|
393
|
+
if (owner && owner.exitCode === null && owner.signalCode === null) owner.kill("SIGKILL")
|
|
394
|
+
partialGuardian?.kill("SIGTERM")
|
|
395
|
+
if (partialGuardian && partialGuardian.exitCode === null && partialGuardian.signalCode === null) await once(partialGuardian, "exit")
|
|
396
|
+
if (backendGuardianIdentity) {
|
|
397
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8").catch(() => "null"))
|
|
398
|
+
|
|
399
|
+
if (state?.recovery?.guardian?.socketPath === partialSocketPath) {
|
|
400
|
+
state.recovery.guardian = backendGuardianIdentity
|
|
401
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
await stopGuardian(statePath)
|
|
405
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
406
|
+
}
|
|
407
|
+
})
|
|
408
|
+
|
|
409
|
+
test("partial guardian classification failures preserve the incumbent, children, and retained stream", async (t) => {
|
|
410
|
+
for (const fault of ["malformed-capability", "wrong-pid", "wrong-provenance"]) {
|
|
411
|
+
await t.test(fault, async () => {
|
|
412
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), `rollbridge-owner-replacement-partial-${fault}-`))
|
|
413
|
+
const socketPath = path.join(root, "rollbridge.sock")
|
|
414
|
+
const statePath = path.join(root, "state.json")
|
|
415
|
+
const configPath = path.join(root, "rollbridge.cjs")
|
|
416
|
+
const releasePath = path.join(root, "v1")
|
|
417
|
+
const partialSocketPath = path.join(root, "partial-guardian.sock")
|
|
418
|
+
let owner
|
|
419
|
+
let partialGuardian
|
|
420
|
+
let retainedConnection
|
|
421
|
+
let candidate = /** @type {RollbridgeDaemon | undefined} */ (undefined)
|
|
422
|
+
|
|
423
|
+
try {
|
|
424
|
+
await fs.mkdir(releasePath)
|
|
425
|
+
await makeFifo(path.join(releasePath, "worker.fifo"))
|
|
426
|
+
await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
|
|
427
|
+
owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
|
|
428
|
+
await waitForLog(owner, "control socket listening")
|
|
429
|
+
assert.ok(owner.pid)
|
|
430
|
+
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: socketPath})
|
|
431
|
+
const before = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
432
|
+
const processPids = [releaseProcessPid(before, "v1", "worker"), releaseProcessPid(before, "v1", "web")]
|
|
433
|
+
const proxyPort = /** @type {{port?: number}} */ (before.proxy).port
|
|
434
|
+
const state = JSON.parse(await fs.readFile(statePath, "utf8"))
|
|
435
|
+
const backendIdentity = {...state.recovery.guardian}
|
|
436
|
+
|
|
437
|
+
if (typeof proxyPort !== "number") throw new Error("Partial guardian failure fixture proxy is missing its bound port")
|
|
438
|
+
partialGuardian = await startPartialGuardian({
|
|
439
|
+
backendPath: backendIdentity.socketPath,
|
|
440
|
+
mode: fault === "wrong-provenance" ? fault : fault === "malformed-capability" ? fault : "partial",
|
|
441
|
+
socketPath: partialSocketPath,
|
|
442
|
+
token: backendIdentity.token
|
|
443
|
+
})
|
|
444
|
+
state.recovery.guardian = {
|
|
445
|
+
...backendIdentity,
|
|
446
|
+
pid: fault === "wrong-pid" ? backendIdentity.pid : partialGuardian.pid,
|
|
447
|
+
socketPath: partialSocketPath
|
|
448
|
+
}
|
|
449
|
+
await fs.writeFile(statePath, `${JSON.stringify(state)}\n`)
|
|
450
|
+
retainedConnection = await openWebSocket(proxyPort)
|
|
451
|
+
let retainedClosed = false
|
|
452
|
+
|
|
453
|
+
retainedConnection.once("close", () => { retainedClosed = true })
|
|
454
|
+
const replacement = new RollbridgeDaemon({
|
|
455
|
+
config: normalizeConfig(config({controlPath: socketPath, extraCompanion: false, statePath})),
|
|
456
|
+
configPath,
|
|
457
|
+
legacyIncumbentPid: owner.pid,
|
|
458
|
+
logger: () => {}
|
|
459
|
+
})
|
|
460
|
+
candidate = replacement
|
|
461
|
+
const expected = fault === "malformed-capability"
|
|
462
|
+
? /invalid owner-replacement capability response/
|
|
463
|
+
: fault === "wrong-pid"
|
|
464
|
+
? /does not own socket|does not match the retained guardian command and socket/
|
|
465
|
+
: /provenance mismatch/
|
|
466
|
+
|
|
467
|
+
await assert.rejects(() => replacement.replaceIncompatibleOwner(), expected)
|
|
468
|
+
assert.equal(owner.exitCode, null)
|
|
469
|
+
assert.equal(owner.signalCode, null)
|
|
470
|
+
assert.equal(retainedClosed, false)
|
|
471
|
+
assert.equal(retainedConnection.destroyed, false)
|
|
472
|
+
for (const pid of processPids) assert.doesNotThrow(() => process.kill(pid, 0))
|
|
473
|
+
assert.equal(releaseProcessPid(await sendControlCommand({command: {command: "status"}, path: socketPath}), "v1", "worker"), processPids[0])
|
|
474
|
+
await assert.rejects(fs.access(`${statePath}.split3-guardian.sock`), {code: "ENOENT"})
|
|
475
|
+
|
|
476
|
+
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
|
|
477
|
+
|
|
478
|
+
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
479
|
+
await shutdown
|
|
480
|
+
} finally {
|
|
481
|
+
retainedConnection?.destroy()
|
|
482
|
+
candidate?.guardian?.disconnect()
|
|
483
|
+
if (owner && owner.exitCode === null && owner.signalCode === null) owner.kill("SIGKILL")
|
|
484
|
+
partialGuardian?.kill("SIGTERM")
|
|
485
|
+
if (partialGuardian && partialGuardian.exitCode === null && partialGuardian.signalCode === null) await once(partialGuardian, "exit")
|
|
486
|
+
await stopGuardian(statePath)
|
|
487
|
+
await fs.rm(root, {force: true, recursive: true})
|
|
488
|
+
}
|
|
489
|
+
})
|
|
490
|
+
}
|
|
491
|
+
})
|
|
22
492
|
|
|
23
493
|
test("first pre-split package upgrade is explicitly disruptive and later replacements are atomic", async () => {
|
|
24
494
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-legacy-"))
|
|
@@ -81,7 +551,7 @@ test("first pre-split package upgrade is explicitly disruptive and later replace
|
|
|
81
551
|
assert.deepEqual(JSON.parse(firstUpgrade.stdout).ownerTransition, {
|
|
82
552
|
disruptive: true,
|
|
83
553
|
mode: "legacy-first-upgrade",
|
|
84
|
-
reason: "
|
|
554
|
+
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
85
555
|
})
|
|
86
556
|
await interrupted
|
|
87
557
|
const bridged = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
@@ -89,7 +559,7 @@ test("first pre-split package upgrade is explicitly disruptive and later replace
|
|
|
89
559
|
assert.deepEqual(bridged.ownerTransition, {
|
|
90
560
|
disruptive: true,
|
|
91
561
|
mode: "legacy-first-upgrade",
|
|
92
|
-
reason: "
|
|
562
|
+
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
93
563
|
})
|
|
94
564
|
assert.equal(releaseProcessPid(bridged, "v1", "worker"), legacyWorkerPid)
|
|
95
565
|
assert.equal(bridged.activeReleaseId, "v1")
|
|
@@ -970,6 +1440,32 @@ async function runEnsureDaemon({configPath, daemonPidPath, logPath, packagePath,
|
|
|
970
1440
|
])
|
|
971
1441
|
}
|
|
972
1442
|
|
|
1443
|
+
/**
|
|
1444
|
+
* Starts an authenticated protocol proxy that exposes the exact partial guardian surface.
|
|
1445
|
+
* @param {{backendPath: string, mode: string, socketPath: string, token: string}} options - Fixture identity.
|
|
1446
|
+
* @returns {Promise<import("node:child_process").ChildProcess>} Ready guardian proxy.
|
|
1447
|
+
*/
|
|
1448
|
+
async function startPartialGuardian({backendPath, mode, socketPath, token}) {
|
|
1449
|
+
const child = spawn(process.execPath, [partialGuardianPath, socketPath, backendPath, mode], {
|
|
1450
|
+
stdio: ["ignore", "ignore", "ignore", "ipc"]
|
|
1451
|
+
})
|
|
1452
|
+
|
|
1453
|
+
await new Promise((resolve, reject) => {
|
|
1454
|
+
child.once("error", reject)
|
|
1455
|
+
child.once("exit", (code) => reject(new Error(`Partial guardian exited before readiness with status ${code}`)))
|
|
1456
|
+
child.once("message", (message) => {
|
|
1457
|
+
if (message && typeof message === "object" && "error" in message) reject(new Error(String(message.error)))
|
|
1458
|
+
else resolve(undefined)
|
|
1459
|
+
})
|
|
1460
|
+
child.send({token}, (error) => {
|
|
1461
|
+
if (error) reject(error)
|
|
1462
|
+
})
|
|
1463
|
+
})
|
|
1464
|
+
if (child.connected) await once(child, "disconnect")
|
|
1465
|
+
assert.ok(child.pid)
|
|
1466
|
+
return child
|
|
1467
|
+
}
|
|
1468
|
+
|
|
973
1469
|
/**
|
|
974
1470
|
* @param {net.Server} server - Unix server.
|
|
975
1471
|
* @param {string} socketPath - Unix socket path.
|
|
@@ -1013,6 +1509,27 @@ function releaseProcessPid(status, releaseId, processId) {
|
|
|
1013
1509
|
return pid
|
|
1014
1510
|
}
|
|
1015
1511
|
|
|
1512
|
+
/**
|
|
1513
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} status - Daemon status.
|
|
1514
|
+
* @param {string} releaseId - Release identity.
|
|
1515
|
+
* @returns {string} Release lifecycle state.
|
|
1516
|
+
*/
|
|
1517
|
+
function releaseState(status, releaseId) {
|
|
1518
|
+
const releases = /** @type {{releaseId: string, state: string}[]} */ (status.releases)
|
|
1519
|
+
const state = releases.find((release) => release.releaseId === releaseId)?.state
|
|
1520
|
+
|
|
1521
|
+
if (!state) throw new Error(`Missing release state for ${releaseId}`)
|
|
1522
|
+
return state
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
/** @returns {{promise: Promise<void>, resolve: (value: void) => void}} Controllable event barrier. */
|
|
1526
|
+
function deferred() {
|
|
1527
|
+
let resolve = /** @type {(value: void) => void} */ (() => {})
|
|
1528
|
+
const promise = new Promise((promiseResolve) => { resolve = promiseResolve })
|
|
1529
|
+
|
|
1530
|
+
return {promise, resolve}
|
|
1531
|
+
}
|
|
1532
|
+
|
|
1016
1533
|
/**
|
|
1017
1534
|
* @param {string} command - Executable.
|
|
1018
1535
|
* @param {string[]} args - Arguments.
|