rollbridge 0.1.39 → 0.1.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +12 -1
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +16 -2
- package/changelog.d/20260830055159-guardian-daemon-restart.md +2 -0
- package/docs/cli.md +23 -9
- package/docs/config.md +37 -13
- package/docs/logging.md +4 -3
- package/docs/troubleshooting.md +7 -5
- package/examples/tensorbuzz.com.js +5 -2
- package/package.json +1 -1
- package/src/cli.js +118 -24
- package/src/config.js +35 -8
- package/src/daemon.js +822 -152
- package/src/guardian-client.js +121 -16
- package/src/managed-process.js +117 -15
- package/src/process-guardian.js +734 -43
- package/src/release-group.js +45 -7
- package/test/completion.test.js +4 -2
- package/test/config-examples.test.js +1 -0
- package/test/config-validation.test.js +22 -0
- package/test/fixtures/guardian-recovery-owner.js +86 -0
- package/test/fixtures/pre-split3-process-guardian.js +14 -0
- package/test/guardian-client.test.js +1420 -62
- package/test/managed-process.test.js +163 -7
- package/test/owner-recovery.test.js +575 -73
- package/test/owner-replacement.test.js +525 -28
- package/test/release-group.test.js +19 -2
- package/test/release-runtime-retention.test.js +121 -4
- package/test/rollbridge.test.js +263 -13
- package/test/support/process.js +41 -0
|
@@ -9,13 +9,16 @@ import path from "node:path"
|
|
|
9
9
|
import test from "node:test"
|
|
10
10
|
import {fileURLToPath} from "node:url"
|
|
11
11
|
import GuardianClient from "../src/guardian-client.js"
|
|
12
|
+
import {waitForProcessExit} from "./support/process.js"
|
|
12
13
|
|
|
13
14
|
const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
|
|
15
|
+
const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
|
|
14
16
|
|
|
15
17
|
test("guardian bootstrap capability is absent from process argv", async () => {
|
|
16
18
|
const fixture = await createGuardian()
|
|
17
19
|
|
|
18
20
|
try {
|
|
21
|
+
assert.deepEqual(await fixture.client.capabilities(), {daemonRecovery: 1, generationReactivation: 1})
|
|
19
22
|
const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
|
|
20
23
|
const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
|
|
21
24
|
const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
|
|
@@ -67,6 +70,106 @@ test("guardian runs a strict activation lifecycle command for the exact register
|
|
|
67
70
|
}
|
|
68
71
|
})
|
|
69
72
|
|
|
73
|
+
test("client reactivates a retained process through a guardian without the reactivation command", async () => {
|
|
74
|
+
const fixture = await createGuardian()
|
|
75
|
+
const lifecyclePath = path.join(fixture.root, "lifecycle.log")
|
|
76
|
+
const processInstance = fixture.client.process("compatible-reactivation", {
|
|
77
|
+
...definition("compatible-reactivation"),
|
|
78
|
+
lifecycle: {
|
|
79
|
+
activateCommand: `printf 'activate\n' >> ${JSON.stringify(lifecyclePath)}`,
|
|
80
|
+
drainTimeoutMs: 0,
|
|
81
|
+
quietCommand: `printf 'retire\n' >> ${JSON.stringify(lifecyclePath)}`
|
|
82
|
+
},
|
|
83
|
+
shouldRestart: () => true
|
|
84
|
+
})
|
|
85
|
+
const request = fixture.client.request.bind(fixture.client)
|
|
86
|
+
|
|
87
|
+
fixture.client.request = async command => {
|
|
88
|
+
if (command.command === "reactivate") throw new Error("Unknown guardian command: reactivate")
|
|
89
|
+
return await request(command)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
try {
|
|
93
|
+
await processInstance.start()
|
|
94
|
+
await processInstance.activateStrict()
|
|
95
|
+
const pid = processInstance.status().pid
|
|
96
|
+
|
|
97
|
+
await processInstance.quiesceStrict()
|
|
98
|
+
await processInstance.reactivateStrict()
|
|
99
|
+
|
|
100
|
+
assert.equal(processInstance.status().pid, pid)
|
|
101
|
+
assert.equal(processInstance.status().state, "running")
|
|
102
|
+
assert.equal(processInstance.status().lifecycleRole, "active")
|
|
103
|
+
assert.equal(await fs.readFile(lifecyclePath, "utf8"), "activate\nretire\nactivate\n")
|
|
104
|
+
|
|
105
|
+
const restarted = once(processInstance, "started")
|
|
106
|
+
|
|
107
|
+
assert.ok(pid)
|
|
108
|
+
process.kill(-pid, "SIGKILL")
|
|
109
|
+
await restarted
|
|
110
|
+
assert.notEqual(processInstance.status().pid, pid)
|
|
111
|
+
assert.equal(processInstance.status().state, "running")
|
|
112
|
+
assert.equal(processInstance.status().lifecycleRole, "active")
|
|
113
|
+
assert.equal(await fs.readFile(lifecyclePath, "utf8"), "activate\nretire\nactivate\nactivate\n")
|
|
114
|
+
} finally {
|
|
115
|
+
await cleanupGuardian(fixture)
|
|
116
|
+
}
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
test("client reverses a worker quiet hook through a pre-reactivation guardian", async () => {
|
|
120
|
+
const fixture = await createGuardian()
|
|
121
|
+
const lifecyclePath = path.join(fixture.root, "worker-lifecycle.log")
|
|
122
|
+
const processInstance = fixture.client.process("compatible-worker-reactivation", {
|
|
123
|
+
...definition("compatible-worker-reactivation"),
|
|
124
|
+
lifecycle: {
|
|
125
|
+
drainTimeoutMs: 0,
|
|
126
|
+
quietCommand: `printf 'quiet\n' >> ${JSON.stringify(lifecyclePath)}`,
|
|
127
|
+
reactivateCommand: `printf 'resume\n' >> ${JSON.stringify(lifecyclePath)}`
|
|
128
|
+
},
|
|
129
|
+
shouldRestart: () => true
|
|
130
|
+
})
|
|
131
|
+
const request = fixture.client.request.bind(fixture.client)
|
|
132
|
+
|
|
133
|
+
fixture.client.request = async command => {
|
|
134
|
+
if (command.command === "reactivate-with-command") throw new Error("Unknown guardian command: reactivate-with-command")
|
|
135
|
+
return await request(command)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
try {
|
|
139
|
+
await processInstance.start()
|
|
140
|
+
const pid = processInstance.status().pid
|
|
141
|
+
|
|
142
|
+
await processInstance.quiesceStrict()
|
|
143
|
+
await processInstance.reactivateStrict()
|
|
144
|
+
|
|
145
|
+
assert.equal(processInstance.status().pid, pid)
|
|
146
|
+
assert.equal(processInstance.status().state, "running")
|
|
147
|
+
assert.equal(processInstance.status().lifecycleRole, "active")
|
|
148
|
+
assert.equal(await fs.readFile(lifecyclePath, "utf8"), "quiet\nresume\n")
|
|
149
|
+
} finally {
|
|
150
|
+
await cleanupGuardian(fixture)
|
|
151
|
+
}
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
test("guardian atomically updates process provenance with private owner state", async () => {
|
|
155
|
+
const fixture = await createGuardian()
|
|
156
|
+
const processInstance = fixture.client.process("service", definition("service"))
|
|
157
|
+
const previousOwnerState = {authority: null, serviceReleaseIds: {service: "v1"}}
|
|
158
|
+
const nextOwnerState = {authority: null, serviceReleaseIds: {service: "v2"}}
|
|
159
|
+
|
|
160
|
+
try {
|
|
161
|
+
await fixture.client.publishOwnerState(previousOwnerState)
|
|
162
|
+
await processInstance.start()
|
|
163
|
+
const previousProvenance = (await fixture.client.inventory())[0]?.provenance
|
|
164
|
+
|
|
165
|
+
await processInstance.updateDefinition({...definition("service"), env: {RELEASE: "v2"}}, nextOwnerState)
|
|
166
|
+
assert.deepEqual(await fixture.client.ownerState(), nextOwnerState)
|
|
167
|
+
assert.notEqual((await fixture.client.inventory())[0]?.provenance, previousProvenance)
|
|
168
|
+
} finally {
|
|
169
|
+
await cleanupGuardian(fixture)
|
|
170
|
+
}
|
|
171
|
+
})
|
|
172
|
+
|
|
70
173
|
test("guardian forwards each retained output line to its exact process proxy", async () => {
|
|
71
174
|
const fixture = await createGuardian()
|
|
72
175
|
const marker = "guardian-output-ready"
|
|
@@ -75,19 +178,102 @@ test("guardian forwards each retained output line to its exact process proxy", a
|
|
|
75
178
|
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`console.log(${JSON.stringify(marker)})`)}`
|
|
76
179
|
})
|
|
77
180
|
const logged = once(processInstance, "log")
|
|
181
|
+
const forwarded = fixture.client.waitForEvent("process-log")
|
|
78
182
|
const exitedFirst = once(processInstance, "exit").then(() => { throw new Error("Guardian process exited before forwarding retained output") })
|
|
79
183
|
|
|
80
184
|
try {
|
|
81
185
|
await processInstance.start()
|
|
82
186
|
const [entry] = await Promise.race([logged, exitedFirst])
|
|
187
|
+
const event = await forwarded
|
|
83
188
|
|
|
84
189
|
assert.equal(entry.line, marker)
|
|
190
|
+
assert.equal(event.status, undefined, "log events must not resend the complete retained process status")
|
|
85
191
|
assert.ok(processInstance.status().logs.some((candidate) => candidate.line === marker))
|
|
86
192
|
} finally {
|
|
87
193
|
await cleanupGuardian(fixture)
|
|
88
194
|
}
|
|
89
195
|
})
|
|
90
196
|
|
|
197
|
+
test("guardian delivers the final process status after dropping logs for a backpressured client", async () => {
|
|
198
|
+
const fixture = await createGuardian()
|
|
199
|
+
const gatePath = path.join(fixture.root, "write-output")
|
|
200
|
+
const script = `const fs = require("node:fs"); const {once} = require("node:events"); (async () => { while (!fs.existsSync(${JSON.stringify(gatePath)})) await new Promise((resolve) => setTimeout(resolve, 5)); const line = "x".repeat(1024) + "\\n"; for (let index = 0; index < 8192; index += 1) if (!process.stdout.write(line)) await once(process.stdout, "drain"); })()`
|
|
201
|
+
const processInstance = fixture.client.process("backpressured-output", {
|
|
202
|
+
...definition("backpressured-output"),
|
|
203
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(script)}`
|
|
204
|
+
})
|
|
205
|
+
const socket = fixture.client.socket
|
|
206
|
+
|
|
207
|
+
assert.ok(socket)
|
|
208
|
+
try {
|
|
209
|
+
await processInstance.start()
|
|
210
|
+
const pid = processInstance.status().pid
|
|
211
|
+
|
|
212
|
+
assert.ok(pid)
|
|
213
|
+
const finalStatus = fixture.client.waitForEvent("process")
|
|
214
|
+
|
|
215
|
+
socket.pause()
|
|
216
|
+
await fs.writeFile(gatePath, "write\n")
|
|
217
|
+
await waitForProcessExit(pid, 10000)
|
|
218
|
+
socket.resume()
|
|
219
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
220
|
+
let timeout
|
|
221
|
+
|
|
222
|
+
try {
|
|
223
|
+
await Promise.race([
|
|
224
|
+
finalStatus,
|
|
225
|
+
new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Guardian did not flush the final process status after backpressure")), 3000) })
|
|
226
|
+
])
|
|
227
|
+
} finally {
|
|
228
|
+
clearTimeout(timeout)
|
|
229
|
+
}
|
|
230
|
+
assert.equal(processInstance.status().state, "failed")
|
|
231
|
+
} finally {
|
|
232
|
+
socket.resume()
|
|
233
|
+
await cleanupGuardian(fixture)
|
|
234
|
+
}
|
|
235
|
+
})
|
|
236
|
+
|
|
237
|
+
test("guardian resynchronizes retained logs after dropping output for a backpressured client", async () => {
|
|
238
|
+
const fixture = await createGuardian()
|
|
239
|
+
const gatePath = path.join(fixture.root, "write-retained-output")
|
|
240
|
+
const completedPath = path.join(fixture.root, "retained-output-complete")
|
|
241
|
+
const payload = "x".repeat(1024)
|
|
242
|
+
const finalLine = `8191:${payload}`
|
|
243
|
+
const script = `const fs = require("node:fs"); const {once} = require("node:events"); (async () => { while (!fs.existsSync(${JSON.stringify(gatePath)})) await new Promise((resolve) => setTimeout(resolve, 5)); for (let index = 0; index < 8192; index += 1) if (!process.stdout.write(index + ":" + ${JSON.stringify(payload)} + "\\n")) await once(process.stdout, "drain"); fs.writeFileSync(${JSON.stringify(completedPath)}, "complete\\n"); setInterval(() => {}, 1000); })()`
|
|
244
|
+
const processInstance = fixture.client.process("backpressured-retained-output", {
|
|
245
|
+
...definition("backpressured-retained-output"),
|
|
246
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(script)}`
|
|
247
|
+
})
|
|
248
|
+
const socket = fixture.client.socket
|
|
249
|
+
|
|
250
|
+
assert.ok(socket)
|
|
251
|
+
try {
|
|
252
|
+
await processInstance.start()
|
|
253
|
+
socket.pause()
|
|
254
|
+
await fs.writeFile(gatePath, "write\n")
|
|
255
|
+
await waitForFileText(completedPath)
|
|
256
|
+
const resynchronized = fixture.client.waitForEvent("status")
|
|
257
|
+
|
|
258
|
+
socket.resume()
|
|
259
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
260
|
+
let timeout
|
|
261
|
+
|
|
262
|
+
try {
|
|
263
|
+
await Promise.race([
|
|
264
|
+
resynchronized,
|
|
265
|
+
new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Guardian did not resynchronize retained output after backpressure")), 3000) })
|
|
266
|
+
])
|
|
267
|
+
} finally {
|
|
268
|
+
clearTimeout(timeout)
|
|
269
|
+
}
|
|
270
|
+
assert.equal(processInstance.status().logs.at(-1)?.line, finalLine)
|
|
271
|
+
} finally {
|
|
272
|
+
socket.resume()
|
|
273
|
+
await cleanupGuardian(fixture)
|
|
274
|
+
}
|
|
275
|
+
})
|
|
276
|
+
|
|
91
277
|
test("guardian shutdown reports an exact owned process stop failure", async () => {
|
|
92
278
|
const fixture = await createGuardian()
|
|
93
279
|
const processInstance = fixture.client.process("broken-stop", {...definition("broken-stop"), stopSignal: "NOT_A_SIGNAL"})
|
|
@@ -153,6 +339,403 @@ test("successful guardian shutdown closes an authenticated waiting contender bef
|
|
|
153
339
|
}
|
|
154
340
|
})
|
|
155
341
|
|
|
342
|
+
test("guardian restart uses the latest accepted command and exact environment", async () => {
|
|
343
|
+
const fixture = await createGuardian()
|
|
344
|
+
const markerPath = path.join(fixture.root, "restarts.jsonl")
|
|
345
|
+
const authority = {configDigest: "same", runtime: null}
|
|
346
|
+
const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
347
|
+
|
|
348
|
+
try {
|
|
349
|
+
await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "old", 250))
|
|
350
|
+
fixture.client.disconnect()
|
|
351
|
+
await replacement.connect()
|
|
352
|
+
await replacement.claimOwner(250, authority)
|
|
353
|
+
await replacement.publishOwnerState(recoveryOwnerState(authority, markerPath, "new", 40))
|
|
354
|
+
replacement.disconnect()
|
|
355
|
+
|
|
356
|
+
const [restart] = await waitForRestartRecords(markerPath, 1)
|
|
357
|
+
|
|
358
|
+
assert.deepEqual({home: restart.home, marker: restart.marker}, {home: null, marker: "new"})
|
|
359
|
+
} finally {
|
|
360
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
361
|
+
}
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
test("guardian rearms recovery after an ownerless replacement aborts", async () => {
|
|
365
|
+
const fixture = await createGuardian()
|
|
366
|
+
const markerPath = path.join(fixture.root, "replacement-abort.jsonl")
|
|
367
|
+
const authority = {configDigest: "same", runtime: null}
|
|
368
|
+
const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
369
|
+
|
|
370
|
+
try {
|
|
371
|
+
await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "accepted", 40))
|
|
372
|
+
fixture.client.disconnect()
|
|
373
|
+
await replacement.connect()
|
|
374
|
+
const prepared = await replacement.prepareOwnerReplacement(authority, authority)
|
|
375
|
+
|
|
376
|
+
await new Promise((resolve) => setTimeout(resolve, 80))
|
|
377
|
+
await replacement.abortOwnerReplacement(prepared.replacementId)
|
|
378
|
+
assert.deepEqual((await waitForRestartRecords(markerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "accepted"}])
|
|
379
|
+
} finally {
|
|
380
|
+
replacement.disconnect()
|
|
381
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
382
|
+
}
|
|
383
|
+
})
|
|
384
|
+
|
|
385
|
+
test("guardian retry backoff remains nonzero when reconnect grace is zero", async () => {
|
|
386
|
+
const fixture = await createGuardian()
|
|
387
|
+
const markerPath = path.join(fixture.root, "retry-backoff.jsonl")
|
|
388
|
+
const authority = {configDigest: "same", runtime: null}
|
|
389
|
+
|
|
390
|
+
try {
|
|
391
|
+
await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "retry", 0))
|
|
392
|
+
fixture.client.disconnect()
|
|
393
|
+
const records = await waitForRestartRecords(markerPath, 2)
|
|
394
|
+
|
|
395
|
+
assert.equal(typeof records[0]?.at, "number")
|
|
396
|
+
assert.equal(typeof records[1]?.at, "number")
|
|
397
|
+
assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `failed owner recovery retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
|
|
398
|
+
} finally {
|
|
399
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
400
|
+
}
|
|
401
|
+
})
|
|
402
|
+
|
|
403
|
+
test("a retired guardian-started owner does not block recovery of its replacement", async () => {
|
|
404
|
+
const fixture = await createGuardian()
|
|
405
|
+
const firstMarkerPath = path.join(fixture.root, "first-owner.pid")
|
|
406
|
+
const secondMarkerPath = path.join(fixture.root, "second-owner.jsonl")
|
|
407
|
+
const authority = {configDigest: "same", runtime: null}
|
|
408
|
+
const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
409
|
+
let firstOwnerPid
|
|
410
|
+
|
|
411
|
+
try {
|
|
412
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, firstMarkerPath))
|
|
413
|
+
fixture.client.disconnect()
|
|
414
|
+
firstOwnerPid = Number((await waitForFileText(firstMarkerPath)).trim())
|
|
415
|
+
await replacement.connect()
|
|
416
|
+
const prepared = await replacement.prepareOwnerReplacement(authority, authority)
|
|
417
|
+
const committed = replacement.waitForEvent("replacement-committed")
|
|
418
|
+
|
|
419
|
+
await replacement.stageOwnerReplacement(prepared.replacementId, recoveryOwnerState(authority, secondMarkerPath, "replacement", 20))
|
|
420
|
+
process.kill(firstOwnerPid, "SIGUSR1")
|
|
421
|
+
await committed
|
|
422
|
+
replacement.disconnect()
|
|
423
|
+
|
|
424
|
+
assert.deepEqual((await waitForRestartRecords(secondMarkerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "replacement"}])
|
|
425
|
+
} finally {
|
|
426
|
+
replacement.disconnect()
|
|
427
|
+
if (firstOwnerPid) {
|
|
428
|
+
try { process.kill(firstOwnerPid, "SIGKILL") } catch (_error) { /* Exact retired fixture owner already exited. */ }
|
|
429
|
+
}
|
|
430
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
431
|
+
}
|
|
432
|
+
})
|
|
433
|
+
|
|
434
|
+
test("guardian terminates a restart attempt that never claims ownership", async () => {
|
|
435
|
+
const fixture = await createGuardian()
|
|
436
|
+
const markerPath = path.join(fixture.root, "hung-owner.pid")
|
|
437
|
+
const authority = {configDigest: "same", runtime: null}
|
|
438
|
+
let hungPid
|
|
439
|
+
|
|
440
|
+
try {
|
|
441
|
+
await fixture.client.publishOwnerState({
|
|
442
|
+
authority,
|
|
443
|
+
recovery: {
|
|
444
|
+
command: {
|
|
445
|
+
args: ["-e", "require('node:fs').writeFileSync(process.argv[1], String(process.pid)); setInterval(() => {}, 1000)", markerPath],
|
|
446
|
+
cwd: fixture.root,
|
|
447
|
+
env: {},
|
|
448
|
+
executable: process.execPath
|
|
449
|
+
},
|
|
450
|
+
reconnectGraceMs: 0,
|
|
451
|
+
startupTimeoutMs: 1000
|
|
452
|
+
},
|
|
453
|
+
snapshot: {activeReleaseId: null}
|
|
454
|
+
})
|
|
455
|
+
fixture.client.disconnect()
|
|
456
|
+
hungPid = Number((await waitForFileText(markerPath)).trim())
|
|
457
|
+
|
|
458
|
+
await waitForProcessExit(hungPid)
|
|
459
|
+
} finally {
|
|
460
|
+
if (hungPid) {
|
|
461
|
+
try { process.kill(hungPid, "SIGKILL") } catch (_error) { /* Exact hung fixture owner already exited. */ }
|
|
462
|
+
}
|
|
463
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
464
|
+
}
|
|
465
|
+
})
|
|
466
|
+
|
|
467
|
+
test("guardian terminates descendants when a restart leader exits before claiming ownership", async () => {
|
|
468
|
+
const fixture = await createGuardian()
|
|
469
|
+
const descendantPath = path.join(fixture.root, "early-exit-descendant.pid")
|
|
470
|
+
const authority = {configDigest: "same", runtime: null}
|
|
471
|
+
let descendantPid
|
|
472
|
+
|
|
473
|
+
try {
|
|
474
|
+
await fixture.client.publishOwnerState({
|
|
475
|
+
authority,
|
|
476
|
+
recovery: {
|
|
477
|
+
command: {
|
|
478
|
+
args: ["-e", `const {spawn} = require("node:child_process"); const fs = require("node:fs"); const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {stdio: "ignore"}); fs.writeFileSync(${JSON.stringify(descendantPath)}, String(child.pid)); child.unref()`],
|
|
479
|
+
cwd: fixture.root,
|
|
480
|
+
env: {},
|
|
481
|
+
executable: process.execPath
|
|
482
|
+
},
|
|
483
|
+
reconnectGraceMs: 0,
|
|
484
|
+
startupTimeoutMs: 1000
|
|
485
|
+
},
|
|
486
|
+
snapshot: {activeReleaseId: null}
|
|
487
|
+
})
|
|
488
|
+
fixture.client.disconnect()
|
|
489
|
+
descendantPid = Number((await waitForFileText(descendantPath)).trim())
|
|
490
|
+
|
|
491
|
+
await waitForProcessExit(descendantPid)
|
|
492
|
+
} finally {
|
|
493
|
+
if (descendantPid) {
|
|
494
|
+
try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
|
|
495
|
+
}
|
|
496
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
497
|
+
}
|
|
498
|
+
})
|
|
499
|
+
|
|
500
|
+
test("guardian terminates a restarted owner process group that claims but never becomes ready", async () => {
|
|
501
|
+
const fixture = await createGuardian()
|
|
502
|
+
const descendantPath = path.join(fixture.root, "claimed-hung-descendant.pid")
|
|
503
|
+
const markerPath = path.join(fixture.root, "claimed-hung-owner.pid")
|
|
504
|
+
const authority = {configDigest: "same", runtime: null}
|
|
505
|
+
let descendantPid
|
|
506
|
+
let ownerPid
|
|
507
|
+
|
|
508
|
+
try {
|
|
509
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
|
|
510
|
+
descendantPath,
|
|
511
|
+
ready: false,
|
|
512
|
+
startupTimeoutMs: 100
|
|
513
|
+
}))
|
|
514
|
+
fixture.client.disconnect()
|
|
515
|
+
ownerPid = Number((await waitForFileText(markerPath)).trim())
|
|
516
|
+
descendantPid = Number((await waitForFileText(descendantPath)).trim())
|
|
517
|
+
|
|
518
|
+
await Promise.all([waitForProcessExit(ownerPid), waitForProcessExit(descendantPid)])
|
|
519
|
+
} finally {
|
|
520
|
+
if (ownerPid) killExactProcessGroup(ownerPid)
|
|
521
|
+
if (descendantPid) {
|
|
522
|
+
try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
|
|
523
|
+
}
|
|
524
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
525
|
+
}
|
|
526
|
+
})
|
|
527
|
+
|
|
528
|
+
test("guardian backs off when a restarted owner exits after claiming but before readiness", async () => {
|
|
529
|
+
const fixture = await createGuardian()
|
|
530
|
+
const descendantPath = path.join(fixture.root, "post-claim-descendant.pid")
|
|
531
|
+
const markerPath = path.join(fixture.root, "post-claim-exit.pid")
|
|
532
|
+
const startedLogPath = path.join(fixture.root, "post-claim-starts.jsonl")
|
|
533
|
+
const authority = {configDigest: "same", runtime: null}
|
|
534
|
+
let descendantPid
|
|
535
|
+
|
|
536
|
+
try {
|
|
537
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
|
|
538
|
+
descendantPath,
|
|
539
|
+
exitAfterClaim: true,
|
|
540
|
+
startedLogPath
|
|
541
|
+
}))
|
|
542
|
+
fixture.client.disconnect()
|
|
543
|
+
descendantPid = Number((await waitForFileText(descendantPath)).trim())
|
|
544
|
+
const records = await waitForRestartRecords(startedLogPath, 2)
|
|
545
|
+
|
|
546
|
+
await waitForProcessExit(descendantPid)
|
|
547
|
+
assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `post-claim failure retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
|
|
548
|
+
} finally {
|
|
549
|
+
if (descendantPid) {
|
|
550
|
+
try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
|
|
551
|
+
}
|
|
552
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
553
|
+
}
|
|
554
|
+
})
|
|
555
|
+
|
|
556
|
+
test("guardian preserves restart backoff when an unready owner disconnect aborts a prepared replacement", async () => {
|
|
557
|
+
const fixture = await createGuardian()
|
|
558
|
+
const markerPath = path.join(fixture.root, "prepared-post-claim-exit.pid")
|
|
559
|
+
const startedLogPath = path.join(fixture.root, "prepared-post-claim-starts.jsonl")
|
|
560
|
+
const authority = {configDigest: "same", runtime: null}
|
|
561
|
+
const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
562
|
+
|
|
563
|
+
try {
|
|
564
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
|
|
565
|
+
ready: false,
|
|
566
|
+
startedLogPath,
|
|
567
|
+
startupTimeoutMs: 150
|
|
568
|
+
}))
|
|
569
|
+
fixture.client.disconnect()
|
|
570
|
+
await waitForFileText(markerPath)
|
|
571
|
+
await replacement.connect()
|
|
572
|
+
const aborted = replacement.waitForEvent("replacement-aborted")
|
|
573
|
+
|
|
574
|
+
await replacement.prepareOwnerReplacement(authority, authority)
|
|
575
|
+
await aborted
|
|
576
|
+
const records = await waitForRestartRecords(startedLogPath, 2)
|
|
577
|
+
|
|
578
|
+
assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `replacement abort retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
|
|
579
|
+
} finally {
|
|
580
|
+
replacement.disconnect()
|
|
581
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
582
|
+
}
|
|
583
|
+
})
|
|
584
|
+
|
|
585
|
+
test("replacement commit preserves a claimed guardian restart child through listener retirement", async () => {
|
|
586
|
+
const fixture = await createGuardian()
|
|
587
|
+
const claimedPath = path.join(fixture.root, "claimed-restart-owner.pid")
|
|
588
|
+
const committedPath = path.join(fixture.root, "claimed-restart-owner-committed.txt")
|
|
589
|
+
const preparedPath = path.join(fixture.root, "claimed-restart-owner-prepared.txt")
|
|
590
|
+
const authority = {configDigest: "old", runtime: null}
|
|
591
|
+
const nextAuthority = {configDigest: "new", runtime: null}
|
|
592
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
593
|
+
/** @type {number | undefined} */
|
|
594
|
+
let recoveredOwnerPid
|
|
595
|
+
|
|
596
|
+
try {
|
|
597
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, claimedPath, {
|
|
598
|
+
ready: false,
|
|
599
|
+
replacementCommittedPath: committedPath,
|
|
600
|
+
replacementPreparedPath: preparedPath,
|
|
601
|
+
startupTimeoutMs: 5000
|
|
602
|
+
}))
|
|
603
|
+
fixture.client.disconnect()
|
|
604
|
+
const incumbentPid = Number((await waitForFileText(claimedPath)).trim())
|
|
605
|
+
|
|
606
|
+
recoveredOwnerPid = incumbentPid
|
|
607
|
+
await candidate.connect()
|
|
608
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
609
|
+
|
|
610
|
+
await waitForFileText(preparedPath, new RegExp(prepared.replacementId))
|
|
611
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: null}})
|
|
612
|
+
const published = candidate.waitForEvent("replacement-committed")
|
|
613
|
+
|
|
614
|
+
process.kill(incumbentPid, "SIGUSR2")
|
|
615
|
+
await Promise.race([waitForFileText(committedPath, new RegExp(prepared.replacementId)), published])
|
|
616
|
+
await new Promise((resolve) => setTimeout(resolve, 50))
|
|
617
|
+
assert.doesNotThrow(() => process.kill(incumbentPid, 0), "claimed incumbent must survive replacement commit until its listeners retire")
|
|
618
|
+
|
|
619
|
+
process.kill(incumbentPid, "SIGUSR1")
|
|
620
|
+
await published
|
|
621
|
+
await candidate.shutdown()
|
|
622
|
+
await fixture.client.guardianExit()
|
|
623
|
+
} finally {
|
|
624
|
+
candidate.disconnect()
|
|
625
|
+
if (recoveredOwnerPid) {
|
|
626
|
+
try { process.kill(-recoveredOwnerPid, "SIGKILL") } catch (_error) { /* Exact recovered fixture owner already exited. */ }
|
|
627
|
+
}
|
|
628
|
+
await cleanupGuardian(fixture)
|
|
629
|
+
}
|
|
630
|
+
})
|
|
631
|
+
|
|
632
|
+
test("ownerless replacement commit kills a superseded restart candidate", async () => {
|
|
633
|
+
const fixture = await createGuardian()
|
|
634
|
+
const delayedClaimPath = path.join(fixture.root, "delayed-claim.pid")
|
|
635
|
+
const firstMarkerPath = path.join(fixture.root, "delayed-owner-started.pid")
|
|
636
|
+
const secondMarkerPath = path.join(fixture.root, "committed-replacement.jsonl")
|
|
637
|
+
const authority = {configDigest: "old", runtime: null}
|
|
638
|
+
const nextAuthority = {configDigest: "new", runtime: null}
|
|
639
|
+
const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
640
|
+
let delayedOwnerPid
|
|
641
|
+
let staged = false
|
|
642
|
+
|
|
643
|
+
try {
|
|
644
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, delayedClaimPath, {claimDelayMs: 500, startedPath: firstMarkerPath}))
|
|
645
|
+
fixture.client.disconnect()
|
|
646
|
+
delayedOwnerPid = Number((await waitForFileText(firstMarkerPath)).trim())
|
|
647
|
+
await replacement.connect()
|
|
648
|
+
const prepared = await replacement.prepareOwnerReplacement(authority, nextAuthority)
|
|
649
|
+
|
|
650
|
+
await waitForProcessExit(delayedOwnerPid)
|
|
651
|
+
const committed = replacement.waitForEvent("replacement-committed")
|
|
652
|
+
|
|
653
|
+
await replacement.stageOwnerReplacement(prepared.replacementId, recoveryOwnerState(nextAuthority, secondMarkerPath, "committed", 20))
|
|
654
|
+
staged = true
|
|
655
|
+
await committed
|
|
656
|
+
replacement.disconnect()
|
|
657
|
+
|
|
658
|
+
assert.deepEqual((await waitForRestartRecords(secondMarkerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "committed"}])
|
|
659
|
+
} finally {
|
|
660
|
+
replacement.disconnect()
|
|
661
|
+
if (delayedOwnerPid) {
|
|
662
|
+
try { process.kill(delayedOwnerPid, "SIGKILL") } catch (_error) { /* Exact superseded fixture owner already exited. */ }
|
|
663
|
+
}
|
|
664
|
+
await reconnectAndShutdownGuardian(fixture, staged ? nextAuthority : authority)
|
|
665
|
+
}
|
|
666
|
+
})
|
|
667
|
+
|
|
668
|
+
test("guardian logs an asynchronous daemon spawn failure before retrying", async () => {
|
|
669
|
+
const fixture = await createGuardian()
|
|
670
|
+
const logPath = path.join(fixture.root, "daemon.log")
|
|
671
|
+
const authority = {configDigest: "same", runtime: null}
|
|
672
|
+
const privateArgs = "private-argument-value"
|
|
673
|
+
const privateEnvironment = "private-environment-value"
|
|
674
|
+
const privateExecutable = path.join(fixture.root, "private-runtime", "missing-rollbridge")
|
|
675
|
+
|
|
676
|
+
try {
|
|
677
|
+
await fixture.client.publishOwnerState({
|
|
678
|
+
authority,
|
|
679
|
+
recovery: {
|
|
680
|
+
command: {
|
|
681
|
+
args: [privateArgs],
|
|
682
|
+
cwd: fixture.root,
|
|
683
|
+
env: {PRIVATE_ENVIRONMENT: privateEnvironment},
|
|
684
|
+
executable: privateExecutable,
|
|
685
|
+
logPath
|
|
686
|
+
},
|
|
687
|
+
reconnectGraceMs: 0,
|
|
688
|
+
startupTimeoutMs: 1000
|
|
689
|
+
},
|
|
690
|
+
snapshot: {activeReleaseId: null}
|
|
691
|
+
})
|
|
692
|
+
fixture.client.disconnect()
|
|
693
|
+
const diagnosticPattern = /"code":"ENOENT".*"message":"guardian failed to restart daemon"/
|
|
694
|
+
const diagnostic = await waitForFileText(logPath, diagnosticPattern)
|
|
695
|
+
|
|
696
|
+
assert.match(diagnostic, diagnosticPattern)
|
|
697
|
+
for (const privateValue of [privateArgs, privateEnvironment, privateExecutable, fixture.root, logPath]) {
|
|
698
|
+
assert.ok(!diagnostic.includes(privateValue), `guardian diagnostic exposed ${privateValue}`)
|
|
699
|
+
}
|
|
700
|
+
} finally {
|
|
701
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
702
|
+
}
|
|
703
|
+
})
|
|
704
|
+
|
|
705
|
+
test("guardian publishes the authenticated ready owner's PID file", async () => {
|
|
706
|
+
const fixture = await createGuardian()
|
|
707
|
+
const pidPath = path.join(fixture.root, "run", "daemon.pid")
|
|
708
|
+
const victimPath = path.join(fixture.root, "victim")
|
|
709
|
+
const authority = {configDigest: "same", runtime: null}
|
|
710
|
+
|
|
711
|
+
try {
|
|
712
|
+
await fixture.client.publishOwnerState({
|
|
713
|
+
authority,
|
|
714
|
+
recovery: {
|
|
715
|
+
command: {
|
|
716
|
+
args: [],
|
|
717
|
+
cwd: fixture.root,
|
|
718
|
+
env: {},
|
|
719
|
+
executable: process.execPath,
|
|
720
|
+
pidPath
|
|
721
|
+
},
|
|
722
|
+
reconnectGraceMs: 10,
|
|
723
|
+
startupTimeoutMs: 1000
|
|
724
|
+
},
|
|
725
|
+
snapshot: {activeReleaseId: null}
|
|
726
|
+
})
|
|
727
|
+
await fs.mkdir(path.dirname(pidPath), {recursive: true})
|
|
728
|
+
await fs.writeFile(victimPath, "unchanged\n")
|
|
729
|
+
await fs.symlink(victimPath, pidPath)
|
|
730
|
+
await fixture.client.ownerReady()
|
|
731
|
+
|
|
732
|
+
assert.equal(await fs.readFile(pidPath, "utf8"), `${process.pid}\n`)
|
|
733
|
+
assert.equal(await fs.readFile(victimPath, "utf8"), "unchanged\n")
|
|
734
|
+
} finally {
|
|
735
|
+
await cleanupGuardian(fixture)
|
|
736
|
+
}
|
|
737
|
+
})
|
|
738
|
+
|
|
156
739
|
test("replacement commit notification waits for incumbent listener retirement", async () => {
|
|
157
740
|
const fixture = await createGuardian()
|
|
158
741
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
@@ -170,6 +753,10 @@ test("replacement commit notification waits for incumbent listener retirement",
|
|
|
170
753
|
await fixture.client.commitOwnerReplacement(prepared.replacementId)
|
|
171
754
|
await new Promise((resolve) => setImmediate(resolve))
|
|
172
755
|
assert.equal(committed, false, "candidate publication must remain fenced while incumbent listener retirement is delayed")
|
|
756
|
+
const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
|
|
757
|
+
|
|
758
|
+
await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
|
|
759
|
+
await listenersRetired
|
|
173
760
|
await fixture.client.request({command: "finalize-owner-replacement", replacementId: prepared.replacementId})
|
|
174
761
|
await notification
|
|
175
762
|
assert.equal(committed, true)
|
|
@@ -181,6 +768,40 @@ test("replacement commit notification waits for incumbent listener retirement",
|
|
|
181
768
|
}
|
|
182
769
|
})
|
|
183
770
|
|
|
771
|
+
test("completed direct listener retirement finalizes when the incumbent disconnects", {timeout: 3000}, async () => {
|
|
772
|
+
const fixture = await createGuardian()
|
|
773
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
774
|
+
const authority = {configDigest: "incumbent", runtime: null}
|
|
775
|
+
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
776
|
+
|
|
777
|
+
try {
|
|
778
|
+
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v1"}})
|
|
779
|
+
await candidate.connect()
|
|
780
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
781
|
+
|
|
782
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})
|
|
783
|
+
const committed = candidate.waitForEvent("replacement-committed")
|
|
784
|
+
|
|
785
|
+
await fixture.client.commitOwnerReplacement(prepared.replacementId)
|
|
786
|
+
await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
|
|
787
|
+
fixture.client.disconnect()
|
|
788
|
+
await committed
|
|
789
|
+
await candidate.finalizeOwnerReplacement(prepared.replacementId)
|
|
790
|
+
assert.deepEqual(await candidate.replacementStatus(), {
|
|
791
|
+
committedReplacementId: prepared.replacementId,
|
|
792
|
+
ownerClaimed: true,
|
|
793
|
+
retirementFailed: false,
|
|
794
|
+
retirementPending: false,
|
|
795
|
+
retirementReady: false
|
|
796
|
+
})
|
|
797
|
+
await candidate.shutdown()
|
|
798
|
+
await fixture.client.guardianExit()
|
|
799
|
+
} finally {
|
|
800
|
+
candidate.disconnect()
|
|
801
|
+
await cleanupGuardian(fixture)
|
|
802
|
+
}
|
|
803
|
+
})
|
|
804
|
+
|
|
184
805
|
test("replacement staging rejects owner state published after prepare", async () => {
|
|
185
806
|
const fixture = await createGuardian()
|
|
186
807
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
@@ -210,6 +831,115 @@ test("replacement staging rejects owner state published after prepare", async ()
|
|
|
210
831
|
}
|
|
211
832
|
})
|
|
212
833
|
|
|
834
|
+
test("staged replacement receives cleared local sources when the incumbent disconnects", {timeout: 3000}, async () => {
|
|
835
|
+
const fixture = await createGuardian()
|
|
836
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
837
|
+
const authority = {configDigest: "incumbent", runtime: null}
|
|
838
|
+
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
839
|
+
const snapshot = {activeReleaseId: "v1"}
|
|
840
|
+
|
|
841
|
+
try {
|
|
842
|
+
await fixture.client.publishOwnerState({
|
|
843
|
+
authority,
|
|
844
|
+
listenerConnectionSources: {"incumbent-local": {v1: {http: 0, websocket: 1}}},
|
|
845
|
+
listenerSourceId: "incumbent-local",
|
|
846
|
+
snapshot
|
|
847
|
+
})
|
|
848
|
+
await candidate.connect()
|
|
849
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
850
|
+
|
|
851
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {
|
|
852
|
+
authority: nextAuthority,
|
|
853
|
+
listenerConnectionSources: {"incumbent-local": {v1: {http: 0, websocket: 1}}},
|
|
854
|
+
listenerSourceId: "candidate-local",
|
|
855
|
+
snapshot
|
|
856
|
+
})
|
|
857
|
+
const cleared = candidate.waitForEvent("owner-connection-state")
|
|
858
|
+
const committed = candidate.waitForEvent("replacement-committed")
|
|
859
|
+
|
|
860
|
+
fixture.client.disconnect()
|
|
861
|
+
assert.deepEqual(await cleared, {
|
|
862
|
+
connections: {http: 0, websocket: 0},
|
|
863
|
+
event: "owner-connection-state",
|
|
864
|
+
releaseId: "v1",
|
|
865
|
+
sourceId: "incumbent-local"
|
|
866
|
+
})
|
|
867
|
+
await committed
|
|
868
|
+
const state = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await candidate.ownerState())
|
|
869
|
+
|
|
870
|
+
assert.deepEqual(state.listenerConnectionSources, {})
|
|
871
|
+
await candidate.shutdown()
|
|
872
|
+
await fixture.client.guardianExit()
|
|
873
|
+
} finally {
|
|
874
|
+
candidate.disconnect()
|
|
875
|
+
await cleanupGuardian(fixture)
|
|
876
|
+
}
|
|
877
|
+
})
|
|
878
|
+
|
|
879
|
+
test("staged successor state receives tombstones when an older completed listener disconnects", {timeout: 3000}, async () => {
|
|
880
|
+
const fixture = await createGuardian()
|
|
881
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
882
|
+
const successor = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
883
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
884
|
+
const snapshot = {activeReleaseId: "v1"}
|
|
885
|
+
|
|
886
|
+
try {
|
|
887
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
888
|
+
await candidate.connect()
|
|
889
|
+
const first = await candidate.prepareOwnerReplacement(authority, authority)
|
|
890
|
+
|
|
891
|
+
await candidate.stageOwnerReplacement(first.replacementId, {authority, listenerSourceId: "candidate-local", snapshot})
|
|
892
|
+
const firstCommitted = candidate.waitForEvent("replacement-committed")
|
|
893
|
+
|
|
894
|
+
await fixture.client.commitOwnerReplacement(first.replacementId)
|
|
895
|
+
const sourcePublished = candidate.waitForEvent("owner-connection-state")
|
|
896
|
+
|
|
897
|
+
await fixture.client.publishOwnerConnectionState(first.replacementId, "retired-local", "v1", {http: 0, websocket: 1}, true)
|
|
898
|
+
await sourcePublished
|
|
899
|
+
const firstListenersRetired = candidate.waitForEvent("replacement-listeners-retired")
|
|
900
|
+
|
|
901
|
+
await fixture.client.completeOwnerListenerRetirement(first.replacementId)
|
|
902
|
+
await firstListenersRetired
|
|
903
|
+
await candidate.finalizeOwnerReplacement(first.replacementId)
|
|
904
|
+
await firstCommitted
|
|
905
|
+
|
|
906
|
+
await successor.connect()
|
|
907
|
+
const second = await successor.prepareOwnerReplacement(authority, authority)
|
|
908
|
+
|
|
909
|
+
await successor.stageOwnerReplacement(second.replacementId, {
|
|
910
|
+
authority,
|
|
911
|
+
listenerConnectionSources: {"retired-local": {v1: {http: 0, websocket: 1}}},
|
|
912
|
+
listenerSourceId: "successor-local",
|
|
913
|
+
snapshot
|
|
914
|
+
})
|
|
915
|
+
const sourceCleared = candidate.waitForEvent("owner-connection-state")
|
|
916
|
+
const stagedSourceCleared = successor.waitForEvent("owner-connection-state")
|
|
917
|
+
|
|
918
|
+
fixture.client.disconnect()
|
|
919
|
+
const tombstone = {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "retired-local"}
|
|
920
|
+
|
|
921
|
+
assert.deepEqual(await Promise.all([sourceCleared, stagedSourceCleared]), [tombstone, tombstone])
|
|
922
|
+
|
|
923
|
+
await candidate.commitOwnerReplacement(second.replacementId)
|
|
924
|
+
const successorState = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await successor.ownerState())
|
|
925
|
+
|
|
926
|
+
assert.deepEqual(successorState.listenerConnectionSources, {})
|
|
927
|
+
const secondCommitted = successor.waitForEvent("replacement-committed")
|
|
928
|
+
const secondListenersRetired = successor.waitForEvent("replacement-listeners-retired")
|
|
929
|
+
|
|
930
|
+
await candidate.completeOwnerListenerRetirement(second.replacementId)
|
|
931
|
+
await secondListenersRetired
|
|
932
|
+
await successor.finalizeOwnerReplacement(second.replacementId)
|
|
933
|
+
await secondCommitted
|
|
934
|
+
await successor.shutdown()
|
|
935
|
+
await fixture.client.guardianExit()
|
|
936
|
+
} finally {
|
|
937
|
+
successor.disconnect()
|
|
938
|
+
candidate.disconnect()
|
|
939
|
+
await cleanupGuardian(fixture)
|
|
940
|
+
}
|
|
941
|
+
})
|
|
942
|
+
|
|
213
943
|
test("replacement abort notifies both the candidate and committed owner", async () => {
|
|
214
944
|
const fixture = await createGuardian()
|
|
215
945
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
@@ -298,76 +1028,365 @@ test("guardian owner-replacement capability classification is explicit and fail
|
|
|
298
1028
|
}
|
|
299
1029
|
})
|
|
300
1030
|
|
|
301
|
-
test("reserved process recovery rejects a reconstructed definition with different provenance", async () => {
|
|
1031
|
+
test("reserved process recovery rejects a reconstructed definition with different provenance", async () => {
|
|
1032
|
+
const fixture = await createGuardian()
|
|
1033
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1034
|
+
const processKey = "release:v1:worker"
|
|
1035
|
+
|
|
1036
|
+
try {
|
|
1037
|
+
await fixture.client.process(processKey, definition("worker")).recover()
|
|
1038
|
+
const [registration] = await fixture.client.inventory()
|
|
1039
|
+
|
|
1040
|
+
assert.ok(registration)
|
|
1041
|
+
await candidate.connect()
|
|
1042
|
+
candidate.reserveProcessRecovery(processKey, registration.provenance)
|
|
1043
|
+
await assert.rejects(
|
|
1044
|
+
() => candidate.process(processKey, definition("different-worker")).recover(),
|
|
1045
|
+
/provenance mismatch for reserved process/
|
|
1046
|
+
)
|
|
1047
|
+
assert.deepEqual((await fixture.client.inventory()).map(({key}) => key), [processKey])
|
|
1048
|
+
} finally {
|
|
1049
|
+
candidate.disconnect()
|
|
1050
|
+
await cleanupGuardian(fixture)
|
|
1051
|
+
}
|
|
1052
|
+
})
|
|
1053
|
+
|
|
1054
|
+
test("retired owner replacement rejects a registered process absent from committed owner state", async () => {
|
|
1055
|
+
const fixture = await createGuardian()
|
|
1056
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1057
|
+
const committedProcessKey = "release:v1:worker"
|
|
1058
|
+
const candidateProcessKey = "release:candidate:worker"
|
|
1059
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
1060
|
+
const snapshot = {
|
|
1061
|
+
activeReleaseId: "v1",
|
|
1062
|
+
control: {path: path.join(fixture.root, "rollbridge.sock")},
|
|
1063
|
+
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
1064
|
+
services: [],
|
|
1065
|
+
singletons: []
|
|
1066
|
+
}
|
|
1067
|
+
const candidateSnapshot = {
|
|
1068
|
+
...snapshot,
|
|
1069
|
+
releases: [...snapshot.releases, {processes: [{id: "worker"}], releaseId: "candidate"}]
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
try {
|
|
1073
|
+
await fixture.client.process(committedProcessKey, definition("worker")).recover()
|
|
1074
|
+
await fixture.client.process(candidateProcessKey, definition("candidate-worker")).recover()
|
|
1075
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
1076
|
+
await candidate.connect()
|
|
1077
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
1078
|
+
|
|
1079
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot: candidateSnapshot})
|
|
1080
|
+
await assert.rejects(
|
|
1081
|
+
() => candidate.commitRetiredOwnerReplacement(prepared.replacementId, candidateProcessKey),
|
|
1082
|
+
/process .* does not belong to the committed owner/
|
|
1083
|
+
)
|
|
1084
|
+
} finally {
|
|
1085
|
+
candidate.disconnect()
|
|
1086
|
+
await cleanupGuardian(fixture)
|
|
1087
|
+
}
|
|
1088
|
+
})
|
|
1089
|
+
|
|
1090
|
+
test("retired owner replacement requires unchanged authority and the exact control path absent", async () => {
|
|
1091
|
+
const fixture = await createGuardian()
|
|
1092
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1093
|
+
const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1094
|
+
const controlPath = path.join(fixture.root, "rollbridge.sock")
|
|
1095
|
+
const processKey = "release:v1:worker"
|
|
1096
|
+
const authority = {configDigest: "incumbent", runtime: null}
|
|
1097
|
+
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
1098
|
+
const snapshot = {
|
|
1099
|
+
activeReleaseId: "v1",
|
|
1100
|
+
control: {path: controlPath},
|
|
1101
|
+
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
1102
|
+
services: [],
|
|
1103
|
+
singletons: []
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
try {
|
|
1107
|
+
await fixture.client.process(processKey, definition("worker")).recover()
|
|
1108
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
1109
|
+
await candidate.connect()
|
|
1110
|
+
const changed = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
1111
|
+
|
|
1112
|
+
await candidate.stageOwnerReplacement(changed.replacementId, {authority: nextAuthority, snapshot})
|
|
1113
|
+
await assert.rejects(() => candidate.commitRetiredOwnerReplacement(changed.replacementId, processKey), /unchanged owner authority/)
|
|
1114
|
+
await candidate.abortOwnerReplacement(changed.replacementId)
|
|
1115
|
+
|
|
1116
|
+
const occupied = await candidate.prepareOwnerReplacement(authority, authority)
|
|
1117
|
+
|
|
1118
|
+
await candidate.stageOwnerReplacement(occupied.replacementId, {authority, snapshot})
|
|
1119
|
+
await fs.writeFile(controlPath, "occupied\n")
|
|
1120
|
+
await assert.rejects(() => candidate.commitRetiredOwnerReplacement(occupied.replacementId, processKey), /control socket .* still exists/)
|
|
1121
|
+
await candidate.abortOwnerReplacement(occupied.replacementId)
|
|
1122
|
+
|
|
1123
|
+
await fs.rm(controlPath)
|
|
1124
|
+
const ready = await candidate.prepareOwnerReplacement(authority, authority)
|
|
1125
|
+
|
|
1126
|
+
await candidate.stageOwnerReplacement(ready.replacementId, {authority, snapshot})
|
|
1127
|
+
await contender.connect()
|
|
1128
|
+
await assert.rejects(
|
|
1129
|
+
() => contender.request({command: "commit-retired-owner-replacement", key: processKey, replacementId: ready.replacementId}),
|
|
1130
|
+
/not the prepared candidate/
|
|
1131
|
+
)
|
|
1132
|
+
await assert.rejects(
|
|
1133
|
+
() => candidate.commitRetiredOwnerReplacement("stale-replacement", processKey),
|
|
1134
|
+
/not the prepared candidate/
|
|
1135
|
+
)
|
|
1136
|
+
await assert.rejects(
|
|
1137
|
+
() => candidate.commitRetiredOwnerReplacement(ready.replacementId, "release:v1:wrong"),
|
|
1138
|
+
/process .* is not registered/
|
|
1139
|
+
)
|
|
1140
|
+
const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
|
|
1141
|
+
|
|
1142
|
+
void handoffRequested.catch(() => undefined)
|
|
1143
|
+
|
|
1144
|
+
await candidate.prepareRetiredOwnerListenerHandoff(ready.replacementId, processKey)
|
|
1145
|
+
await handoffRequested
|
|
1146
|
+
await assert.rejects(
|
|
1147
|
+
() => contender.prepareOwnerReplacement(authority, authority),
|
|
1148
|
+
/listener retirement is pending/
|
|
1149
|
+
)
|
|
1150
|
+
const committed = candidate.waitForEvent("replacement-committed")
|
|
1151
|
+
const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
|
|
1152
|
+
const connectionState = candidate.waitForEvent("owner-connection-state")
|
|
1153
|
+
|
|
1154
|
+
await fixture.client.publishOwnerConnectionState(ready.replacementId, "listener-a", "v1", {http: 1, websocket: 2}, true)
|
|
1155
|
+
assert.deepEqual(await connectionState, {connections: {http: 1, websocket: 2}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
|
|
1156
|
+
await fixture.client.completeOwnerListenerRetirement(ready.replacementId)
|
|
1157
|
+
await listenersRetired
|
|
1158
|
+
const retirementRequested = fixture.client.waitForEvent("replacement-retirement-requested")
|
|
1159
|
+
|
|
1160
|
+
await candidate.commitRetiredOwnerReplacement(ready.replacementId, processKey)
|
|
1161
|
+
await retirementRequested
|
|
1162
|
+
await candidate.finalizeOwnerReplacement(ready.replacementId)
|
|
1163
|
+
await committed
|
|
1164
|
+
await candidate.shutdown()
|
|
1165
|
+
await fixture.client.guardianExit()
|
|
1166
|
+
} finally {
|
|
1167
|
+
contender.disconnect()
|
|
1168
|
+
candidate.disconnect()
|
|
1169
|
+
await cleanupGuardian(fixture)
|
|
1170
|
+
}
|
|
1171
|
+
})
|
|
1172
|
+
|
|
1173
|
+
test("completed listener retirement survives owner recovery and clears a crashed local source", async () => {
|
|
1174
|
+
const fixture = await createGuardian()
|
|
1175
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1176
|
+
const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1177
|
+
const controlPath = path.join(fixture.root, "rollbridge.sock")
|
|
1178
|
+
const processKey = "release:v1:worker"
|
|
1179
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
1180
|
+
const snapshot = {
|
|
1181
|
+
activeReleaseId: "v1",
|
|
1182
|
+
control: {path: controlPath},
|
|
1183
|
+
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
1184
|
+
services: [],
|
|
1185
|
+
singletons: []
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
try {
|
|
1189
|
+
await fixture.client.process(processKey, definition("worker")).recover()
|
|
1190
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
1191
|
+
await candidate.connect()
|
|
1192
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
1193
|
+
|
|
1194
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {
|
|
1195
|
+
authority,
|
|
1196
|
+
listenerConnectionSources: {"candidate-local": {v1: {http: 0, websocket: 1}}},
|
|
1197
|
+
listenerSourceId: "candidate-local",
|
|
1198
|
+
snapshot
|
|
1199
|
+
})
|
|
1200
|
+
const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
|
|
1201
|
+
|
|
1202
|
+
void handoffRequested.catch(() => undefined)
|
|
1203
|
+
|
|
1204
|
+
await candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
|
|
1205
|
+
await handoffRequested
|
|
1206
|
+
const initial = candidate.waitForEvent("owner-connection-state")
|
|
1207
|
+
|
|
1208
|
+
await fixture.client.publishOwnerConnectionState(prepared.replacementId, "listener-a", "v1", {http: 0, websocket: 1}, true)
|
|
1209
|
+
assert.deepEqual(await initial, {connections: {http: 0, websocket: 1}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
|
|
1210
|
+
const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
|
|
1211
|
+
|
|
1212
|
+
await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
|
|
1213
|
+
await listenersRetired
|
|
1214
|
+
await candidate.commitRetiredOwnerReplacement(prepared.replacementId, processKey)
|
|
1215
|
+
candidate.disconnect()
|
|
1216
|
+
await recovered.connect()
|
|
1217
|
+
await recovered.claimOwner(1000, authority)
|
|
1218
|
+
const stateAfterCandidateCrash = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
|
|
1219
|
+
|
|
1220
|
+
assert.equal(stateAfterCandidateCrash.listenerConnectionSources?.["candidate-local"], undefined)
|
|
1221
|
+
assert.deepEqual(stateAfterCandidateCrash.listenerConnectionSources?.["listener-a"], {v1: {http: 0, websocket: 1}})
|
|
1222
|
+
assert.deepEqual(await recovered.replacementStatus(), {
|
|
1223
|
+
committedReplacementId: prepared.replacementId,
|
|
1224
|
+
ownerClaimed: true,
|
|
1225
|
+
retirementFailed: false,
|
|
1226
|
+
retirementPending: true,
|
|
1227
|
+
retirementReady: true
|
|
1228
|
+
})
|
|
1229
|
+
await recovered.finalizeOwnerReplacement(prepared.replacementId)
|
|
1230
|
+
const cleared = recovered.waitForEvent("owner-connection-state")
|
|
1231
|
+
|
|
1232
|
+
fixture.client.disconnect()
|
|
1233
|
+
assert.deepEqual(await Promise.race([
|
|
1234
|
+
cleared,
|
|
1235
|
+
new Promise((_, reject) => {
|
|
1236
|
+
const timer = setTimeout(() => reject(new Error("Recovered owner did not receive the retired source tombstone")), 500)
|
|
1237
|
+
|
|
1238
|
+
timer.unref()
|
|
1239
|
+
})
|
|
1240
|
+
]), {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
|
|
1241
|
+
const recoveredOwnerState = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
|
|
1242
|
+
|
|
1243
|
+
assert.deepEqual(recoveredOwnerState.listenerConnectionSources, {})
|
|
1244
|
+
await recovered.publishOwnerState({
|
|
1245
|
+
authority,
|
|
1246
|
+
listenerConnectionSources: {"listener-a": {v1: {http: 0, websocket: 1}}},
|
|
1247
|
+
listenerSourceId: "recovered-local",
|
|
1248
|
+
snapshot
|
|
1249
|
+
})
|
|
1250
|
+
const afterStalePublication = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
|
|
1251
|
+
|
|
1252
|
+
assert.deepEqual(afterStalePublication.listenerConnectionSources, {})
|
|
1253
|
+
const recoveredProcess = recovered.process(processKey, definition("worker"))
|
|
1254
|
+
|
|
1255
|
+
await recoveredProcess.recover()
|
|
1256
|
+
await recoveredProcess.updateDefinition({...definition("worker"), env: {REVISION: "stale"}}, {
|
|
1257
|
+
authority,
|
|
1258
|
+
listenerConnectionSources: {"listener-a": {v1: {http: 0, websocket: 1}}},
|
|
1259
|
+
listenerSourceId: "recovered-local",
|
|
1260
|
+
snapshot
|
|
1261
|
+
})
|
|
1262
|
+
const afterStaleProcessUpdate = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
|
|
1263
|
+
|
|
1264
|
+
assert.deepEqual(afterStaleProcessUpdate.listenerConnectionSources, {})
|
|
1265
|
+
await recovered.shutdown()
|
|
1266
|
+
await fixture.client.guardianExit()
|
|
1267
|
+
} finally {
|
|
1268
|
+
recovered.disconnect()
|
|
1269
|
+
candidate.disconnect()
|
|
1270
|
+
await cleanupGuardian(fixture)
|
|
1271
|
+
}
|
|
1272
|
+
})
|
|
1273
|
+
|
|
1274
|
+
test("direct retired-listener source relay survives committed owner recovery", async () => {
|
|
302
1275
|
const fixture = await createGuardian()
|
|
303
1276
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
304
|
-
const
|
|
1277
|
+
const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1278
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
1279
|
+
const snapshot = {activeReleaseId: "v1", control: {path: path.join(fixture.root, "rollbridge.sock")}, releases: [], services: [], singletons: []}
|
|
305
1280
|
|
|
306
1281
|
try {
|
|
307
|
-
await fixture.client.
|
|
308
|
-
const [registration] = await fixture.client.inventory()
|
|
309
|
-
|
|
310
|
-
assert.ok(registration)
|
|
1282
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
311
1283
|
await candidate.connect()
|
|
312
|
-
candidate.
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
)
|
|
317
|
-
|
|
1284
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
1285
|
+
|
|
1286
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, listenerSourceId: "candidate-local", snapshot})
|
|
1287
|
+
await fixture.client.commitOwnerReplacement(prepared.replacementId)
|
|
1288
|
+
const initial = candidate.waitForEvent("owner-connection-state")
|
|
1289
|
+
|
|
1290
|
+
await fixture.client.publishOwnerConnectionState(prepared.replacementId, "direct-listener", "v1", {http: 0, websocket: 1}, true)
|
|
1291
|
+
assert.deepEqual(await initial, {connections: {http: 0, websocket: 1}, event: "owner-connection-state", releaseId: "v1", sourceId: "direct-listener"})
|
|
1292
|
+
const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
|
|
1293
|
+
|
|
1294
|
+
await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
|
|
1295
|
+
await listenersRetired
|
|
1296
|
+
await fixture.client.finalizeOwnerReplacement(prepared.replacementId)
|
|
1297
|
+
candidate.disconnect()
|
|
1298
|
+
await recovered.connect()
|
|
1299
|
+
await recovered.claimOwner(1000, authority)
|
|
1300
|
+
const cleared = recovered.waitForEvent("owner-connection-state")
|
|
1301
|
+
|
|
1302
|
+
await fixture.client.publishOwnerConnectionState(prepared.replacementId, "direct-listener", "v1", {http: 0, websocket: 0}, true)
|
|
1303
|
+
assert.deepEqual(await cleared, {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "direct-listener"})
|
|
1304
|
+
await recovered.shutdown()
|
|
1305
|
+
await fixture.client.guardianExit()
|
|
318
1306
|
} finally {
|
|
1307
|
+
recovered.disconnect()
|
|
319
1308
|
candidate.disconnect()
|
|
320
1309
|
await cleanupGuardian(fixture)
|
|
321
1310
|
}
|
|
322
1311
|
})
|
|
323
1312
|
|
|
324
|
-
test("
|
|
1313
|
+
test("incumbent listener disconnect before state completion aborts without committing the candidate", async () => {
|
|
325
1314
|
const fixture = await createGuardian()
|
|
326
1315
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
327
|
-
const
|
|
328
|
-
const
|
|
1316
|
+
const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1317
|
+
const controlPath = path.join(fixture.root, "rollbridge.sock")
|
|
1318
|
+
const processKey = "release:v1:worker"
|
|
329
1319
|
const authority = {configDigest: "owner", runtime: null}
|
|
330
1320
|
const snapshot = {
|
|
331
1321
|
activeReleaseId: "v1",
|
|
332
|
-
control: {path:
|
|
1322
|
+
control: {path: controlPath},
|
|
333
1323
|
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
334
1324
|
services: [],
|
|
335
1325
|
singletons: []
|
|
336
1326
|
}
|
|
337
|
-
const candidateSnapshot = {
|
|
338
|
-
...snapshot,
|
|
339
|
-
releases: [...snapshot.releases, {processes: [{id: "worker"}], releaseId: "candidate"}]
|
|
340
|
-
}
|
|
341
1327
|
|
|
342
1328
|
try {
|
|
343
|
-
await fixture.client.process(
|
|
344
|
-
await fixture.client.process(candidateProcessKey, definition("candidate-worker")).recover()
|
|
1329
|
+
await fixture.client.process(processKey, definition("worker")).recover()
|
|
345
1330
|
await fixture.client.publishOwnerState({authority, snapshot})
|
|
346
1331
|
await candidate.connect()
|
|
347
1332
|
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
348
1333
|
|
|
349
|
-
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
1334
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot})
|
|
1335
|
+
const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
|
|
1336
|
+
const failed = candidate.waitForEvent("replacement-retirement-failed")
|
|
1337
|
+
const aborted = candidate.waitForEvent("replacement-aborted")
|
|
1338
|
+
let tombstones = 0
|
|
1339
|
+
|
|
1340
|
+
candidate.onEvent("owner-connection-state", (event) => {
|
|
1341
|
+
if (event.sourceId === "incumbent-local" && event.releaseId === "v1" && event.connections && typeof event.connections === "object" && !Array.isArray(event.connections) && event.connections.http === 0 && event.connections.websocket === 0) {
|
|
1342
|
+
tombstones += 1
|
|
1343
|
+
}
|
|
1344
|
+
})
|
|
1345
|
+
|
|
1346
|
+
await candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
|
|
1347
|
+
await handoffRequested
|
|
1348
|
+
const sourcePublished = candidate.waitForEvent("owner-connection-state")
|
|
1349
|
+
|
|
1350
|
+
await fixture.client.publishOwnerConnectionState(prepared.replacementId, "incumbent-local", "v1", {http: 0, websocket: 1}, true)
|
|
1351
|
+
await sourcePublished
|
|
1352
|
+
fixture.client.disconnect()
|
|
1353
|
+
assert.deepEqual(await failed, {
|
|
1354
|
+
event: "replacement-retirement-failed",
|
|
1355
|
+
reason: "Incumbent listener disconnected during the prepared handoff",
|
|
1356
|
+
replacementId: prepared.replacementId
|
|
1357
|
+
})
|
|
1358
|
+
assert.deepEqual(await aborted, {
|
|
1359
|
+
event: "replacement-aborted",
|
|
1360
|
+
reason: "Incumbent listener disconnected during the prepared handoff"
|
|
1361
|
+
})
|
|
1362
|
+
assert.equal(tombstones, 1)
|
|
1363
|
+
assert.deepEqual(await candidate.replacementStatus(), {
|
|
1364
|
+
committedReplacementId: null,
|
|
1365
|
+
ownerClaimed: false,
|
|
1366
|
+
retirementFailed: false,
|
|
1367
|
+
retirementPending: false,
|
|
1368
|
+
retirementReady: false
|
|
1369
|
+
})
|
|
1370
|
+
await recovered.connect()
|
|
1371
|
+
await recovered.claimOwner(1000, authority)
|
|
1372
|
+
await recovered.shutdown()
|
|
1373
|
+
await fixture.client.guardianExit()
|
|
354
1374
|
} finally {
|
|
1375
|
+
recovered.disconnect()
|
|
355
1376
|
candidate.disconnect()
|
|
356
1377
|
await cleanupGuardian(fixture)
|
|
357
1378
|
}
|
|
358
1379
|
})
|
|
359
1380
|
|
|
360
|
-
test("
|
|
1381
|
+
test("replacement abort during listener handoff validation leaves the incumbent available", async () => {
|
|
361
1382
|
const fixture = await createGuardian()
|
|
362
1383
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
363
1384
|
const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
364
|
-
const controlPath = path.join(fixture.root, "rollbridge.sock")
|
|
365
1385
|
const processKey = "release:v1:worker"
|
|
366
|
-
const authority = {configDigest: "
|
|
367
|
-
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
1386
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
368
1387
|
const snapshot = {
|
|
369
1388
|
activeReleaseId: "v1",
|
|
370
|
-
control: {path:
|
|
1389
|
+
control: {path: path.join(fixture.root, "rollbridge.sock")},
|
|
371
1390
|
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
372
1391
|
services: [],
|
|
373
1392
|
singletons: []
|
|
@@ -377,45 +1396,68 @@ test("retired owner replacement requires unchanged authority and the exact contr
|
|
|
377
1396
|
await fixture.client.process(processKey, definition("worker")).recover()
|
|
378
1397
|
await fixture.client.publishOwnerState({authority, snapshot})
|
|
379
1398
|
await candidate.connect()
|
|
380
|
-
const
|
|
1399
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
381
1400
|
|
|
382
|
-
await candidate.stageOwnerReplacement(
|
|
383
|
-
|
|
384
|
-
await candidate.abortOwnerReplacement(changed.replacementId)
|
|
1401
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot})
|
|
1402
|
+
const abandonedHandoff = candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
|
|
385
1403
|
|
|
386
|
-
|
|
1404
|
+
await candidate.abortOwnerReplacement(prepared.replacementId)
|
|
1405
|
+
await assert.rejects(() => abandonedHandoff, /prepared candidate/)
|
|
1406
|
+
await contender.connect()
|
|
1407
|
+
const fresh = await contender.prepareOwnerReplacement(authority, authority)
|
|
387
1408
|
|
|
388
|
-
await
|
|
389
|
-
await
|
|
390
|
-
await
|
|
391
|
-
|
|
1409
|
+
await contender.abortOwnerReplacement(fresh.replacementId)
|
|
1410
|
+
await fixture.client.shutdown()
|
|
1411
|
+
await fixture.client.guardianExit()
|
|
1412
|
+
} finally {
|
|
1413
|
+
contender.disconnect()
|
|
1414
|
+
candidate.disconnect()
|
|
1415
|
+
await cleanupGuardian(fixture)
|
|
1416
|
+
}
|
|
1417
|
+
})
|
|
392
1418
|
|
|
393
|
-
|
|
394
|
-
|
|
1419
|
+
test("explicit replacement abort notifies the incumbent owner", async () => {
|
|
1420
|
+
const fixture = await createGuardian()
|
|
1421
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1422
|
+
const authority = {configDigest: "incumbent", runtime: null}
|
|
1423
|
+
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
395
1424
|
|
|
396
|
-
|
|
1425
|
+
try {
|
|
1426
|
+
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v1"}})
|
|
1427
|
+
await candidate.connect()
|
|
1428
|
+
const aborted = fixture.client.waitForEvent("replacement-aborted")
|
|
1429
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
1430
|
+
|
|
1431
|
+
await candidate.abortOwnerReplacement(prepared.replacementId)
|
|
1432
|
+
await aborted
|
|
1433
|
+
await fixture.client.shutdown()
|
|
1434
|
+
await fixture.client.guardianExit()
|
|
1435
|
+
} finally {
|
|
1436
|
+
candidate.disconnect()
|
|
1437
|
+
await cleanupGuardian(fixture)
|
|
1438
|
+
}
|
|
1439
|
+
})
|
|
1440
|
+
|
|
1441
|
+
test("queued owner claim is revalidated against the latest committed authority", async () => {
|
|
1442
|
+
const fixture = await createGuardian()
|
|
1443
|
+
const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1444
|
+
const authority = {configDigest: "old", runtime: null}
|
|
1445
|
+
const nextAuthority = {configDigest: "new", runtime: null}
|
|
1446
|
+
|
|
1447
|
+
try {
|
|
1448
|
+
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: null}})
|
|
397
1449
|
await contender.connect()
|
|
398
|
-
|
|
399
|
-
() => contender.request({command: "commit-retired-owner-replacement", key: processKey, replacementId: ready.replacementId}),
|
|
400
|
-
/not the prepared candidate/
|
|
401
|
-
)
|
|
402
|
-
await assert.rejects(
|
|
403
|
-
() => candidate.commitRetiredOwnerReplacement("stale-replacement", processKey),
|
|
404
|
-
/not the prepared candidate/
|
|
405
|
-
)
|
|
406
|
-
await assert.rejects(
|
|
407
|
-
() => candidate.commitRetiredOwnerReplacement(ready.replacementId, "release:v1:wrong"),
|
|
408
|
-
/process .* is not registered/
|
|
409
|
-
)
|
|
410
|
-
const committed = candidate.waitForEvent("replacement-committed")
|
|
1450
|
+
const claim = contender.claimOwner(500, authority)
|
|
411
1451
|
|
|
412
|
-
await
|
|
413
|
-
await
|
|
414
|
-
|
|
1452
|
+
await new Promise((resolve) => setTimeout(resolve, 20))
|
|
1453
|
+
await fixture.client.publishOwnerState({authority: nextAuthority, snapshot: {activeReleaseId: null}})
|
|
1454
|
+
fixture.client.disconnect()
|
|
1455
|
+
await assert.rejects(claim, /authority changed while the claim was queued/)
|
|
1456
|
+
await contender.claimOwner(500, nextAuthority)
|
|
1457
|
+
await contender.shutdown()
|
|
415
1458
|
await fixture.client.guardianExit()
|
|
416
1459
|
} finally {
|
|
417
1460
|
contender.disconnect()
|
|
418
|
-
candidate.disconnect()
|
|
419
1461
|
await cleanupGuardian(fixture)
|
|
420
1462
|
}
|
|
421
1463
|
})
|
|
@@ -429,6 +1471,7 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
|
|
|
429
1471
|
|
|
430
1472
|
try {
|
|
431
1473
|
await legacyProcess.start()
|
|
1474
|
+
await assert.rejects(() => fixture.client.capabilities(), /Guardian capabilities requires a process key/)
|
|
432
1475
|
legacyPid = legacyProcess.status().pid
|
|
433
1476
|
const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
|
|
434
1477
|
const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
|
|
@@ -469,6 +1512,190 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
|
|
|
469
1512
|
}
|
|
470
1513
|
})
|
|
471
1514
|
|
|
1515
|
+
test("split guardian rejects an owner-state update when its nested legacy definition update fails", async () => {
|
|
1516
|
+
const fixture = await createLegacyGuardian()
|
|
1517
|
+
const processDefinition = definition("legacy-worker")
|
|
1518
|
+
const legacyProcess = fixture.client.process("release:v1:legacy-worker", processDefinition)
|
|
1519
|
+
const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
|
|
1520
|
+
const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
|
|
1521
|
+
const ownerState = {
|
|
1522
|
+
authority,
|
|
1523
|
+
snapshot: {activeReleaseId: "v1", releases: [{processes: [{id: "legacy-worker"}], releaseId: "v1"}], services: [], singletons: []}
|
|
1524
|
+
}
|
|
1525
|
+
let upgraded
|
|
1526
|
+
|
|
1527
|
+
try {
|
|
1528
|
+
await legacyProcess.start()
|
|
1529
|
+
upgraded = await fixture.client.upgradeLegacyGuardian({
|
|
1530
|
+
ownerState,
|
|
1531
|
+
socketPath: path.join(fixture.root, "guardian-v2.sock"),
|
|
1532
|
+
token: "candidate-guardian-capability"
|
|
1533
|
+
})
|
|
1534
|
+
const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
|
|
1535
|
+
const committedOwnerState = {authority: nextAuthority, snapshot: ownerState.snapshot}
|
|
1536
|
+
|
|
1537
|
+
await upgraded.stageOwnerReplacement(prepared.replacementId, committedOwnerState)
|
|
1538
|
+
const restored = upgraded.process("release:v1:legacy-worker", processDefinition)
|
|
1539
|
+
|
|
1540
|
+
await restored.recover()
|
|
1541
|
+
await legacyProcess.updateDefinition({...processDefinition, env: {REVISION: "external"}})
|
|
1542
|
+
await assert.rejects(
|
|
1543
|
+
() => restored.updateDefinition({...processDefinition, env: {REVISION: "candidate"}}, {authority: nextAuthority, snapshot: {...ownerState.snapshot, serviceReleaseIds: {service: "v2"}}}),
|
|
1544
|
+
/provenance mismatch/
|
|
1545
|
+
)
|
|
1546
|
+
assert.deepEqual(await upgraded.ownerState(), committedOwnerState)
|
|
1547
|
+
} finally {
|
|
1548
|
+
await legacyProcess.stop().catch(() => {})
|
|
1549
|
+
upgraded?.disconnect()
|
|
1550
|
+
fixture.client.disconnect()
|
|
1551
|
+
if (upgraded?.pid) killExactProcessGroup(upgraded.pid)
|
|
1552
|
+
if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
|
|
1553
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1554
|
+
}
|
|
1555
|
+
})
|
|
1556
|
+
|
|
1557
|
+
test("split guardian defers owner handoff until a nested legacy definition update commits", async () => {
|
|
1558
|
+
const fixture = await createLegacyGuardian()
|
|
1559
|
+
const processDefinition = definition("legacy-worker")
|
|
1560
|
+
const legacyProcess = fixture.client.process("release:v1:legacy-worker", processDefinition)
|
|
1561
|
+
const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
|
|
1562
|
+
const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
|
|
1563
|
+
const ownerState = {
|
|
1564
|
+
authority,
|
|
1565
|
+
snapshot: {activeReleaseId: "v1", releases: [{processes: [{id: "legacy-worker"}], releaseId: "v1"}], services: [], singletons: []}
|
|
1566
|
+
}
|
|
1567
|
+
const socketPath = path.join(fixture.root, "guardian-v2.sock")
|
|
1568
|
+
const token = "candidate-guardian-capability"
|
|
1569
|
+
const gatePath = path.join(fixture.root, "legacy-update.allow")
|
|
1570
|
+
const committedOwnerState = {
|
|
1571
|
+
authority: nextAuthority,
|
|
1572
|
+
listenerConnectionSources: {"updating-owner": {v1: {http: 0, websocket: 1}}},
|
|
1573
|
+
listenerSourceId: "updating-owner",
|
|
1574
|
+
snapshot: {...ownerState.snapshot, update: "committed"}
|
|
1575
|
+
}
|
|
1576
|
+
const contender = new GuardianClient({socketPath, token})
|
|
1577
|
+
let upgraded
|
|
1578
|
+
|
|
1579
|
+
try {
|
|
1580
|
+
await legacyProcess.start()
|
|
1581
|
+
upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, socketPath, token})
|
|
1582
|
+
const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
|
|
1583
|
+
|
|
1584
|
+
await upgraded.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: ownerState.snapshot})
|
|
1585
|
+
const restored = upgraded.process("release:v1:legacy-worker", processDefinition)
|
|
1586
|
+
|
|
1587
|
+
await restored.recover()
|
|
1588
|
+
await contender.connect()
|
|
1589
|
+
const updateResult = restored.updateDefinition({...processDefinition, env: {ROLLBRIDGE_TEST_UPDATE_GATE: gatePath}}, committedOwnerState)
|
|
1590
|
+
.then(() => undefined, (error) => error)
|
|
1591
|
+
|
|
1592
|
+
await waitForFileText(`${gatePath}.waiting`)
|
|
1593
|
+
const claim = contender.claimOwner(1000, nextAuthority)
|
|
1594
|
+
let claimSettled = false
|
|
1595
|
+
|
|
1596
|
+
void claim.finally(() => { claimSettled = true })
|
|
1597
|
+
upgraded.disconnect()
|
|
1598
|
+
await new Promise((resolve) => setTimeout(resolve, 50))
|
|
1599
|
+
assert.equal(claimSettled, false, "owner handoff must wait for the nested definition update")
|
|
1600
|
+
|
|
1601
|
+
await fs.writeFile(gatePath, "allow\n")
|
|
1602
|
+
await claim
|
|
1603
|
+
const updateError = await updateResult
|
|
1604
|
+
|
|
1605
|
+
assert.match(String(updateError), /connection closed while awaiting update/)
|
|
1606
|
+
assert.deepEqual(await contender.ownerState(), {...committedOwnerState, listenerConnectionSources: {}})
|
|
1607
|
+
} finally {
|
|
1608
|
+
await fs.writeFile(gatePath, "allow\n").catch(() => {})
|
|
1609
|
+
await contender.shutdown().catch(() => {})
|
|
1610
|
+
contender.disconnect()
|
|
1611
|
+
upgraded?.disconnect()
|
|
1612
|
+
await legacyProcess.stop().catch(() => {})
|
|
1613
|
+
fixture.client.disconnect()
|
|
1614
|
+
if (upgraded?.pid) killExactProcessGroup(upgraded.pid)
|
|
1615
|
+
if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
|
|
1616
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1617
|
+
}
|
|
1618
|
+
})
|
|
1619
|
+
|
|
1620
|
+
test("a disconnected legacy upgrade candidate does not strand its bridge guardian", async () => {
|
|
1621
|
+
const fixture = await createLegacyGuardian()
|
|
1622
|
+
const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
|
|
1623
|
+
const ownerState = {
|
|
1624
|
+
authority,
|
|
1625
|
+
snapshot: {activeReleaseId: null, releases: [], services: [], singletons: []}
|
|
1626
|
+
}
|
|
1627
|
+
let upgraded
|
|
1628
|
+
let bridgeExited = false
|
|
1629
|
+
|
|
1630
|
+
try {
|
|
1631
|
+
upgraded = await fixture.client.upgradeLegacyGuardian({
|
|
1632
|
+
ownerState,
|
|
1633
|
+
socketPath: path.join(fixture.root, "guardian-v2.sock"),
|
|
1634
|
+
token: "candidate-guardian-capability"
|
|
1635
|
+
})
|
|
1636
|
+
await upgraded.prepareOwnerReplacement(authority, {...authority, runtime: {...authority.runtime, digest: "candidate-runtime"}})
|
|
1637
|
+
upgraded.disconnect()
|
|
1638
|
+
let timeout = /** @type {ReturnType<typeof setTimeout> | undefined} */ (undefined)
|
|
1639
|
+
|
|
1640
|
+
await Promise.race([
|
|
1641
|
+
upgraded.guardianExit(),
|
|
1642
|
+
new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Legacy upgrade bridge guardian did not exit after candidate disconnect")), 1000) })
|
|
1643
|
+
]).finally(() => { if (timeout) clearTimeout(timeout) })
|
|
1644
|
+
bridgeExited = true
|
|
1645
|
+
assert.equal(fixture.child.exitCode, null, "the pre-split guardian must remain available after bridge abandonment")
|
|
1646
|
+
} finally {
|
|
1647
|
+
upgraded?.disconnect()
|
|
1648
|
+
if (!bridgeExited && upgraded?.pid) killExactProcessGroup(upgraded.pid)
|
|
1649
|
+
await cleanupGuardian({client: fixture.client, root: fixture.root})
|
|
1650
|
+
}
|
|
1651
|
+
})
|
|
1652
|
+
|
|
1653
|
+
test("a legacy bridge remains discoverable when its candidate disconnects after the disruptive boundary", async () => {
|
|
1654
|
+
const fixture = await createLegacyGuardian()
|
|
1655
|
+
const statePath = path.join(fixture.root, "state.json")
|
|
1656
|
+
const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
|
|
1657
|
+
const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime"}}
|
|
1658
|
+
const ownerState = {
|
|
1659
|
+
authority,
|
|
1660
|
+
config: {statePath},
|
|
1661
|
+
snapshot: {activeReleaseId: null, control: {path: path.join(fixture.root, "control.sock")}, releases: [], services: [], singletons: []}
|
|
1662
|
+
}
|
|
1663
|
+
let upgraded
|
|
1664
|
+
let replacement
|
|
1665
|
+
|
|
1666
|
+
try {
|
|
1667
|
+
const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
|
|
1668
|
+
|
|
1669
|
+
upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
|
|
1670
|
+
const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
|
|
1671
|
+
const recoverySnapshot = {
|
|
1672
|
+
...ownerState.snapshot,
|
|
1673
|
+
recovery: {configDigest: authority.configDigest, format: 1, guardian: {...identity, pid: upgraded.pid}, reconnectGraceMs: 3000}
|
|
1674
|
+
}
|
|
1675
|
+
|
|
1676
|
+
await fs.writeFile(statePath, `${JSON.stringify(ownerState.snapshot, null, 2)}\n`)
|
|
1677
|
+
await upgraded.beginLegacyOwnerClaim(prepared.replacementId, 1000, statePath, recoverySnapshot)
|
|
1678
|
+
fixture.client.disconnect()
|
|
1679
|
+
await upgraded.completeLegacyOwnerClaim(prepared.replacementId)
|
|
1680
|
+
assert.deepEqual(JSON.parse(await fs.readFile(statePath, "utf8")), recoverySnapshot)
|
|
1681
|
+
upgraded.disconnect()
|
|
1682
|
+
|
|
1683
|
+
replacement = new GuardianClient({...identity, pid: upgraded.pid})
|
|
1684
|
+
await replacement.connect()
|
|
1685
|
+
const resumed = await replacement.prepareOwnerReplacement(authority, nextAuthority)
|
|
1686
|
+
const committed = replacement.waitForEvent("replacement-committed")
|
|
1687
|
+
|
|
1688
|
+
assert.deepEqual(await replacement.stageOwnerReplacement(resumed.replacementId, {authority: nextAuthority, config: {statePath}, snapshot: ownerState.snapshot}), {committed: true})
|
|
1689
|
+
await committed
|
|
1690
|
+
await replacement.shutdown()
|
|
1691
|
+
await upgraded.guardianExit()
|
|
1692
|
+
} finally {
|
|
1693
|
+
upgraded?.disconnect()
|
|
1694
|
+
replacement?.disconnect()
|
|
1695
|
+
await cleanupGuardian({client: fixture.client, root: fixture.root})
|
|
1696
|
+
}
|
|
1697
|
+
})
|
|
1698
|
+
|
|
472
1699
|
/** @returns {Promise<{client: GuardianClient, root: string, token: string}>} Started guardian fixture. */
|
|
473
1700
|
async function createGuardian() {
|
|
474
1701
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-guardian-client-"))
|
|
@@ -524,9 +1751,139 @@ async function cleanupGuardian(fixture) {
|
|
|
524
1751
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
525
1752
|
}
|
|
526
1753
|
|
|
1754
|
+
/**
|
|
1755
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} authority - Recovery authority.
|
|
1756
|
+
* @param {string} markerPath - Restart marker path.
|
|
1757
|
+
* @param {string} marker - Expected accepted command marker.
|
|
1758
|
+
* @param {number} reconnectGraceMs - Initial reconnect grace.
|
|
1759
|
+
* @returns {Record<string, import("../src/json.js").JsonValue>} Guardian owner state.
|
|
1760
|
+
*/
|
|
1761
|
+
function recoveryOwnerState(authority, markerPath, marker, reconnectGraceMs) {
|
|
1762
|
+
return {
|
|
1763
|
+
authority,
|
|
1764
|
+
recovery: {
|
|
1765
|
+
command: {
|
|
1766
|
+
args: ["-e", "require('node:fs').appendFileSync(process.argv[1], JSON.stringify({at: Date.now(), home: process.env.HOME ?? null, marker: process.env.RESTART_MARKER}) + '\\n')", markerPath],
|
|
1767
|
+
cwd: path.dirname(markerPath),
|
|
1768
|
+
env: {RESTART_MARKER: marker},
|
|
1769
|
+
executable: process.execPath
|
|
1770
|
+
},
|
|
1771
|
+
reconnectGraceMs,
|
|
1772
|
+
startupTimeoutMs: 1000
|
|
1773
|
+
},
|
|
1774
|
+
snapshot: {activeReleaseId: null}
|
|
1775
|
+
}
|
|
1776
|
+
}
|
|
1777
|
+
|
|
1778
|
+
/**
|
|
1779
|
+
* @param {{client: GuardianClient, root: string, token: string}} fixture - Guardian fixture.
|
|
1780
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} authority - Recovery authority.
|
|
1781
|
+
* @param {string} markerPath - Claim marker path.
|
|
1782
|
+
* @param {{claimDelayMs?: number, descendantPath?: string, exitAfterClaim?: boolean, ready?: boolean, replacementCommittedPath?: string, replacementPreparedPath?: string, startedLogPath?: string, startedPath?: string, startupTimeoutMs?: number}} [options] - Optional recovery behavior.
|
|
1783
|
+
* @returns {Record<string, import("../src/json.js").JsonValue>} Guardian owner state.
|
|
1784
|
+
*/
|
|
1785
|
+
function claimingRecoveryOwnerState(fixture, authority, markerPath, options = {}) {
|
|
1786
|
+
return {
|
|
1787
|
+
authority,
|
|
1788
|
+
recovery: {
|
|
1789
|
+
command: {
|
|
1790
|
+
args: [recoveryOwnerPath],
|
|
1791
|
+
cwd: fixture.root,
|
|
1792
|
+
env: {
|
|
1793
|
+
GUARDIAN_AUTHORITY: JSON.stringify(authority),
|
|
1794
|
+
GUARDIAN_CLAIM_DELAY_MS: String(options.claimDelayMs || 0),
|
|
1795
|
+
...(options.descendantPath ? {GUARDIAN_DESCENDANT_PATH: options.descendantPath} : {}),
|
|
1796
|
+
...(options.exitAfterClaim ? {GUARDIAN_EXIT_AFTER_CLAIM: "1"} : {}),
|
|
1797
|
+
GUARDIAN_MARKER_PATH: markerPath,
|
|
1798
|
+
...(options.replacementCommittedPath ? {GUARDIAN_REPLACEMENT_COMMITTED_PATH: options.replacementCommittedPath} : {}),
|
|
1799
|
+
...(options.replacementPreparedPath ? {GUARDIAN_REPLACEMENT_PREPARED_PATH: options.replacementPreparedPath} : {}),
|
|
1800
|
+
...(options.ready === false ? {GUARDIAN_SKIP_READY: "1"} : {}),
|
|
1801
|
+
GUARDIAN_SOCKET_PATH: fixture.client.socketPath,
|
|
1802
|
+
...(options.startedLogPath ? {GUARDIAN_STARTED_LOG_PATH: options.startedLogPath} : {}),
|
|
1803
|
+
...(options.startedPath ? {GUARDIAN_STARTED_PATH: options.startedPath} : {}),
|
|
1804
|
+
GUARDIAN_TOKEN: fixture.token
|
|
1805
|
+
},
|
|
1806
|
+
executable: process.execPath
|
|
1807
|
+
},
|
|
1808
|
+
reconnectGraceMs: 10,
|
|
1809
|
+
startupTimeoutMs: options.startupTimeoutMs || 1000
|
|
1810
|
+
},
|
|
1811
|
+
snapshot: {activeReleaseId: null}
|
|
1812
|
+
}
|
|
1813
|
+
}
|
|
1814
|
+
|
|
1815
|
+
/**
|
|
1816
|
+
* @param {string} markerPath - Restart marker path.
|
|
1817
|
+
* @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Restart records.
|
|
1818
|
+
*/
|
|
1819
|
+
async function restartRecords(markerPath) {
|
|
1820
|
+
try {
|
|
1821
|
+
return (await fs.readFile(markerPath, "utf8")).trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))
|
|
1822
|
+
} catch (error) {
|
|
1823
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return []
|
|
1824
|
+
throw error
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
/**
|
|
1829
|
+
* @param {string} markerPath - Restart marker path.
|
|
1830
|
+
* @param {number} count - Required record count.
|
|
1831
|
+
* @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Restart records.
|
|
1832
|
+
*/
|
|
1833
|
+
async function waitForRestartRecords(markerPath, count) {
|
|
1834
|
+
const deadline = Date.now() + 3000
|
|
1835
|
+
|
|
1836
|
+
while (Date.now() < deadline) {
|
|
1837
|
+
const records = await restartRecords(markerPath)
|
|
1838
|
+
|
|
1839
|
+
if (records.length >= count) return records
|
|
1840
|
+
await new Promise((resolve) => setTimeout(resolve, 10))
|
|
1841
|
+
}
|
|
1842
|
+
throw new Error(`Timed out waiting for ${count} guardian restart record${count === 1 ? "" : "s"}`)
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
/**
|
|
1846
|
+
* @param {string} filePath - File to read after creation.
|
|
1847
|
+
* @param {RegExp} [expected] - Content that must be present before returning.
|
|
1848
|
+
* @returns {Promise<string>} Non-empty accepted file contents.
|
|
1849
|
+
*/
|
|
1850
|
+
async function waitForFileText(filePath, expected) {
|
|
1851
|
+
const deadline = Date.now() + 3000
|
|
1852
|
+
|
|
1853
|
+
while (Date.now() < deadline) {
|
|
1854
|
+
try {
|
|
1855
|
+
const contents = await fs.readFile(filePath, "utf8")
|
|
1856
|
+
|
|
1857
|
+
if (contents && (!expected || expected.test(contents))) return contents
|
|
1858
|
+
} catch (error) {
|
|
1859
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
1860
|
+
}
|
|
1861
|
+
await new Promise((resolve) => setTimeout(resolve, 10))
|
|
1862
|
+
}
|
|
1863
|
+
throw new Error(`Timed out waiting for ${filePath}`)
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
/**
|
|
1867
|
+
* @param {{client: GuardianClient, root: string, token: string}} fixture - Guardian fixture.
|
|
1868
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} authority - Committed authority.
|
|
1869
|
+
*/
|
|
1870
|
+
async function reconnectAndShutdownGuardian(fixture, authority) {
|
|
1871
|
+
const cleanup = new GuardianClient({pid: fixture.client.pid, socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1872
|
+
|
|
1873
|
+
try {
|
|
1874
|
+
await cleanup.connect()
|
|
1875
|
+
await cleanup.claimOwner(500, authority)
|
|
1876
|
+
await cleanup.shutdown()
|
|
1877
|
+
} finally {
|
|
1878
|
+
cleanup.disconnect()
|
|
1879
|
+
await fixture.client.guardianExit().catch(() => {})
|
|
1880
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1881
|
+
}
|
|
1882
|
+
}
|
|
1883
|
+
|
|
527
1884
|
/**
|
|
528
1885
|
* @param {string} id - Process id.
|
|
529
|
-
* @returns {Parameters<GuardianClient["process"]>[1]} Managed process definition.
|
|
1886
|
+
* @returns {Parameters<GuardianClient["process"]>[1] & import("../src/managed-process.js").ManagedProcessDefinition} Managed process definition.
|
|
530
1887
|
*/
|
|
531
1888
|
function definition(id) {
|
|
532
1889
|
return {
|
|
@@ -536,6 +1893,7 @@ function definition(id) {
|
|
|
536
1893
|
id,
|
|
537
1894
|
lifecycle: {drainTimeoutMs: 0},
|
|
538
1895
|
logger: () => {},
|
|
1896
|
+
memory: undefined,
|
|
539
1897
|
outputLines: 10,
|
|
540
1898
|
restart: {backoffFactor: 1, maxDelayMs: 0, maxRestarts: 0, windowMs: 0},
|
|
541
1899
|
restartDelayMs: 0,
|