rollbridge 0.1.38 → 0.1.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +12 -1
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +16 -2
- package/changelog.d/20260830055159-guardian-daemon-restart.md +2 -0
- package/docs/cli.md +33 -9
- package/docs/config.md +37 -13
- package/docs/logging.md +4 -3
- package/docs/troubleshooting.md +7 -5
- package/package.json +1 -1
- package/src/cli.js +100 -26
- package/src/config.js +14 -6
- package/src/daemon.js +741 -153
- package/src/guardian-client.js +68 -16
- package/src/managed-process.js +55 -8
- package/src/process-guardian.js +731 -43
- package/src/release-group.js +77 -6
- package/test/config-validation.test.js +4 -0
- package/test/fixtures/guardian-recovery-owner.js +86 -0
- package/test/fixtures/pre-split3-process-guardian.js +14 -0
- package/test/guardian-client.test.js +1339 -62
- package/test/managed-process.test.js +136 -7
- package/test/owner-recovery.test.js +814 -52
- package/test/owner-replacement.test.js +526 -23
- package/test/release-group.test.js +19 -2
- package/test/release-runtime-retention.test.js +121 -4
- package/test/rollbridge.test.js +21 -0
- package/test/support/process.js +41 -0
|
@@ -9,13 +9,16 @@ import path from "node:path"
|
|
|
9
9
|
import test from "node:test"
|
|
10
10
|
import {fileURLToPath} from "node:url"
|
|
11
11
|
import GuardianClient from "../src/guardian-client.js"
|
|
12
|
+
import {waitForProcessExit} from "./support/process.js"
|
|
12
13
|
|
|
13
14
|
const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
|
|
15
|
+
const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
|
|
14
16
|
|
|
15
17
|
test("guardian bootstrap capability is absent from process argv", async () => {
|
|
16
18
|
const fixture = await createGuardian()
|
|
17
19
|
|
|
18
20
|
try {
|
|
21
|
+
assert.deepEqual(await fixture.client.capabilities(), {daemonRecovery: 1})
|
|
19
22
|
const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
|
|
20
23
|
const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
|
|
21
24
|
const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
|
|
@@ -67,6 +70,25 @@ test("guardian runs a strict activation lifecycle command for the exact register
|
|
|
67
70
|
}
|
|
68
71
|
})
|
|
69
72
|
|
|
73
|
+
test("guardian atomically updates process provenance with private owner state", async () => {
|
|
74
|
+
const fixture = await createGuardian()
|
|
75
|
+
const processInstance = fixture.client.process("service", definition("service"))
|
|
76
|
+
const previousOwnerState = {authority: null, serviceReleaseIds: {service: "v1"}}
|
|
77
|
+
const nextOwnerState = {authority: null, serviceReleaseIds: {service: "v2"}}
|
|
78
|
+
|
|
79
|
+
try {
|
|
80
|
+
await fixture.client.publishOwnerState(previousOwnerState)
|
|
81
|
+
await processInstance.start()
|
|
82
|
+
const previousProvenance = (await fixture.client.inventory())[0]?.provenance
|
|
83
|
+
|
|
84
|
+
await processInstance.updateDefinition({...definition("service"), env: {RELEASE: "v2"}}, nextOwnerState)
|
|
85
|
+
assert.deepEqual(await fixture.client.ownerState(), nextOwnerState)
|
|
86
|
+
assert.notEqual((await fixture.client.inventory())[0]?.provenance, previousProvenance)
|
|
87
|
+
} finally {
|
|
88
|
+
await cleanupGuardian(fixture)
|
|
89
|
+
}
|
|
90
|
+
})
|
|
91
|
+
|
|
70
92
|
test("guardian forwards each retained output line to its exact process proxy", async () => {
|
|
71
93
|
const fixture = await createGuardian()
|
|
72
94
|
const marker = "guardian-output-ready"
|
|
@@ -75,19 +97,102 @@ test("guardian forwards each retained output line to its exact process proxy", a
|
|
|
75
97
|
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`console.log(${JSON.stringify(marker)})`)}`
|
|
76
98
|
})
|
|
77
99
|
const logged = once(processInstance, "log")
|
|
100
|
+
const forwarded = fixture.client.waitForEvent("process-log")
|
|
78
101
|
const exitedFirst = once(processInstance, "exit").then(() => { throw new Error("Guardian process exited before forwarding retained output") })
|
|
79
102
|
|
|
80
103
|
try {
|
|
81
104
|
await processInstance.start()
|
|
82
105
|
const [entry] = await Promise.race([logged, exitedFirst])
|
|
106
|
+
const event = await forwarded
|
|
83
107
|
|
|
84
108
|
assert.equal(entry.line, marker)
|
|
109
|
+
assert.equal(event.status, undefined, "log events must not resend the complete retained process status")
|
|
85
110
|
assert.ok(processInstance.status().logs.some((candidate) => candidate.line === marker))
|
|
86
111
|
} finally {
|
|
87
112
|
await cleanupGuardian(fixture)
|
|
88
113
|
}
|
|
89
114
|
})
|
|
90
115
|
|
|
116
|
+
test("guardian delivers the final process status after dropping logs for a backpressured client", async () => {
|
|
117
|
+
const fixture = await createGuardian()
|
|
118
|
+
const gatePath = path.join(fixture.root, "write-output")
|
|
119
|
+
const script = `const fs = require("node:fs"); const {once} = require("node:events"); (async () => { while (!fs.existsSync(${JSON.stringify(gatePath)})) await new Promise((resolve) => setTimeout(resolve, 5)); const line = "x".repeat(1024) + "\\n"; for (let index = 0; index < 8192; index += 1) if (!process.stdout.write(line)) await once(process.stdout, "drain"); })()`
|
|
120
|
+
const processInstance = fixture.client.process("backpressured-output", {
|
|
121
|
+
...definition("backpressured-output"),
|
|
122
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(script)}`
|
|
123
|
+
})
|
|
124
|
+
const socket = fixture.client.socket
|
|
125
|
+
|
|
126
|
+
assert.ok(socket)
|
|
127
|
+
try {
|
|
128
|
+
await processInstance.start()
|
|
129
|
+
const pid = processInstance.status().pid
|
|
130
|
+
|
|
131
|
+
assert.ok(pid)
|
|
132
|
+
const finalStatus = fixture.client.waitForEvent("process")
|
|
133
|
+
|
|
134
|
+
socket.pause()
|
|
135
|
+
await fs.writeFile(gatePath, "write\n")
|
|
136
|
+
await waitForProcessExit(pid, 10000)
|
|
137
|
+
socket.resume()
|
|
138
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
139
|
+
let timeout
|
|
140
|
+
|
|
141
|
+
try {
|
|
142
|
+
await Promise.race([
|
|
143
|
+
finalStatus,
|
|
144
|
+
new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Guardian did not flush the final process status after backpressure")), 3000) })
|
|
145
|
+
])
|
|
146
|
+
} finally {
|
|
147
|
+
clearTimeout(timeout)
|
|
148
|
+
}
|
|
149
|
+
assert.equal(processInstance.status().state, "failed")
|
|
150
|
+
} finally {
|
|
151
|
+
socket.resume()
|
|
152
|
+
await cleanupGuardian(fixture)
|
|
153
|
+
}
|
|
154
|
+
})
|
|
155
|
+
|
|
156
|
+
test("guardian resynchronizes retained logs after dropping output for a backpressured client", async () => {
|
|
157
|
+
const fixture = await createGuardian()
|
|
158
|
+
const gatePath = path.join(fixture.root, "write-retained-output")
|
|
159
|
+
const completedPath = path.join(fixture.root, "retained-output-complete")
|
|
160
|
+
const payload = "x".repeat(1024)
|
|
161
|
+
const finalLine = `8191:${payload}`
|
|
162
|
+
const script = `const fs = require("node:fs"); const {once} = require("node:events"); (async () => { while (!fs.existsSync(${JSON.stringify(gatePath)})) await new Promise((resolve) => setTimeout(resolve, 5)); for (let index = 0; index < 8192; index += 1) if (!process.stdout.write(index + ":" + ${JSON.stringify(payload)} + "\\n")) await once(process.stdout, "drain"); fs.writeFileSync(${JSON.stringify(completedPath)}, "complete\\n"); setInterval(() => {}, 1000); })()`
|
|
163
|
+
const processInstance = fixture.client.process("backpressured-retained-output", {
|
|
164
|
+
...definition("backpressured-retained-output"),
|
|
165
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(script)}`
|
|
166
|
+
})
|
|
167
|
+
const socket = fixture.client.socket
|
|
168
|
+
|
|
169
|
+
assert.ok(socket)
|
|
170
|
+
try {
|
|
171
|
+
await processInstance.start()
|
|
172
|
+
socket.pause()
|
|
173
|
+
await fs.writeFile(gatePath, "write\n")
|
|
174
|
+
await waitForFileText(completedPath)
|
|
175
|
+
const resynchronized = fixture.client.waitForEvent("status")
|
|
176
|
+
|
|
177
|
+
socket.resume()
|
|
178
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
179
|
+
let timeout
|
|
180
|
+
|
|
181
|
+
try {
|
|
182
|
+
await Promise.race([
|
|
183
|
+
resynchronized,
|
|
184
|
+
new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Guardian did not resynchronize retained output after backpressure")), 3000) })
|
|
185
|
+
])
|
|
186
|
+
} finally {
|
|
187
|
+
clearTimeout(timeout)
|
|
188
|
+
}
|
|
189
|
+
assert.equal(processInstance.status().logs.at(-1)?.line, finalLine)
|
|
190
|
+
} finally {
|
|
191
|
+
socket.resume()
|
|
192
|
+
await cleanupGuardian(fixture)
|
|
193
|
+
}
|
|
194
|
+
})
|
|
195
|
+
|
|
91
196
|
test("guardian shutdown reports an exact owned process stop failure", async () => {
|
|
92
197
|
const fixture = await createGuardian()
|
|
93
198
|
const processInstance = fixture.client.process("broken-stop", {...definition("broken-stop"), stopSignal: "NOT_A_SIGNAL"})
|
|
@@ -153,6 +258,403 @@ test("successful guardian shutdown closes an authenticated waiting contender bef
|
|
|
153
258
|
}
|
|
154
259
|
})
|
|
155
260
|
|
|
261
|
+
test("guardian restart uses the latest accepted command and exact environment", async () => {
|
|
262
|
+
const fixture = await createGuardian()
|
|
263
|
+
const markerPath = path.join(fixture.root, "restarts.jsonl")
|
|
264
|
+
const authority = {configDigest: "same", runtime: null}
|
|
265
|
+
const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
266
|
+
|
|
267
|
+
try {
|
|
268
|
+
await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "old", 250))
|
|
269
|
+
fixture.client.disconnect()
|
|
270
|
+
await replacement.connect()
|
|
271
|
+
await replacement.claimOwner(250, authority)
|
|
272
|
+
await replacement.publishOwnerState(recoveryOwnerState(authority, markerPath, "new", 40))
|
|
273
|
+
replacement.disconnect()
|
|
274
|
+
|
|
275
|
+
const [restart] = await waitForRestartRecords(markerPath, 1)
|
|
276
|
+
|
|
277
|
+
assert.deepEqual({home: restart.home, marker: restart.marker}, {home: null, marker: "new"})
|
|
278
|
+
} finally {
|
|
279
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
280
|
+
}
|
|
281
|
+
})
|
|
282
|
+
|
|
283
|
+
test("guardian rearms recovery after an ownerless replacement aborts", async () => {
|
|
284
|
+
const fixture = await createGuardian()
|
|
285
|
+
const markerPath = path.join(fixture.root, "replacement-abort.jsonl")
|
|
286
|
+
const authority = {configDigest: "same", runtime: null}
|
|
287
|
+
const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
288
|
+
|
|
289
|
+
try {
|
|
290
|
+
await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "accepted", 40))
|
|
291
|
+
fixture.client.disconnect()
|
|
292
|
+
await replacement.connect()
|
|
293
|
+
const prepared = await replacement.prepareOwnerReplacement(authority, authority)
|
|
294
|
+
|
|
295
|
+
await new Promise((resolve) => setTimeout(resolve, 80))
|
|
296
|
+
await replacement.abortOwnerReplacement(prepared.replacementId)
|
|
297
|
+
assert.deepEqual((await waitForRestartRecords(markerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "accepted"}])
|
|
298
|
+
} finally {
|
|
299
|
+
replacement.disconnect()
|
|
300
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
301
|
+
}
|
|
302
|
+
})
|
|
303
|
+
|
|
304
|
+
test("guardian retry backoff remains nonzero when reconnect grace is zero", async () => {
|
|
305
|
+
const fixture = await createGuardian()
|
|
306
|
+
const markerPath = path.join(fixture.root, "retry-backoff.jsonl")
|
|
307
|
+
const authority = {configDigest: "same", runtime: null}
|
|
308
|
+
|
|
309
|
+
try {
|
|
310
|
+
await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "retry", 0))
|
|
311
|
+
fixture.client.disconnect()
|
|
312
|
+
const records = await waitForRestartRecords(markerPath, 2)
|
|
313
|
+
|
|
314
|
+
assert.equal(typeof records[0]?.at, "number")
|
|
315
|
+
assert.equal(typeof records[1]?.at, "number")
|
|
316
|
+
assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `failed owner recovery retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
|
|
317
|
+
} finally {
|
|
318
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
319
|
+
}
|
|
320
|
+
})
|
|
321
|
+
|
|
322
|
+
test("a retired guardian-started owner does not block recovery of its replacement", async () => {
|
|
323
|
+
const fixture = await createGuardian()
|
|
324
|
+
const firstMarkerPath = path.join(fixture.root, "first-owner.pid")
|
|
325
|
+
const secondMarkerPath = path.join(fixture.root, "second-owner.jsonl")
|
|
326
|
+
const authority = {configDigest: "same", runtime: null}
|
|
327
|
+
const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
328
|
+
let firstOwnerPid
|
|
329
|
+
|
|
330
|
+
try {
|
|
331
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, firstMarkerPath))
|
|
332
|
+
fixture.client.disconnect()
|
|
333
|
+
firstOwnerPid = Number((await waitForFileText(firstMarkerPath)).trim())
|
|
334
|
+
await replacement.connect()
|
|
335
|
+
const prepared = await replacement.prepareOwnerReplacement(authority, authority)
|
|
336
|
+
const committed = replacement.waitForEvent("replacement-committed")
|
|
337
|
+
|
|
338
|
+
await replacement.stageOwnerReplacement(prepared.replacementId, recoveryOwnerState(authority, secondMarkerPath, "replacement", 20))
|
|
339
|
+
process.kill(firstOwnerPid, "SIGUSR1")
|
|
340
|
+
await committed
|
|
341
|
+
replacement.disconnect()
|
|
342
|
+
|
|
343
|
+
assert.deepEqual((await waitForRestartRecords(secondMarkerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "replacement"}])
|
|
344
|
+
} finally {
|
|
345
|
+
replacement.disconnect()
|
|
346
|
+
if (firstOwnerPid) {
|
|
347
|
+
try { process.kill(firstOwnerPid, "SIGKILL") } catch (_error) { /* Exact retired fixture owner already exited. */ }
|
|
348
|
+
}
|
|
349
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
350
|
+
}
|
|
351
|
+
})
|
|
352
|
+
|
|
353
|
+
test("guardian terminates a restart attempt that never claims ownership", async () => {
|
|
354
|
+
const fixture = await createGuardian()
|
|
355
|
+
const markerPath = path.join(fixture.root, "hung-owner.pid")
|
|
356
|
+
const authority = {configDigest: "same", runtime: null}
|
|
357
|
+
let hungPid
|
|
358
|
+
|
|
359
|
+
try {
|
|
360
|
+
await fixture.client.publishOwnerState({
|
|
361
|
+
authority,
|
|
362
|
+
recovery: {
|
|
363
|
+
command: {
|
|
364
|
+
args: ["-e", "require('node:fs').writeFileSync(process.argv[1], String(process.pid)); setInterval(() => {}, 1000)", markerPath],
|
|
365
|
+
cwd: fixture.root,
|
|
366
|
+
env: {},
|
|
367
|
+
executable: process.execPath
|
|
368
|
+
},
|
|
369
|
+
reconnectGraceMs: 0,
|
|
370
|
+
startupTimeoutMs: 1000
|
|
371
|
+
},
|
|
372
|
+
snapshot: {activeReleaseId: null}
|
|
373
|
+
})
|
|
374
|
+
fixture.client.disconnect()
|
|
375
|
+
hungPid = Number((await waitForFileText(markerPath)).trim())
|
|
376
|
+
|
|
377
|
+
await waitForProcessExit(hungPid)
|
|
378
|
+
} finally {
|
|
379
|
+
if (hungPid) {
|
|
380
|
+
try { process.kill(hungPid, "SIGKILL") } catch (_error) { /* Exact hung fixture owner already exited. */ }
|
|
381
|
+
}
|
|
382
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
383
|
+
}
|
|
384
|
+
})
|
|
385
|
+
|
|
386
|
+
test("guardian terminates descendants when a restart leader exits before claiming ownership", async () => {
|
|
387
|
+
const fixture = await createGuardian()
|
|
388
|
+
const descendantPath = path.join(fixture.root, "early-exit-descendant.pid")
|
|
389
|
+
const authority = {configDigest: "same", runtime: null}
|
|
390
|
+
let descendantPid
|
|
391
|
+
|
|
392
|
+
try {
|
|
393
|
+
await fixture.client.publishOwnerState({
|
|
394
|
+
authority,
|
|
395
|
+
recovery: {
|
|
396
|
+
command: {
|
|
397
|
+
args: ["-e", `const {spawn} = require("node:child_process"); const fs = require("node:fs"); const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {stdio: "ignore"}); fs.writeFileSync(${JSON.stringify(descendantPath)}, String(child.pid)); child.unref()`],
|
|
398
|
+
cwd: fixture.root,
|
|
399
|
+
env: {},
|
|
400
|
+
executable: process.execPath
|
|
401
|
+
},
|
|
402
|
+
reconnectGraceMs: 0,
|
|
403
|
+
startupTimeoutMs: 1000
|
|
404
|
+
},
|
|
405
|
+
snapshot: {activeReleaseId: null}
|
|
406
|
+
})
|
|
407
|
+
fixture.client.disconnect()
|
|
408
|
+
descendantPid = Number((await waitForFileText(descendantPath)).trim())
|
|
409
|
+
|
|
410
|
+
await waitForProcessExit(descendantPid)
|
|
411
|
+
} finally {
|
|
412
|
+
if (descendantPid) {
|
|
413
|
+
try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
|
|
414
|
+
}
|
|
415
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
416
|
+
}
|
|
417
|
+
})
|
|
418
|
+
|
|
419
|
+
test("guardian terminates a restarted owner process group that claims but never becomes ready", async () => {
|
|
420
|
+
const fixture = await createGuardian()
|
|
421
|
+
const descendantPath = path.join(fixture.root, "claimed-hung-descendant.pid")
|
|
422
|
+
const markerPath = path.join(fixture.root, "claimed-hung-owner.pid")
|
|
423
|
+
const authority = {configDigest: "same", runtime: null}
|
|
424
|
+
let descendantPid
|
|
425
|
+
let ownerPid
|
|
426
|
+
|
|
427
|
+
try {
|
|
428
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
|
|
429
|
+
descendantPath,
|
|
430
|
+
ready: false,
|
|
431
|
+
startupTimeoutMs: 100
|
|
432
|
+
}))
|
|
433
|
+
fixture.client.disconnect()
|
|
434
|
+
ownerPid = Number((await waitForFileText(markerPath)).trim())
|
|
435
|
+
descendantPid = Number((await waitForFileText(descendantPath)).trim())
|
|
436
|
+
|
|
437
|
+
await Promise.all([waitForProcessExit(ownerPid), waitForProcessExit(descendantPid)])
|
|
438
|
+
} finally {
|
|
439
|
+
if (ownerPid) killExactProcessGroup(ownerPid)
|
|
440
|
+
if (descendantPid) {
|
|
441
|
+
try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
|
|
442
|
+
}
|
|
443
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
444
|
+
}
|
|
445
|
+
})
|
|
446
|
+
|
|
447
|
+
test("guardian backs off when a restarted owner exits after claiming but before readiness", async () => {
|
|
448
|
+
const fixture = await createGuardian()
|
|
449
|
+
const descendantPath = path.join(fixture.root, "post-claim-descendant.pid")
|
|
450
|
+
const markerPath = path.join(fixture.root, "post-claim-exit.pid")
|
|
451
|
+
const startedLogPath = path.join(fixture.root, "post-claim-starts.jsonl")
|
|
452
|
+
const authority = {configDigest: "same", runtime: null}
|
|
453
|
+
let descendantPid
|
|
454
|
+
|
|
455
|
+
try {
|
|
456
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
|
|
457
|
+
descendantPath,
|
|
458
|
+
exitAfterClaim: true,
|
|
459
|
+
startedLogPath
|
|
460
|
+
}))
|
|
461
|
+
fixture.client.disconnect()
|
|
462
|
+
descendantPid = Number((await waitForFileText(descendantPath)).trim())
|
|
463
|
+
const records = await waitForRestartRecords(startedLogPath, 2)
|
|
464
|
+
|
|
465
|
+
await waitForProcessExit(descendantPid)
|
|
466
|
+
assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `post-claim failure retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
|
|
467
|
+
} finally {
|
|
468
|
+
if (descendantPid) {
|
|
469
|
+
try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
|
|
470
|
+
}
|
|
471
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
472
|
+
}
|
|
473
|
+
})
|
|
474
|
+
|
|
475
|
+
test("guardian preserves restart backoff when an unready owner disconnect aborts a prepared replacement", async () => {
|
|
476
|
+
const fixture = await createGuardian()
|
|
477
|
+
const markerPath = path.join(fixture.root, "prepared-post-claim-exit.pid")
|
|
478
|
+
const startedLogPath = path.join(fixture.root, "prepared-post-claim-starts.jsonl")
|
|
479
|
+
const authority = {configDigest: "same", runtime: null}
|
|
480
|
+
const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
481
|
+
|
|
482
|
+
try {
|
|
483
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
|
|
484
|
+
ready: false,
|
|
485
|
+
startedLogPath,
|
|
486
|
+
startupTimeoutMs: 150
|
|
487
|
+
}))
|
|
488
|
+
fixture.client.disconnect()
|
|
489
|
+
await waitForFileText(markerPath)
|
|
490
|
+
await replacement.connect()
|
|
491
|
+
const aborted = replacement.waitForEvent("replacement-aborted")
|
|
492
|
+
|
|
493
|
+
await replacement.prepareOwnerReplacement(authority, authority)
|
|
494
|
+
await aborted
|
|
495
|
+
const records = await waitForRestartRecords(startedLogPath, 2)
|
|
496
|
+
|
|
497
|
+
assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `replacement abort retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
|
|
498
|
+
} finally {
|
|
499
|
+
replacement.disconnect()
|
|
500
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
501
|
+
}
|
|
502
|
+
})
|
|
503
|
+
|
|
504
|
+
test("replacement commit preserves a claimed guardian restart child through listener retirement", async () => {
|
|
505
|
+
const fixture = await createGuardian()
|
|
506
|
+
const claimedPath = path.join(fixture.root, "claimed-restart-owner.pid")
|
|
507
|
+
const committedPath = path.join(fixture.root, "claimed-restart-owner-committed.txt")
|
|
508
|
+
const preparedPath = path.join(fixture.root, "claimed-restart-owner-prepared.txt")
|
|
509
|
+
const authority = {configDigest: "old", runtime: null}
|
|
510
|
+
const nextAuthority = {configDigest: "new", runtime: null}
|
|
511
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
512
|
+
/** @type {number | undefined} */
|
|
513
|
+
let recoveredOwnerPid
|
|
514
|
+
|
|
515
|
+
try {
|
|
516
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, claimedPath, {
|
|
517
|
+
ready: false,
|
|
518
|
+
replacementCommittedPath: committedPath,
|
|
519
|
+
replacementPreparedPath: preparedPath,
|
|
520
|
+
startupTimeoutMs: 5000
|
|
521
|
+
}))
|
|
522
|
+
fixture.client.disconnect()
|
|
523
|
+
const incumbentPid = Number((await waitForFileText(claimedPath)).trim())
|
|
524
|
+
|
|
525
|
+
recoveredOwnerPid = incumbentPid
|
|
526
|
+
await candidate.connect()
|
|
527
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
528
|
+
|
|
529
|
+
await waitForFileText(preparedPath, new RegExp(prepared.replacementId))
|
|
530
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: null}})
|
|
531
|
+
const published = candidate.waitForEvent("replacement-committed")
|
|
532
|
+
|
|
533
|
+
process.kill(incumbentPid, "SIGUSR2")
|
|
534
|
+
await Promise.race([waitForFileText(committedPath, new RegExp(prepared.replacementId)), published])
|
|
535
|
+
await new Promise((resolve) => setTimeout(resolve, 50))
|
|
536
|
+
assert.doesNotThrow(() => process.kill(incumbentPid, 0), "claimed incumbent must survive replacement commit until its listeners retire")
|
|
537
|
+
|
|
538
|
+
process.kill(incumbentPid, "SIGUSR1")
|
|
539
|
+
await published
|
|
540
|
+
await candidate.shutdown()
|
|
541
|
+
await fixture.client.guardianExit()
|
|
542
|
+
} finally {
|
|
543
|
+
candidate.disconnect()
|
|
544
|
+
if (recoveredOwnerPid) {
|
|
545
|
+
try { process.kill(-recoveredOwnerPid, "SIGKILL") } catch (_error) { /* Exact recovered fixture owner already exited. */ }
|
|
546
|
+
}
|
|
547
|
+
await cleanupGuardian(fixture)
|
|
548
|
+
}
|
|
549
|
+
})
|
|
550
|
+
|
|
551
|
+
test("ownerless replacement commit kills a superseded restart candidate", async () => {
|
|
552
|
+
const fixture = await createGuardian()
|
|
553
|
+
const delayedClaimPath = path.join(fixture.root, "delayed-claim.pid")
|
|
554
|
+
const firstMarkerPath = path.join(fixture.root, "delayed-owner-started.pid")
|
|
555
|
+
const secondMarkerPath = path.join(fixture.root, "committed-replacement.jsonl")
|
|
556
|
+
const authority = {configDigest: "old", runtime: null}
|
|
557
|
+
const nextAuthority = {configDigest: "new", runtime: null}
|
|
558
|
+
const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
559
|
+
let delayedOwnerPid
|
|
560
|
+
let staged = false
|
|
561
|
+
|
|
562
|
+
try {
|
|
563
|
+
await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, delayedClaimPath, {claimDelayMs: 500, startedPath: firstMarkerPath}))
|
|
564
|
+
fixture.client.disconnect()
|
|
565
|
+
delayedOwnerPid = Number((await waitForFileText(firstMarkerPath)).trim())
|
|
566
|
+
await replacement.connect()
|
|
567
|
+
const prepared = await replacement.prepareOwnerReplacement(authority, nextAuthority)
|
|
568
|
+
|
|
569
|
+
await waitForProcessExit(delayedOwnerPid)
|
|
570
|
+
const committed = replacement.waitForEvent("replacement-committed")
|
|
571
|
+
|
|
572
|
+
await replacement.stageOwnerReplacement(prepared.replacementId, recoveryOwnerState(nextAuthority, secondMarkerPath, "committed", 20))
|
|
573
|
+
staged = true
|
|
574
|
+
await committed
|
|
575
|
+
replacement.disconnect()
|
|
576
|
+
|
|
577
|
+
assert.deepEqual((await waitForRestartRecords(secondMarkerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "committed"}])
|
|
578
|
+
} finally {
|
|
579
|
+
replacement.disconnect()
|
|
580
|
+
if (delayedOwnerPid) {
|
|
581
|
+
try { process.kill(delayedOwnerPid, "SIGKILL") } catch (_error) { /* Exact superseded fixture owner already exited. */ }
|
|
582
|
+
}
|
|
583
|
+
await reconnectAndShutdownGuardian(fixture, staged ? nextAuthority : authority)
|
|
584
|
+
}
|
|
585
|
+
})
|
|
586
|
+
|
|
587
|
+
test("guardian logs an asynchronous daemon spawn failure before retrying", async () => {
|
|
588
|
+
const fixture = await createGuardian()
|
|
589
|
+
const logPath = path.join(fixture.root, "daemon.log")
|
|
590
|
+
const authority = {configDigest: "same", runtime: null}
|
|
591
|
+
const privateArgs = "private-argument-value"
|
|
592
|
+
const privateEnvironment = "private-environment-value"
|
|
593
|
+
const privateExecutable = path.join(fixture.root, "private-runtime", "missing-rollbridge")
|
|
594
|
+
|
|
595
|
+
try {
|
|
596
|
+
await fixture.client.publishOwnerState({
|
|
597
|
+
authority,
|
|
598
|
+
recovery: {
|
|
599
|
+
command: {
|
|
600
|
+
args: [privateArgs],
|
|
601
|
+
cwd: fixture.root,
|
|
602
|
+
env: {PRIVATE_ENVIRONMENT: privateEnvironment},
|
|
603
|
+
executable: privateExecutable,
|
|
604
|
+
logPath
|
|
605
|
+
},
|
|
606
|
+
reconnectGraceMs: 0,
|
|
607
|
+
startupTimeoutMs: 1000
|
|
608
|
+
},
|
|
609
|
+
snapshot: {activeReleaseId: null}
|
|
610
|
+
})
|
|
611
|
+
fixture.client.disconnect()
|
|
612
|
+
const diagnosticPattern = /"code":"ENOENT".*"message":"guardian failed to restart daemon"/
|
|
613
|
+
const diagnostic = await waitForFileText(logPath, diagnosticPattern)
|
|
614
|
+
|
|
615
|
+
assert.match(diagnostic, diagnosticPattern)
|
|
616
|
+
for (const privateValue of [privateArgs, privateEnvironment, privateExecutable, fixture.root, logPath]) {
|
|
617
|
+
assert.ok(!diagnostic.includes(privateValue), `guardian diagnostic exposed ${privateValue}`)
|
|
618
|
+
}
|
|
619
|
+
} finally {
|
|
620
|
+
await reconnectAndShutdownGuardian(fixture, authority)
|
|
621
|
+
}
|
|
622
|
+
})
|
|
623
|
+
|
|
624
|
+
test("guardian publishes the authenticated ready owner's PID file", async () => {
|
|
625
|
+
const fixture = await createGuardian()
|
|
626
|
+
const pidPath = path.join(fixture.root, "run", "daemon.pid")
|
|
627
|
+
const victimPath = path.join(fixture.root, "victim")
|
|
628
|
+
const authority = {configDigest: "same", runtime: null}
|
|
629
|
+
|
|
630
|
+
try {
|
|
631
|
+
await fixture.client.publishOwnerState({
|
|
632
|
+
authority,
|
|
633
|
+
recovery: {
|
|
634
|
+
command: {
|
|
635
|
+
args: [],
|
|
636
|
+
cwd: fixture.root,
|
|
637
|
+
env: {},
|
|
638
|
+
executable: process.execPath,
|
|
639
|
+
pidPath
|
|
640
|
+
},
|
|
641
|
+
reconnectGraceMs: 10,
|
|
642
|
+
startupTimeoutMs: 1000
|
|
643
|
+
},
|
|
644
|
+
snapshot: {activeReleaseId: null}
|
|
645
|
+
})
|
|
646
|
+
await fs.mkdir(path.dirname(pidPath), {recursive: true})
|
|
647
|
+
await fs.writeFile(victimPath, "unchanged\n")
|
|
648
|
+
await fs.symlink(victimPath, pidPath)
|
|
649
|
+
await fixture.client.ownerReady()
|
|
650
|
+
|
|
651
|
+
assert.equal(await fs.readFile(pidPath, "utf8"), `${process.pid}\n`)
|
|
652
|
+
assert.equal(await fs.readFile(victimPath, "utf8"), "unchanged\n")
|
|
653
|
+
} finally {
|
|
654
|
+
await cleanupGuardian(fixture)
|
|
655
|
+
}
|
|
656
|
+
})
|
|
657
|
+
|
|
156
658
|
test("replacement commit notification waits for incumbent listener retirement", async () => {
|
|
157
659
|
const fixture = await createGuardian()
|
|
158
660
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
@@ -170,6 +672,10 @@ test("replacement commit notification waits for incumbent listener retirement",
|
|
|
170
672
|
await fixture.client.commitOwnerReplacement(prepared.replacementId)
|
|
171
673
|
await new Promise((resolve) => setImmediate(resolve))
|
|
172
674
|
assert.equal(committed, false, "candidate publication must remain fenced while incumbent listener retirement is delayed")
|
|
675
|
+
const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
|
|
676
|
+
|
|
677
|
+
await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
|
|
678
|
+
await listenersRetired
|
|
173
679
|
await fixture.client.request({command: "finalize-owner-replacement", replacementId: prepared.replacementId})
|
|
174
680
|
await notification
|
|
175
681
|
assert.equal(committed, true)
|
|
@@ -181,6 +687,40 @@ test("replacement commit notification waits for incumbent listener retirement",
|
|
|
181
687
|
}
|
|
182
688
|
})
|
|
183
689
|
|
|
690
|
+
test("completed direct listener retirement finalizes when the incumbent disconnects", {timeout: 3000}, async () => {
|
|
691
|
+
const fixture = await createGuardian()
|
|
692
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
693
|
+
const authority = {configDigest: "incumbent", runtime: null}
|
|
694
|
+
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
695
|
+
|
|
696
|
+
try {
|
|
697
|
+
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v1"}})
|
|
698
|
+
await candidate.connect()
|
|
699
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
700
|
+
|
|
701
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})
|
|
702
|
+
const committed = candidate.waitForEvent("replacement-committed")
|
|
703
|
+
|
|
704
|
+
await fixture.client.commitOwnerReplacement(prepared.replacementId)
|
|
705
|
+
await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
|
|
706
|
+
fixture.client.disconnect()
|
|
707
|
+
await committed
|
|
708
|
+
await candidate.finalizeOwnerReplacement(prepared.replacementId)
|
|
709
|
+
assert.deepEqual(await candidate.replacementStatus(), {
|
|
710
|
+
committedReplacementId: prepared.replacementId,
|
|
711
|
+
ownerClaimed: true,
|
|
712
|
+
retirementFailed: false,
|
|
713
|
+
retirementPending: false,
|
|
714
|
+
retirementReady: false
|
|
715
|
+
})
|
|
716
|
+
await candidate.shutdown()
|
|
717
|
+
await fixture.client.guardianExit()
|
|
718
|
+
} finally {
|
|
719
|
+
candidate.disconnect()
|
|
720
|
+
await cleanupGuardian(fixture)
|
|
721
|
+
}
|
|
722
|
+
})
|
|
723
|
+
|
|
184
724
|
test("replacement staging rejects owner state published after prepare", async () => {
|
|
185
725
|
const fixture = await createGuardian()
|
|
186
726
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
@@ -210,6 +750,115 @@ test("replacement staging rejects owner state published after prepare", async ()
|
|
|
210
750
|
}
|
|
211
751
|
})
|
|
212
752
|
|
|
753
|
+
test("staged replacement receives cleared local sources when the incumbent disconnects", {timeout: 3000}, async () => {
|
|
754
|
+
const fixture = await createGuardian()
|
|
755
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
756
|
+
const authority = {configDigest: "incumbent", runtime: null}
|
|
757
|
+
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
758
|
+
const snapshot = {activeReleaseId: "v1"}
|
|
759
|
+
|
|
760
|
+
try {
|
|
761
|
+
await fixture.client.publishOwnerState({
|
|
762
|
+
authority,
|
|
763
|
+
listenerConnectionSources: {"incumbent-local": {v1: {http: 0, websocket: 1}}},
|
|
764
|
+
listenerSourceId: "incumbent-local",
|
|
765
|
+
snapshot
|
|
766
|
+
})
|
|
767
|
+
await candidate.connect()
|
|
768
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
769
|
+
|
|
770
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {
|
|
771
|
+
authority: nextAuthority,
|
|
772
|
+
listenerConnectionSources: {"incumbent-local": {v1: {http: 0, websocket: 1}}},
|
|
773
|
+
listenerSourceId: "candidate-local",
|
|
774
|
+
snapshot
|
|
775
|
+
})
|
|
776
|
+
const cleared = candidate.waitForEvent("owner-connection-state")
|
|
777
|
+
const committed = candidate.waitForEvent("replacement-committed")
|
|
778
|
+
|
|
779
|
+
fixture.client.disconnect()
|
|
780
|
+
assert.deepEqual(await cleared, {
|
|
781
|
+
connections: {http: 0, websocket: 0},
|
|
782
|
+
event: "owner-connection-state",
|
|
783
|
+
releaseId: "v1",
|
|
784
|
+
sourceId: "incumbent-local"
|
|
785
|
+
})
|
|
786
|
+
await committed
|
|
787
|
+
const state = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await candidate.ownerState())
|
|
788
|
+
|
|
789
|
+
assert.deepEqual(state.listenerConnectionSources, {})
|
|
790
|
+
await candidate.shutdown()
|
|
791
|
+
await fixture.client.guardianExit()
|
|
792
|
+
} finally {
|
|
793
|
+
candidate.disconnect()
|
|
794
|
+
await cleanupGuardian(fixture)
|
|
795
|
+
}
|
|
796
|
+
})
|
|
797
|
+
|
|
798
|
+
test("staged successor state receives tombstones when an older completed listener disconnects", {timeout: 3000}, async () => {
|
|
799
|
+
const fixture = await createGuardian()
|
|
800
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
801
|
+
const successor = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
802
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
803
|
+
const snapshot = {activeReleaseId: "v1"}
|
|
804
|
+
|
|
805
|
+
try {
|
|
806
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
807
|
+
await candidate.connect()
|
|
808
|
+
const first = await candidate.prepareOwnerReplacement(authority, authority)
|
|
809
|
+
|
|
810
|
+
await candidate.stageOwnerReplacement(first.replacementId, {authority, listenerSourceId: "candidate-local", snapshot})
|
|
811
|
+
const firstCommitted = candidate.waitForEvent("replacement-committed")
|
|
812
|
+
|
|
813
|
+
await fixture.client.commitOwnerReplacement(first.replacementId)
|
|
814
|
+
const sourcePublished = candidate.waitForEvent("owner-connection-state")
|
|
815
|
+
|
|
816
|
+
await fixture.client.publishOwnerConnectionState(first.replacementId, "retired-local", "v1", {http: 0, websocket: 1}, true)
|
|
817
|
+
await sourcePublished
|
|
818
|
+
const firstListenersRetired = candidate.waitForEvent("replacement-listeners-retired")
|
|
819
|
+
|
|
820
|
+
await fixture.client.completeOwnerListenerRetirement(first.replacementId)
|
|
821
|
+
await firstListenersRetired
|
|
822
|
+
await candidate.finalizeOwnerReplacement(first.replacementId)
|
|
823
|
+
await firstCommitted
|
|
824
|
+
|
|
825
|
+
await successor.connect()
|
|
826
|
+
const second = await successor.prepareOwnerReplacement(authority, authority)
|
|
827
|
+
|
|
828
|
+
await successor.stageOwnerReplacement(second.replacementId, {
|
|
829
|
+
authority,
|
|
830
|
+
listenerConnectionSources: {"retired-local": {v1: {http: 0, websocket: 1}}},
|
|
831
|
+
listenerSourceId: "successor-local",
|
|
832
|
+
snapshot
|
|
833
|
+
})
|
|
834
|
+
const sourceCleared = candidate.waitForEvent("owner-connection-state")
|
|
835
|
+
const stagedSourceCleared = successor.waitForEvent("owner-connection-state")
|
|
836
|
+
|
|
837
|
+
fixture.client.disconnect()
|
|
838
|
+
const tombstone = {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "retired-local"}
|
|
839
|
+
|
|
840
|
+
assert.deepEqual(await Promise.all([sourceCleared, stagedSourceCleared]), [tombstone, tombstone])
|
|
841
|
+
|
|
842
|
+
await candidate.commitOwnerReplacement(second.replacementId)
|
|
843
|
+
const successorState = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await successor.ownerState())
|
|
844
|
+
|
|
845
|
+
assert.deepEqual(successorState.listenerConnectionSources, {})
|
|
846
|
+
const secondCommitted = successor.waitForEvent("replacement-committed")
|
|
847
|
+
const secondListenersRetired = successor.waitForEvent("replacement-listeners-retired")
|
|
848
|
+
|
|
849
|
+
await candidate.completeOwnerListenerRetirement(second.replacementId)
|
|
850
|
+
await secondListenersRetired
|
|
851
|
+
await successor.finalizeOwnerReplacement(second.replacementId)
|
|
852
|
+
await secondCommitted
|
|
853
|
+
await successor.shutdown()
|
|
854
|
+
await fixture.client.guardianExit()
|
|
855
|
+
} finally {
|
|
856
|
+
successor.disconnect()
|
|
857
|
+
candidate.disconnect()
|
|
858
|
+
await cleanupGuardian(fixture)
|
|
859
|
+
}
|
|
860
|
+
})
|
|
861
|
+
|
|
213
862
|
test("replacement abort notifies both the candidate and committed owner", async () => {
|
|
214
863
|
const fixture = await createGuardian()
|
|
215
864
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
@@ -298,76 +947,365 @@ test("guardian owner-replacement capability classification is explicit and fail
|
|
|
298
947
|
}
|
|
299
948
|
})
|
|
300
949
|
|
|
301
|
-
test("reserved process recovery rejects a reconstructed definition with different provenance", async () => {
|
|
950
|
+
test("reserved process recovery rejects a reconstructed definition with different provenance", async () => {
|
|
951
|
+
const fixture = await createGuardian()
|
|
952
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
953
|
+
const processKey = "release:v1:worker"
|
|
954
|
+
|
|
955
|
+
try {
|
|
956
|
+
await fixture.client.process(processKey, definition("worker")).recover()
|
|
957
|
+
const [registration] = await fixture.client.inventory()
|
|
958
|
+
|
|
959
|
+
assert.ok(registration)
|
|
960
|
+
await candidate.connect()
|
|
961
|
+
candidate.reserveProcessRecovery(processKey, registration.provenance)
|
|
962
|
+
await assert.rejects(
|
|
963
|
+
() => candidate.process(processKey, definition("different-worker")).recover(),
|
|
964
|
+
/provenance mismatch for reserved process/
|
|
965
|
+
)
|
|
966
|
+
assert.deepEqual((await fixture.client.inventory()).map(({key}) => key), [processKey])
|
|
967
|
+
} finally {
|
|
968
|
+
candidate.disconnect()
|
|
969
|
+
await cleanupGuardian(fixture)
|
|
970
|
+
}
|
|
971
|
+
})
|
|
972
|
+
|
|
973
|
+
test("retired owner replacement rejects a registered process absent from committed owner state", async () => {
|
|
974
|
+
const fixture = await createGuardian()
|
|
975
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
976
|
+
const committedProcessKey = "release:v1:worker"
|
|
977
|
+
const candidateProcessKey = "release:candidate:worker"
|
|
978
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
979
|
+
const snapshot = {
|
|
980
|
+
activeReleaseId: "v1",
|
|
981
|
+
control: {path: path.join(fixture.root, "rollbridge.sock")},
|
|
982
|
+
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
983
|
+
services: [],
|
|
984
|
+
singletons: []
|
|
985
|
+
}
|
|
986
|
+
const candidateSnapshot = {
|
|
987
|
+
...snapshot,
|
|
988
|
+
releases: [...snapshot.releases, {processes: [{id: "worker"}], releaseId: "candidate"}]
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
try {
|
|
992
|
+
await fixture.client.process(committedProcessKey, definition("worker")).recover()
|
|
993
|
+
await fixture.client.process(candidateProcessKey, definition("candidate-worker")).recover()
|
|
994
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
995
|
+
await candidate.connect()
|
|
996
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
997
|
+
|
|
998
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot: candidateSnapshot})
|
|
999
|
+
await assert.rejects(
|
|
1000
|
+
() => candidate.commitRetiredOwnerReplacement(prepared.replacementId, candidateProcessKey),
|
|
1001
|
+
/process .* does not belong to the committed owner/
|
|
1002
|
+
)
|
|
1003
|
+
} finally {
|
|
1004
|
+
candidate.disconnect()
|
|
1005
|
+
await cleanupGuardian(fixture)
|
|
1006
|
+
}
|
|
1007
|
+
})
|
|
1008
|
+
|
|
1009
|
+
test("retired owner replacement requires unchanged authority and the exact control path absent", async () => {
|
|
1010
|
+
const fixture = await createGuardian()
|
|
1011
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1012
|
+
const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1013
|
+
const controlPath = path.join(fixture.root, "rollbridge.sock")
|
|
1014
|
+
const processKey = "release:v1:worker"
|
|
1015
|
+
const authority = {configDigest: "incumbent", runtime: null}
|
|
1016
|
+
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
1017
|
+
const snapshot = {
|
|
1018
|
+
activeReleaseId: "v1",
|
|
1019
|
+
control: {path: controlPath},
|
|
1020
|
+
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
1021
|
+
services: [],
|
|
1022
|
+
singletons: []
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
try {
|
|
1026
|
+
await fixture.client.process(processKey, definition("worker")).recover()
|
|
1027
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
1028
|
+
await candidate.connect()
|
|
1029
|
+
const changed = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
1030
|
+
|
|
1031
|
+
await candidate.stageOwnerReplacement(changed.replacementId, {authority: nextAuthority, snapshot})
|
|
1032
|
+
await assert.rejects(() => candidate.commitRetiredOwnerReplacement(changed.replacementId, processKey), /unchanged owner authority/)
|
|
1033
|
+
await candidate.abortOwnerReplacement(changed.replacementId)
|
|
1034
|
+
|
|
1035
|
+
const occupied = await candidate.prepareOwnerReplacement(authority, authority)
|
|
1036
|
+
|
|
1037
|
+
await candidate.stageOwnerReplacement(occupied.replacementId, {authority, snapshot})
|
|
1038
|
+
await fs.writeFile(controlPath, "occupied\n")
|
|
1039
|
+
await assert.rejects(() => candidate.commitRetiredOwnerReplacement(occupied.replacementId, processKey), /control socket .* still exists/)
|
|
1040
|
+
await candidate.abortOwnerReplacement(occupied.replacementId)
|
|
1041
|
+
|
|
1042
|
+
await fs.rm(controlPath)
|
|
1043
|
+
const ready = await candidate.prepareOwnerReplacement(authority, authority)
|
|
1044
|
+
|
|
1045
|
+
await candidate.stageOwnerReplacement(ready.replacementId, {authority, snapshot})
|
|
1046
|
+
await contender.connect()
|
|
1047
|
+
await assert.rejects(
|
|
1048
|
+
() => contender.request({command: "commit-retired-owner-replacement", key: processKey, replacementId: ready.replacementId}),
|
|
1049
|
+
/not the prepared candidate/
|
|
1050
|
+
)
|
|
1051
|
+
await assert.rejects(
|
|
1052
|
+
() => candidate.commitRetiredOwnerReplacement("stale-replacement", processKey),
|
|
1053
|
+
/not the prepared candidate/
|
|
1054
|
+
)
|
|
1055
|
+
await assert.rejects(
|
|
1056
|
+
() => candidate.commitRetiredOwnerReplacement(ready.replacementId, "release:v1:wrong"),
|
|
1057
|
+
/process .* is not registered/
|
|
1058
|
+
)
|
|
1059
|
+
const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
|
|
1060
|
+
|
|
1061
|
+
void handoffRequested.catch(() => undefined)
|
|
1062
|
+
|
|
1063
|
+
await candidate.prepareRetiredOwnerListenerHandoff(ready.replacementId, processKey)
|
|
1064
|
+
await handoffRequested
|
|
1065
|
+
await assert.rejects(
|
|
1066
|
+
() => contender.prepareOwnerReplacement(authority, authority),
|
|
1067
|
+
/listener retirement is pending/
|
|
1068
|
+
)
|
|
1069
|
+
const committed = candidate.waitForEvent("replacement-committed")
|
|
1070
|
+
const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
|
|
1071
|
+
const connectionState = candidate.waitForEvent("owner-connection-state")
|
|
1072
|
+
|
|
1073
|
+
await fixture.client.publishOwnerConnectionState(ready.replacementId, "listener-a", "v1", {http: 1, websocket: 2}, true)
|
|
1074
|
+
assert.deepEqual(await connectionState, {connections: {http: 1, websocket: 2}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
|
|
1075
|
+
await fixture.client.completeOwnerListenerRetirement(ready.replacementId)
|
|
1076
|
+
await listenersRetired
|
|
1077
|
+
const retirementRequested = fixture.client.waitForEvent("replacement-retirement-requested")
|
|
1078
|
+
|
|
1079
|
+
await candidate.commitRetiredOwnerReplacement(ready.replacementId, processKey)
|
|
1080
|
+
await retirementRequested
|
|
1081
|
+
await candidate.finalizeOwnerReplacement(ready.replacementId)
|
|
1082
|
+
await committed
|
|
1083
|
+
await candidate.shutdown()
|
|
1084
|
+
await fixture.client.guardianExit()
|
|
1085
|
+
} finally {
|
|
1086
|
+
contender.disconnect()
|
|
1087
|
+
candidate.disconnect()
|
|
1088
|
+
await cleanupGuardian(fixture)
|
|
1089
|
+
}
|
|
1090
|
+
})
|
|
1091
|
+
|
|
1092
|
+
test("completed listener retirement survives owner recovery and clears a crashed local source", async () => {
|
|
1093
|
+
const fixture = await createGuardian()
|
|
1094
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1095
|
+
const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1096
|
+
const controlPath = path.join(fixture.root, "rollbridge.sock")
|
|
1097
|
+
const processKey = "release:v1:worker"
|
|
1098
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
1099
|
+
const snapshot = {
|
|
1100
|
+
activeReleaseId: "v1",
|
|
1101
|
+
control: {path: controlPath},
|
|
1102
|
+
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
1103
|
+
services: [],
|
|
1104
|
+
singletons: []
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
try {
|
|
1108
|
+
await fixture.client.process(processKey, definition("worker")).recover()
|
|
1109
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
1110
|
+
await candidate.connect()
|
|
1111
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
1112
|
+
|
|
1113
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {
|
|
1114
|
+
authority,
|
|
1115
|
+
listenerConnectionSources: {"candidate-local": {v1: {http: 0, websocket: 1}}},
|
|
1116
|
+
listenerSourceId: "candidate-local",
|
|
1117
|
+
snapshot
|
|
1118
|
+
})
|
|
1119
|
+
const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
|
|
1120
|
+
|
|
1121
|
+
void handoffRequested.catch(() => undefined)
|
|
1122
|
+
|
|
1123
|
+
await candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
|
|
1124
|
+
await handoffRequested
|
|
1125
|
+
const initial = candidate.waitForEvent("owner-connection-state")
|
|
1126
|
+
|
|
1127
|
+
await fixture.client.publishOwnerConnectionState(prepared.replacementId, "listener-a", "v1", {http: 0, websocket: 1}, true)
|
|
1128
|
+
assert.deepEqual(await initial, {connections: {http: 0, websocket: 1}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
|
|
1129
|
+
const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
|
|
1130
|
+
|
|
1131
|
+
await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
|
|
1132
|
+
await listenersRetired
|
|
1133
|
+
await candidate.commitRetiredOwnerReplacement(prepared.replacementId, processKey)
|
|
1134
|
+
candidate.disconnect()
|
|
1135
|
+
await recovered.connect()
|
|
1136
|
+
await recovered.claimOwner(1000, authority)
|
|
1137
|
+
const stateAfterCandidateCrash = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
|
|
1138
|
+
|
|
1139
|
+
assert.equal(stateAfterCandidateCrash.listenerConnectionSources?.["candidate-local"], undefined)
|
|
1140
|
+
assert.deepEqual(stateAfterCandidateCrash.listenerConnectionSources?.["listener-a"], {v1: {http: 0, websocket: 1}})
|
|
1141
|
+
assert.deepEqual(await recovered.replacementStatus(), {
|
|
1142
|
+
committedReplacementId: prepared.replacementId,
|
|
1143
|
+
ownerClaimed: true,
|
|
1144
|
+
retirementFailed: false,
|
|
1145
|
+
retirementPending: true,
|
|
1146
|
+
retirementReady: true
|
|
1147
|
+
})
|
|
1148
|
+
await recovered.finalizeOwnerReplacement(prepared.replacementId)
|
|
1149
|
+
const cleared = recovered.waitForEvent("owner-connection-state")
|
|
1150
|
+
|
|
1151
|
+
fixture.client.disconnect()
|
|
1152
|
+
assert.deepEqual(await Promise.race([
|
|
1153
|
+
cleared,
|
|
1154
|
+
new Promise((_, reject) => {
|
|
1155
|
+
const timer = setTimeout(() => reject(new Error("Recovered owner did not receive the retired source tombstone")), 500)
|
|
1156
|
+
|
|
1157
|
+
timer.unref()
|
|
1158
|
+
})
|
|
1159
|
+
]), {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
|
|
1160
|
+
const recoveredOwnerState = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
|
|
1161
|
+
|
|
1162
|
+
assert.deepEqual(recoveredOwnerState.listenerConnectionSources, {})
|
|
1163
|
+
await recovered.publishOwnerState({
|
|
1164
|
+
authority,
|
|
1165
|
+
listenerConnectionSources: {"listener-a": {v1: {http: 0, websocket: 1}}},
|
|
1166
|
+
listenerSourceId: "recovered-local",
|
|
1167
|
+
snapshot
|
|
1168
|
+
})
|
|
1169
|
+
const afterStalePublication = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
|
|
1170
|
+
|
|
1171
|
+
assert.deepEqual(afterStalePublication.listenerConnectionSources, {})
|
|
1172
|
+
const recoveredProcess = recovered.process(processKey, definition("worker"))
|
|
1173
|
+
|
|
1174
|
+
await recoveredProcess.recover()
|
|
1175
|
+
await recoveredProcess.updateDefinition({...definition("worker"), env: {REVISION: "stale"}}, {
|
|
1176
|
+
authority,
|
|
1177
|
+
listenerConnectionSources: {"listener-a": {v1: {http: 0, websocket: 1}}},
|
|
1178
|
+
listenerSourceId: "recovered-local",
|
|
1179
|
+
snapshot
|
|
1180
|
+
})
|
|
1181
|
+
const afterStaleProcessUpdate = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
|
|
1182
|
+
|
|
1183
|
+
assert.deepEqual(afterStaleProcessUpdate.listenerConnectionSources, {})
|
|
1184
|
+
await recovered.shutdown()
|
|
1185
|
+
await fixture.client.guardianExit()
|
|
1186
|
+
} finally {
|
|
1187
|
+
recovered.disconnect()
|
|
1188
|
+
candidate.disconnect()
|
|
1189
|
+
await cleanupGuardian(fixture)
|
|
1190
|
+
}
|
|
1191
|
+
})
|
|
1192
|
+
|
|
1193
|
+
test("direct retired-listener source relay survives committed owner recovery", async () => {
|
|
302
1194
|
const fixture = await createGuardian()
|
|
303
1195
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
304
|
-
const
|
|
1196
|
+
const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1197
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
1198
|
+
const snapshot = {activeReleaseId: "v1", control: {path: path.join(fixture.root, "rollbridge.sock")}, releases: [], services: [], singletons: []}
|
|
305
1199
|
|
|
306
1200
|
try {
|
|
307
|
-
await fixture.client.
|
|
308
|
-
const [registration] = await fixture.client.inventory()
|
|
309
|
-
|
|
310
|
-
assert.ok(registration)
|
|
1201
|
+
await fixture.client.publishOwnerState({authority, snapshot})
|
|
311
1202
|
await candidate.connect()
|
|
312
|
-
candidate.
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
)
|
|
317
|
-
|
|
1203
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
1204
|
+
|
|
1205
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, listenerSourceId: "candidate-local", snapshot})
|
|
1206
|
+
await fixture.client.commitOwnerReplacement(prepared.replacementId)
|
|
1207
|
+
const initial = candidate.waitForEvent("owner-connection-state")
|
|
1208
|
+
|
|
1209
|
+
await fixture.client.publishOwnerConnectionState(prepared.replacementId, "direct-listener", "v1", {http: 0, websocket: 1}, true)
|
|
1210
|
+
assert.deepEqual(await initial, {connections: {http: 0, websocket: 1}, event: "owner-connection-state", releaseId: "v1", sourceId: "direct-listener"})
|
|
1211
|
+
const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
|
|
1212
|
+
|
|
1213
|
+
await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
|
|
1214
|
+
await listenersRetired
|
|
1215
|
+
await fixture.client.finalizeOwnerReplacement(prepared.replacementId)
|
|
1216
|
+
candidate.disconnect()
|
|
1217
|
+
await recovered.connect()
|
|
1218
|
+
await recovered.claimOwner(1000, authority)
|
|
1219
|
+
const cleared = recovered.waitForEvent("owner-connection-state")
|
|
1220
|
+
|
|
1221
|
+
await fixture.client.publishOwnerConnectionState(prepared.replacementId, "direct-listener", "v1", {http: 0, websocket: 0}, true)
|
|
1222
|
+
assert.deepEqual(await cleared, {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "direct-listener"})
|
|
1223
|
+
await recovered.shutdown()
|
|
1224
|
+
await fixture.client.guardianExit()
|
|
318
1225
|
} finally {
|
|
1226
|
+
recovered.disconnect()
|
|
319
1227
|
candidate.disconnect()
|
|
320
1228
|
await cleanupGuardian(fixture)
|
|
321
1229
|
}
|
|
322
1230
|
})
|
|
323
1231
|
|
|
324
|
-
test("
|
|
1232
|
+
test("incumbent listener disconnect before state completion aborts without committing the candidate", async () => {
|
|
325
1233
|
const fixture = await createGuardian()
|
|
326
1234
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
327
|
-
const
|
|
328
|
-
const
|
|
1235
|
+
const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1236
|
+
const controlPath = path.join(fixture.root, "rollbridge.sock")
|
|
1237
|
+
const processKey = "release:v1:worker"
|
|
329
1238
|
const authority = {configDigest: "owner", runtime: null}
|
|
330
1239
|
const snapshot = {
|
|
331
1240
|
activeReleaseId: "v1",
|
|
332
|
-
control: {path:
|
|
1241
|
+
control: {path: controlPath},
|
|
333
1242
|
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
334
1243
|
services: [],
|
|
335
1244
|
singletons: []
|
|
336
1245
|
}
|
|
337
|
-
const candidateSnapshot = {
|
|
338
|
-
...snapshot,
|
|
339
|
-
releases: [...snapshot.releases, {processes: [{id: "worker"}], releaseId: "candidate"}]
|
|
340
|
-
}
|
|
341
1246
|
|
|
342
1247
|
try {
|
|
343
|
-
await fixture.client.process(
|
|
344
|
-
await fixture.client.process(candidateProcessKey, definition("candidate-worker")).recover()
|
|
1248
|
+
await fixture.client.process(processKey, definition("worker")).recover()
|
|
345
1249
|
await fixture.client.publishOwnerState({authority, snapshot})
|
|
346
1250
|
await candidate.connect()
|
|
347
1251
|
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
348
1252
|
|
|
349
|
-
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
1253
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot})
|
|
1254
|
+
const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
|
|
1255
|
+
const failed = candidate.waitForEvent("replacement-retirement-failed")
|
|
1256
|
+
const aborted = candidate.waitForEvent("replacement-aborted")
|
|
1257
|
+
let tombstones = 0
|
|
1258
|
+
|
|
1259
|
+
candidate.onEvent("owner-connection-state", (event) => {
|
|
1260
|
+
if (event.sourceId === "incumbent-local" && event.releaseId === "v1" && event.connections && typeof event.connections === "object" && !Array.isArray(event.connections) && event.connections.http === 0 && event.connections.websocket === 0) {
|
|
1261
|
+
tombstones += 1
|
|
1262
|
+
}
|
|
1263
|
+
})
|
|
1264
|
+
|
|
1265
|
+
await candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
|
|
1266
|
+
await handoffRequested
|
|
1267
|
+
const sourcePublished = candidate.waitForEvent("owner-connection-state")
|
|
1268
|
+
|
|
1269
|
+
await fixture.client.publishOwnerConnectionState(prepared.replacementId, "incumbent-local", "v1", {http: 0, websocket: 1}, true)
|
|
1270
|
+
await sourcePublished
|
|
1271
|
+
fixture.client.disconnect()
|
|
1272
|
+
assert.deepEqual(await failed, {
|
|
1273
|
+
event: "replacement-retirement-failed",
|
|
1274
|
+
reason: "Incumbent listener disconnected during the prepared handoff",
|
|
1275
|
+
replacementId: prepared.replacementId
|
|
1276
|
+
})
|
|
1277
|
+
assert.deepEqual(await aborted, {
|
|
1278
|
+
event: "replacement-aborted",
|
|
1279
|
+
reason: "Incumbent listener disconnected during the prepared handoff"
|
|
1280
|
+
})
|
|
1281
|
+
assert.equal(tombstones, 1)
|
|
1282
|
+
assert.deepEqual(await candidate.replacementStatus(), {
|
|
1283
|
+
committedReplacementId: null,
|
|
1284
|
+
ownerClaimed: false,
|
|
1285
|
+
retirementFailed: false,
|
|
1286
|
+
retirementPending: false,
|
|
1287
|
+
retirementReady: false
|
|
1288
|
+
})
|
|
1289
|
+
await recovered.connect()
|
|
1290
|
+
await recovered.claimOwner(1000, authority)
|
|
1291
|
+
await recovered.shutdown()
|
|
1292
|
+
await fixture.client.guardianExit()
|
|
354
1293
|
} finally {
|
|
1294
|
+
recovered.disconnect()
|
|
355
1295
|
candidate.disconnect()
|
|
356
1296
|
await cleanupGuardian(fixture)
|
|
357
1297
|
}
|
|
358
1298
|
})
|
|
359
1299
|
|
|
360
|
-
test("
|
|
1300
|
+
test("replacement abort during listener handoff validation leaves the incumbent available", async () => {
|
|
361
1301
|
const fixture = await createGuardian()
|
|
362
1302
|
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
363
1303
|
const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
364
|
-
const controlPath = path.join(fixture.root, "rollbridge.sock")
|
|
365
1304
|
const processKey = "release:v1:worker"
|
|
366
|
-
const authority = {configDigest: "
|
|
367
|
-
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
1305
|
+
const authority = {configDigest: "owner", runtime: null}
|
|
368
1306
|
const snapshot = {
|
|
369
1307
|
activeReleaseId: "v1",
|
|
370
|
-
control: {path:
|
|
1308
|
+
control: {path: path.join(fixture.root, "rollbridge.sock")},
|
|
371
1309
|
releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
|
|
372
1310
|
services: [],
|
|
373
1311
|
singletons: []
|
|
@@ -377,45 +1315,68 @@ test("retired owner replacement requires unchanged authority and the exact contr
|
|
|
377
1315
|
await fixture.client.process(processKey, definition("worker")).recover()
|
|
378
1316
|
await fixture.client.publishOwnerState({authority, snapshot})
|
|
379
1317
|
await candidate.connect()
|
|
380
|
-
const
|
|
1318
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, authority)
|
|
381
1319
|
|
|
382
|
-
await candidate.stageOwnerReplacement(
|
|
383
|
-
|
|
384
|
-
await candidate.abortOwnerReplacement(changed.replacementId)
|
|
1320
|
+
await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot})
|
|
1321
|
+
const abandonedHandoff = candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
|
|
385
1322
|
|
|
386
|
-
|
|
1323
|
+
await candidate.abortOwnerReplacement(prepared.replacementId)
|
|
1324
|
+
await assert.rejects(() => abandonedHandoff, /prepared candidate/)
|
|
1325
|
+
await contender.connect()
|
|
1326
|
+
const fresh = await contender.prepareOwnerReplacement(authority, authority)
|
|
387
1327
|
|
|
388
|
-
await
|
|
389
|
-
await
|
|
390
|
-
await
|
|
391
|
-
|
|
1328
|
+
await contender.abortOwnerReplacement(fresh.replacementId)
|
|
1329
|
+
await fixture.client.shutdown()
|
|
1330
|
+
await fixture.client.guardianExit()
|
|
1331
|
+
} finally {
|
|
1332
|
+
contender.disconnect()
|
|
1333
|
+
candidate.disconnect()
|
|
1334
|
+
await cleanupGuardian(fixture)
|
|
1335
|
+
}
|
|
1336
|
+
})
|
|
392
1337
|
|
|
393
|
-
|
|
394
|
-
|
|
1338
|
+
test("explicit replacement abort notifies the incumbent owner", async () => {
|
|
1339
|
+
const fixture = await createGuardian()
|
|
1340
|
+
const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1341
|
+
const authority = {configDigest: "incumbent", runtime: null}
|
|
1342
|
+
const nextAuthority = {configDigest: "candidate", runtime: null}
|
|
395
1343
|
|
|
396
|
-
|
|
1344
|
+
try {
|
|
1345
|
+
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v1"}})
|
|
1346
|
+
await candidate.connect()
|
|
1347
|
+
const aborted = fixture.client.waitForEvent("replacement-aborted")
|
|
1348
|
+
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
1349
|
+
|
|
1350
|
+
await candidate.abortOwnerReplacement(prepared.replacementId)
|
|
1351
|
+
await aborted
|
|
1352
|
+
await fixture.client.shutdown()
|
|
1353
|
+
await fixture.client.guardianExit()
|
|
1354
|
+
} finally {
|
|
1355
|
+
candidate.disconnect()
|
|
1356
|
+
await cleanupGuardian(fixture)
|
|
1357
|
+
}
|
|
1358
|
+
})
|
|
1359
|
+
|
|
1360
|
+
test("queued owner claim is revalidated against the latest committed authority", async () => {
|
|
1361
|
+
const fixture = await createGuardian()
|
|
1362
|
+
const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1363
|
+
const authority = {configDigest: "old", runtime: null}
|
|
1364
|
+
const nextAuthority = {configDigest: "new", runtime: null}
|
|
1365
|
+
|
|
1366
|
+
try {
|
|
1367
|
+
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: null}})
|
|
397
1368
|
await contender.connect()
|
|
398
|
-
|
|
399
|
-
() => contender.request({command: "commit-retired-owner-replacement", key: processKey, replacementId: ready.replacementId}),
|
|
400
|
-
/not the prepared candidate/
|
|
401
|
-
)
|
|
402
|
-
await assert.rejects(
|
|
403
|
-
() => candidate.commitRetiredOwnerReplacement("stale-replacement", processKey),
|
|
404
|
-
/not the prepared candidate/
|
|
405
|
-
)
|
|
406
|
-
await assert.rejects(
|
|
407
|
-
() => candidate.commitRetiredOwnerReplacement(ready.replacementId, "release:v1:wrong"),
|
|
408
|
-
/process .* is not registered/
|
|
409
|
-
)
|
|
410
|
-
const committed = candidate.waitForEvent("replacement-committed")
|
|
1369
|
+
const claim = contender.claimOwner(500, authority)
|
|
411
1370
|
|
|
412
|
-
await
|
|
413
|
-
await
|
|
414
|
-
|
|
1371
|
+
await new Promise((resolve) => setTimeout(resolve, 20))
|
|
1372
|
+
await fixture.client.publishOwnerState({authority: nextAuthority, snapshot: {activeReleaseId: null}})
|
|
1373
|
+
fixture.client.disconnect()
|
|
1374
|
+
await assert.rejects(claim, /authority changed while the claim was queued/)
|
|
1375
|
+
await contender.claimOwner(500, nextAuthority)
|
|
1376
|
+
await contender.shutdown()
|
|
415
1377
|
await fixture.client.guardianExit()
|
|
416
1378
|
} finally {
|
|
417
1379
|
contender.disconnect()
|
|
418
|
-
candidate.disconnect()
|
|
419
1380
|
await cleanupGuardian(fixture)
|
|
420
1381
|
}
|
|
421
1382
|
})
|
|
@@ -429,6 +1390,7 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
|
|
|
429
1390
|
|
|
430
1391
|
try {
|
|
431
1392
|
await legacyProcess.start()
|
|
1393
|
+
await assert.rejects(() => fixture.client.capabilities(), /Guardian capabilities requires a process key/)
|
|
432
1394
|
legacyPid = legacyProcess.status().pid
|
|
433
1395
|
const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
|
|
434
1396
|
const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
|
|
@@ -469,6 +1431,190 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
|
|
|
469
1431
|
}
|
|
470
1432
|
})
|
|
471
1433
|
|
|
1434
|
+
test("split guardian rejects an owner-state update when its nested legacy definition update fails", async () => {
|
|
1435
|
+
const fixture = await createLegacyGuardian()
|
|
1436
|
+
const processDefinition = definition("legacy-worker")
|
|
1437
|
+
const legacyProcess = fixture.client.process("release:v1:legacy-worker", processDefinition)
|
|
1438
|
+
const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
|
|
1439
|
+
const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
|
|
1440
|
+
const ownerState = {
|
|
1441
|
+
authority,
|
|
1442
|
+
snapshot: {activeReleaseId: "v1", releases: [{processes: [{id: "legacy-worker"}], releaseId: "v1"}], services: [], singletons: []}
|
|
1443
|
+
}
|
|
1444
|
+
let upgraded
|
|
1445
|
+
|
|
1446
|
+
try {
|
|
1447
|
+
await legacyProcess.start()
|
|
1448
|
+
upgraded = await fixture.client.upgradeLegacyGuardian({
|
|
1449
|
+
ownerState,
|
|
1450
|
+
socketPath: path.join(fixture.root, "guardian-v2.sock"),
|
|
1451
|
+
token: "candidate-guardian-capability"
|
|
1452
|
+
})
|
|
1453
|
+
const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
|
|
1454
|
+
const committedOwnerState = {authority: nextAuthority, snapshot: ownerState.snapshot}
|
|
1455
|
+
|
|
1456
|
+
await upgraded.stageOwnerReplacement(prepared.replacementId, committedOwnerState)
|
|
1457
|
+
const restored = upgraded.process("release:v1:legacy-worker", processDefinition)
|
|
1458
|
+
|
|
1459
|
+
await restored.recover()
|
|
1460
|
+
await legacyProcess.updateDefinition({...processDefinition, env: {REVISION: "external"}})
|
|
1461
|
+
await assert.rejects(
|
|
1462
|
+
() => restored.updateDefinition({...processDefinition, env: {REVISION: "candidate"}}, {authority: nextAuthority, snapshot: {...ownerState.snapshot, serviceReleaseIds: {service: "v2"}}}),
|
|
1463
|
+
/provenance mismatch/
|
|
1464
|
+
)
|
|
1465
|
+
assert.deepEqual(await upgraded.ownerState(), committedOwnerState)
|
|
1466
|
+
} finally {
|
|
1467
|
+
await legacyProcess.stop().catch(() => {})
|
|
1468
|
+
upgraded?.disconnect()
|
|
1469
|
+
fixture.client.disconnect()
|
|
1470
|
+
if (upgraded?.pid) killExactProcessGroup(upgraded.pid)
|
|
1471
|
+
if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
|
|
1472
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1473
|
+
}
|
|
1474
|
+
})
|
|
1475
|
+
|
|
1476
|
+
test("split guardian defers owner handoff until a nested legacy definition update commits", async () => {
|
|
1477
|
+
const fixture = await createLegacyGuardian()
|
|
1478
|
+
const processDefinition = definition("legacy-worker")
|
|
1479
|
+
const legacyProcess = fixture.client.process("release:v1:legacy-worker", processDefinition)
|
|
1480
|
+
const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
|
|
1481
|
+
const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
|
|
1482
|
+
const ownerState = {
|
|
1483
|
+
authority,
|
|
1484
|
+
snapshot: {activeReleaseId: "v1", releases: [{processes: [{id: "legacy-worker"}], releaseId: "v1"}], services: [], singletons: []}
|
|
1485
|
+
}
|
|
1486
|
+
const socketPath = path.join(fixture.root, "guardian-v2.sock")
|
|
1487
|
+
const token = "candidate-guardian-capability"
|
|
1488
|
+
const gatePath = path.join(fixture.root, "legacy-update.allow")
|
|
1489
|
+
const committedOwnerState = {
|
|
1490
|
+
authority: nextAuthority,
|
|
1491
|
+
listenerConnectionSources: {"updating-owner": {v1: {http: 0, websocket: 1}}},
|
|
1492
|
+
listenerSourceId: "updating-owner",
|
|
1493
|
+
snapshot: {...ownerState.snapshot, update: "committed"}
|
|
1494
|
+
}
|
|
1495
|
+
const contender = new GuardianClient({socketPath, token})
|
|
1496
|
+
let upgraded
|
|
1497
|
+
|
|
1498
|
+
try {
|
|
1499
|
+
await legacyProcess.start()
|
|
1500
|
+
upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, socketPath, token})
|
|
1501
|
+
const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
|
|
1502
|
+
|
|
1503
|
+
await upgraded.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: ownerState.snapshot})
|
|
1504
|
+
const restored = upgraded.process("release:v1:legacy-worker", processDefinition)
|
|
1505
|
+
|
|
1506
|
+
await restored.recover()
|
|
1507
|
+
await contender.connect()
|
|
1508
|
+
const updateResult = restored.updateDefinition({...processDefinition, env: {ROLLBRIDGE_TEST_UPDATE_GATE: gatePath}}, committedOwnerState)
|
|
1509
|
+
.then(() => undefined, (error) => error)
|
|
1510
|
+
|
|
1511
|
+
await waitForFileText(`${gatePath}.waiting`)
|
|
1512
|
+
const claim = contender.claimOwner(1000, nextAuthority)
|
|
1513
|
+
let claimSettled = false
|
|
1514
|
+
|
|
1515
|
+
void claim.finally(() => { claimSettled = true })
|
|
1516
|
+
upgraded.disconnect()
|
|
1517
|
+
await new Promise((resolve) => setTimeout(resolve, 50))
|
|
1518
|
+
assert.equal(claimSettled, false, "owner handoff must wait for the nested definition update")
|
|
1519
|
+
|
|
1520
|
+
await fs.writeFile(gatePath, "allow\n")
|
|
1521
|
+
await claim
|
|
1522
|
+
const updateError = await updateResult
|
|
1523
|
+
|
|
1524
|
+
assert.match(String(updateError), /connection closed while awaiting update/)
|
|
1525
|
+
assert.deepEqual(await contender.ownerState(), {...committedOwnerState, listenerConnectionSources: {}})
|
|
1526
|
+
} finally {
|
|
1527
|
+
await fs.writeFile(gatePath, "allow\n").catch(() => {})
|
|
1528
|
+
await contender.shutdown().catch(() => {})
|
|
1529
|
+
contender.disconnect()
|
|
1530
|
+
upgraded?.disconnect()
|
|
1531
|
+
await legacyProcess.stop().catch(() => {})
|
|
1532
|
+
fixture.client.disconnect()
|
|
1533
|
+
if (upgraded?.pid) killExactProcessGroup(upgraded.pid)
|
|
1534
|
+
if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
|
|
1535
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1536
|
+
}
|
|
1537
|
+
})
|
|
1538
|
+
|
|
1539
|
+
test("a disconnected legacy upgrade candidate does not strand its bridge guardian", async () => {
|
|
1540
|
+
const fixture = await createLegacyGuardian()
|
|
1541
|
+
const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
|
|
1542
|
+
const ownerState = {
|
|
1543
|
+
authority,
|
|
1544
|
+
snapshot: {activeReleaseId: null, releases: [], services: [], singletons: []}
|
|
1545
|
+
}
|
|
1546
|
+
let upgraded
|
|
1547
|
+
let bridgeExited = false
|
|
1548
|
+
|
|
1549
|
+
try {
|
|
1550
|
+
upgraded = await fixture.client.upgradeLegacyGuardian({
|
|
1551
|
+
ownerState,
|
|
1552
|
+
socketPath: path.join(fixture.root, "guardian-v2.sock"),
|
|
1553
|
+
token: "candidate-guardian-capability"
|
|
1554
|
+
})
|
|
1555
|
+
await upgraded.prepareOwnerReplacement(authority, {...authority, runtime: {...authority.runtime, digest: "candidate-runtime"}})
|
|
1556
|
+
upgraded.disconnect()
|
|
1557
|
+
let timeout = /** @type {ReturnType<typeof setTimeout> | undefined} */ (undefined)
|
|
1558
|
+
|
|
1559
|
+
await Promise.race([
|
|
1560
|
+
upgraded.guardianExit(),
|
|
1561
|
+
new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Legacy upgrade bridge guardian did not exit after candidate disconnect")), 1000) })
|
|
1562
|
+
]).finally(() => { if (timeout) clearTimeout(timeout) })
|
|
1563
|
+
bridgeExited = true
|
|
1564
|
+
assert.equal(fixture.child.exitCode, null, "the pre-split guardian must remain available after bridge abandonment")
|
|
1565
|
+
} finally {
|
|
1566
|
+
upgraded?.disconnect()
|
|
1567
|
+
if (!bridgeExited && upgraded?.pid) killExactProcessGroup(upgraded.pid)
|
|
1568
|
+
await cleanupGuardian({client: fixture.client, root: fixture.root})
|
|
1569
|
+
}
|
|
1570
|
+
})
|
|
1571
|
+
|
|
1572
|
+
test("a legacy bridge remains discoverable when its candidate disconnects after the disruptive boundary", async () => {
|
|
1573
|
+
const fixture = await createLegacyGuardian()
|
|
1574
|
+
const statePath = path.join(fixture.root, "state.json")
|
|
1575
|
+
const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
|
|
1576
|
+
const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime"}}
|
|
1577
|
+
const ownerState = {
|
|
1578
|
+
authority,
|
|
1579
|
+
config: {statePath},
|
|
1580
|
+
snapshot: {activeReleaseId: null, control: {path: path.join(fixture.root, "control.sock")}, releases: [], services: [], singletons: []}
|
|
1581
|
+
}
|
|
1582
|
+
let upgraded
|
|
1583
|
+
let replacement
|
|
1584
|
+
|
|
1585
|
+
try {
|
|
1586
|
+
const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
|
|
1587
|
+
|
|
1588
|
+
upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
|
|
1589
|
+
const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
|
|
1590
|
+
const recoverySnapshot = {
|
|
1591
|
+
...ownerState.snapshot,
|
|
1592
|
+
recovery: {configDigest: authority.configDigest, format: 1, guardian: {...identity, pid: upgraded.pid}, reconnectGraceMs: 3000}
|
|
1593
|
+
}
|
|
1594
|
+
|
|
1595
|
+
await fs.writeFile(statePath, `${JSON.stringify(ownerState.snapshot, null, 2)}\n`)
|
|
1596
|
+
await upgraded.beginLegacyOwnerClaim(prepared.replacementId, 1000, statePath, recoverySnapshot)
|
|
1597
|
+
fixture.client.disconnect()
|
|
1598
|
+
await upgraded.completeLegacyOwnerClaim(prepared.replacementId)
|
|
1599
|
+
assert.deepEqual(JSON.parse(await fs.readFile(statePath, "utf8")), recoverySnapshot)
|
|
1600
|
+
upgraded.disconnect()
|
|
1601
|
+
|
|
1602
|
+
replacement = new GuardianClient({...identity, pid: upgraded.pid})
|
|
1603
|
+
await replacement.connect()
|
|
1604
|
+
const resumed = await replacement.prepareOwnerReplacement(authority, nextAuthority)
|
|
1605
|
+
const committed = replacement.waitForEvent("replacement-committed")
|
|
1606
|
+
|
|
1607
|
+
assert.deepEqual(await replacement.stageOwnerReplacement(resumed.replacementId, {authority: nextAuthority, config: {statePath}, snapshot: ownerState.snapshot}), {committed: true})
|
|
1608
|
+
await committed
|
|
1609
|
+
await replacement.shutdown()
|
|
1610
|
+
await upgraded.guardianExit()
|
|
1611
|
+
} finally {
|
|
1612
|
+
upgraded?.disconnect()
|
|
1613
|
+
replacement?.disconnect()
|
|
1614
|
+
await cleanupGuardian({client: fixture.client, root: fixture.root})
|
|
1615
|
+
}
|
|
1616
|
+
})
|
|
1617
|
+
|
|
472
1618
|
/** @returns {Promise<{client: GuardianClient, root: string, token: string}>} Started guardian fixture. */
|
|
473
1619
|
async function createGuardian() {
|
|
474
1620
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-guardian-client-"))
|
|
@@ -524,9 +1670,139 @@ async function cleanupGuardian(fixture) {
|
|
|
524
1670
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
525
1671
|
}
|
|
526
1672
|
|
|
1673
|
+
/**
|
|
1674
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} authority - Recovery authority.
|
|
1675
|
+
* @param {string} markerPath - Restart marker path.
|
|
1676
|
+
* @param {string} marker - Expected accepted command marker.
|
|
1677
|
+
* @param {number} reconnectGraceMs - Initial reconnect grace.
|
|
1678
|
+
* @returns {Record<string, import("../src/json.js").JsonValue>} Guardian owner state.
|
|
1679
|
+
*/
|
|
1680
|
+
function recoveryOwnerState(authority, markerPath, marker, reconnectGraceMs) {
|
|
1681
|
+
return {
|
|
1682
|
+
authority,
|
|
1683
|
+
recovery: {
|
|
1684
|
+
command: {
|
|
1685
|
+
args: ["-e", "require('node:fs').appendFileSync(process.argv[1], JSON.stringify({at: Date.now(), home: process.env.HOME ?? null, marker: process.env.RESTART_MARKER}) + '\\n')", markerPath],
|
|
1686
|
+
cwd: path.dirname(markerPath),
|
|
1687
|
+
env: {RESTART_MARKER: marker},
|
|
1688
|
+
executable: process.execPath
|
|
1689
|
+
},
|
|
1690
|
+
reconnectGraceMs,
|
|
1691
|
+
startupTimeoutMs: 1000
|
|
1692
|
+
},
|
|
1693
|
+
snapshot: {activeReleaseId: null}
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1697
|
+
/**
|
|
1698
|
+
* @param {{client: GuardianClient, root: string, token: string}} fixture - Guardian fixture.
|
|
1699
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} authority - Recovery authority.
|
|
1700
|
+
* @param {string} markerPath - Claim marker path.
|
|
1701
|
+
* @param {{claimDelayMs?: number, descendantPath?: string, exitAfterClaim?: boolean, ready?: boolean, replacementCommittedPath?: string, replacementPreparedPath?: string, startedLogPath?: string, startedPath?: string, startupTimeoutMs?: number}} [options] - Optional recovery behavior.
|
|
1702
|
+
* @returns {Record<string, import("../src/json.js").JsonValue>} Guardian owner state.
|
|
1703
|
+
*/
|
|
1704
|
+
function claimingRecoveryOwnerState(fixture, authority, markerPath, options = {}) {
|
|
1705
|
+
return {
|
|
1706
|
+
authority,
|
|
1707
|
+
recovery: {
|
|
1708
|
+
command: {
|
|
1709
|
+
args: [recoveryOwnerPath],
|
|
1710
|
+
cwd: fixture.root,
|
|
1711
|
+
env: {
|
|
1712
|
+
GUARDIAN_AUTHORITY: JSON.stringify(authority),
|
|
1713
|
+
GUARDIAN_CLAIM_DELAY_MS: String(options.claimDelayMs || 0),
|
|
1714
|
+
...(options.descendantPath ? {GUARDIAN_DESCENDANT_PATH: options.descendantPath} : {}),
|
|
1715
|
+
...(options.exitAfterClaim ? {GUARDIAN_EXIT_AFTER_CLAIM: "1"} : {}),
|
|
1716
|
+
GUARDIAN_MARKER_PATH: markerPath,
|
|
1717
|
+
...(options.replacementCommittedPath ? {GUARDIAN_REPLACEMENT_COMMITTED_PATH: options.replacementCommittedPath} : {}),
|
|
1718
|
+
...(options.replacementPreparedPath ? {GUARDIAN_REPLACEMENT_PREPARED_PATH: options.replacementPreparedPath} : {}),
|
|
1719
|
+
...(options.ready === false ? {GUARDIAN_SKIP_READY: "1"} : {}),
|
|
1720
|
+
GUARDIAN_SOCKET_PATH: fixture.client.socketPath,
|
|
1721
|
+
...(options.startedLogPath ? {GUARDIAN_STARTED_LOG_PATH: options.startedLogPath} : {}),
|
|
1722
|
+
...(options.startedPath ? {GUARDIAN_STARTED_PATH: options.startedPath} : {}),
|
|
1723
|
+
GUARDIAN_TOKEN: fixture.token
|
|
1724
|
+
},
|
|
1725
|
+
executable: process.execPath
|
|
1726
|
+
},
|
|
1727
|
+
reconnectGraceMs: 10,
|
|
1728
|
+
startupTimeoutMs: options.startupTimeoutMs || 1000
|
|
1729
|
+
},
|
|
1730
|
+
snapshot: {activeReleaseId: null}
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1734
|
+
/**
|
|
1735
|
+
* @param {string} markerPath - Restart marker path.
|
|
1736
|
+
* @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Restart records.
|
|
1737
|
+
*/
|
|
1738
|
+
async function restartRecords(markerPath) {
|
|
1739
|
+
try {
|
|
1740
|
+
return (await fs.readFile(markerPath, "utf8")).trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))
|
|
1741
|
+
} catch (error) {
|
|
1742
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return []
|
|
1743
|
+
throw error
|
|
1744
|
+
}
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
/**
|
|
1748
|
+
* @param {string} markerPath - Restart marker path.
|
|
1749
|
+
* @param {number} count - Required record count.
|
|
1750
|
+
* @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Restart records.
|
|
1751
|
+
*/
|
|
1752
|
+
async function waitForRestartRecords(markerPath, count) {
|
|
1753
|
+
const deadline = Date.now() + 3000
|
|
1754
|
+
|
|
1755
|
+
while (Date.now() < deadline) {
|
|
1756
|
+
const records = await restartRecords(markerPath)
|
|
1757
|
+
|
|
1758
|
+
if (records.length >= count) return records
|
|
1759
|
+
await new Promise((resolve) => setTimeout(resolve, 10))
|
|
1760
|
+
}
|
|
1761
|
+
throw new Error(`Timed out waiting for ${count} guardian restart record${count === 1 ? "" : "s"}`)
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1764
|
+
/**
|
|
1765
|
+
* @param {string} filePath - File to read after creation.
|
|
1766
|
+
* @param {RegExp} [expected] - Content that must be present before returning.
|
|
1767
|
+
* @returns {Promise<string>} Non-empty accepted file contents.
|
|
1768
|
+
*/
|
|
1769
|
+
async function waitForFileText(filePath, expected) {
|
|
1770
|
+
const deadline = Date.now() + 3000
|
|
1771
|
+
|
|
1772
|
+
while (Date.now() < deadline) {
|
|
1773
|
+
try {
|
|
1774
|
+
const contents = await fs.readFile(filePath, "utf8")
|
|
1775
|
+
|
|
1776
|
+
if (contents && (!expected || expected.test(contents))) return contents
|
|
1777
|
+
} catch (error) {
|
|
1778
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
1779
|
+
}
|
|
1780
|
+
await new Promise((resolve) => setTimeout(resolve, 10))
|
|
1781
|
+
}
|
|
1782
|
+
throw new Error(`Timed out waiting for ${filePath}`)
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
/**
|
|
1786
|
+
* @param {{client: GuardianClient, root: string, token: string}} fixture - Guardian fixture.
|
|
1787
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} authority - Committed authority.
|
|
1788
|
+
*/
|
|
1789
|
+
async function reconnectAndShutdownGuardian(fixture, authority) {
|
|
1790
|
+
const cleanup = new GuardianClient({pid: fixture.client.pid, socketPath: fixture.client.socketPath, token: fixture.token})
|
|
1791
|
+
|
|
1792
|
+
try {
|
|
1793
|
+
await cleanup.connect()
|
|
1794
|
+
await cleanup.claimOwner(500, authority)
|
|
1795
|
+
await cleanup.shutdown()
|
|
1796
|
+
} finally {
|
|
1797
|
+
cleanup.disconnect()
|
|
1798
|
+
await fixture.client.guardianExit().catch(() => {})
|
|
1799
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
|
|
527
1803
|
/**
|
|
528
1804
|
* @param {string} id - Process id.
|
|
529
|
-
* @returns {Parameters<GuardianClient["process"]>[1]} Managed process definition.
|
|
1805
|
+
* @returns {Parameters<GuardianClient["process"]>[1] & import("../src/managed-process.js").ManagedProcessDefinition} Managed process definition.
|
|
530
1806
|
*/
|
|
531
1807
|
function definition(id) {
|
|
532
1808
|
return {
|
|
@@ -536,6 +1812,7 @@ function definition(id) {
|
|
|
536
1812
|
id,
|
|
537
1813
|
lifecycle: {drainTimeoutMs: 0},
|
|
538
1814
|
logger: () => {},
|
|
1815
|
+
memory: undefined,
|
|
539
1816
|
outputLines: 10,
|
|
540
1817
|
restart: {backoffFactor: 1, maxDelayMs: 0, maxRestarts: 0, windowMs: 0},
|
|
541
1818
|
restartDelayMs: 0,
|