rollbridge 0.1.39 → 0.1.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,13 +9,16 @@ import path from "node:path"
9
9
  import test from "node:test"
10
10
  import {fileURLToPath} from "node:url"
11
11
  import GuardianClient from "../src/guardian-client.js"
12
+ import {waitForProcessExit} from "./support/process.js"
12
13
 
13
14
  const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
15
+ const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
14
16
 
15
17
  test("guardian bootstrap capability is absent from process argv", async () => {
16
18
  const fixture = await createGuardian()
17
19
 
18
20
  try {
21
+ assert.deepEqual(await fixture.client.capabilities(), {daemonRecovery: 1, generationReactivation: 1})
19
22
  const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
20
23
  const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
21
24
  const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
@@ -67,6 +70,106 @@ test("guardian runs a strict activation lifecycle command for the exact register
67
70
  }
68
71
  })
69
72
 
73
+ test("client reactivates a retained process through a guardian without the reactivation command", async () => {
74
+ const fixture = await createGuardian()
75
+ const lifecyclePath = path.join(fixture.root, "lifecycle.log")
76
+ const processInstance = fixture.client.process("compatible-reactivation", {
77
+ ...definition("compatible-reactivation"),
78
+ lifecycle: {
79
+ activateCommand: `printf 'activate\n' >> ${JSON.stringify(lifecyclePath)}`,
80
+ drainTimeoutMs: 0,
81
+ quietCommand: `printf 'retire\n' >> ${JSON.stringify(lifecyclePath)}`
82
+ },
83
+ shouldRestart: () => true
84
+ })
85
+ const request = fixture.client.request.bind(fixture.client)
86
+
87
+ fixture.client.request = async command => {
88
+ if (command.command === "reactivate") throw new Error("Unknown guardian command: reactivate")
89
+ return await request(command)
90
+ }
91
+
92
+ try {
93
+ await processInstance.start()
94
+ await processInstance.activateStrict()
95
+ const pid = processInstance.status().pid
96
+
97
+ await processInstance.quiesceStrict()
98
+ await processInstance.reactivateStrict()
99
+
100
+ assert.equal(processInstance.status().pid, pid)
101
+ assert.equal(processInstance.status().state, "running")
102
+ assert.equal(processInstance.status().lifecycleRole, "active")
103
+ assert.equal(await fs.readFile(lifecyclePath, "utf8"), "activate\nretire\nactivate\n")
104
+
105
+ const restarted = once(processInstance, "started")
106
+
107
+ assert.ok(pid)
108
+ process.kill(-pid, "SIGKILL")
109
+ await restarted
110
+ assert.notEqual(processInstance.status().pid, pid)
111
+ assert.equal(processInstance.status().state, "running")
112
+ assert.equal(processInstance.status().lifecycleRole, "active")
113
+ assert.equal(await fs.readFile(lifecyclePath, "utf8"), "activate\nretire\nactivate\nactivate\n")
114
+ } finally {
115
+ await cleanupGuardian(fixture)
116
+ }
117
+ })
118
+
119
+ test("client reverses a worker quiet hook through a pre-reactivation guardian", async () => {
120
+ const fixture = await createGuardian()
121
+ const lifecyclePath = path.join(fixture.root, "worker-lifecycle.log")
122
+ const processInstance = fixture.client.process("compatible-worker-reactivation", {
123
+ ...definition("compatible-worker-reactivation"),
124
+ lifecycle: {
125
+ drainTimeoutMs: 0,
126
+ quietCommand: `printf 'quiet\n' >> ${JSON.stringify(lifecyclePath)}`,
127
+ reactivateCommand: `printf 'resume\n' >> ${JSON.stringify(lifecyclePath)}`
128
+ },
129
+ shouldRestart: () => true
130
+ })
131
+ const request = fixture.client.request.bind(fixture.client)
132
+
133
+ fixture.client.request = async command => {
134
+ if (command.command === "reactivate-with-command") throw new Error("Unknown guardian command: reactivate-with-command")
135
+ return await request(command)
136
+ }
137
+
138
+ try {
139
+ await processInstance.start()
140
+ const pid = processInstance.status().pid
141
+
142
+ await processInstance.quiesceStrict()
143
+ await processInstance.reactivateStrict()
144
+
145
+ assert.equal(processInstance.status().pid, pid)
146
+ assert.equal(processInstance.status().state, "running")
147
+ assert.equal(processInstance.status().lifecycleRole, "active")
148
+ assert.equal(await fs.readFile(lifecyclePath, "utf8"), "quiet\nresume\n")
149
+ } finally {
150
+ await cleanupGuardian(fixture)
151
+ }
152
+ })
153
+
154
+ test("guardian atomically updates process provenance with private owner state", async () => {
155
+ const fixture = await createGuardian()
156
+ const processInstance = fixture.client.process("service", definition("service"))
157
+ const previousOwnerState = {authority: null, serviceReleaseIds: {service: "v1"}}
158
+ const nextOwnerState = {authority: null, serviceReleaseIds: {service: "v2"}}
159
+
160
+ try {
161
+ await fixture.client.publishOwnerState(previousOwnerState)
162
+ await processInstance.start()
163
+ const previousProvenance = (await fixture.client.inventory())[0]?.provenance
164
+
165
+ await processInstance.updateDefinition({...definition("service"), env: {RELEASE: "v2"}}, nextOwnerState)
166
+ assert.deepEqual(await fixture.client.ownerState(), nextOwnerState)
167
+ assert.notEqual((await fixture.client.inventory())[0]?.provenance, previousProvenance)
168
+ } finally {
169
+ await cleanupGuardian(fixture)
170
+ }
171
+ })
172
+
70
173
  test("guardian forwards each retained output line to its exact process proxy", async () => {
71
174
  const fixture = await createGuardian()
72
175
  const marker = "guardian-output-ready"
@@ -75,19 +178,102 @@ test("guardian forwards each retained output line to its exact process proxy", a
75
178
  command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`console.log(${JSON.stringify(marker)})`)}`
76
179
  })
77
180
  const logged = once(processInstance, "log")
181
+ const forwarded = fixture.client.waitForEvent("process-log")
78
182
  const exitedFirst = once(processInstance, "exit").then(() => { throw new Error("Guardian process exited before forwarding retained output") })
79
183
 
80
184
  try {
81
185
  await processInstance.start()
82
186
  const [entry] = await Promise.race([logged, exitedFirst])
187
+ const event = await forwarded
83
188
 
84
189
  assert.equal(entry.line, marker)
190
+ assert.equal(event.status, undefined, "log events must not resend the complete retained process status")
85
191
  assert.ok(processInstance.status().logs.some((candidate) => candidate.line === marker))
86
192
  } finally {
87
193
  await cleanupGuardian(fixture)
88
194
  }
89
195
  })
90
196
 
197
+ test("guardian delivers the final process status after dropping logs for a backpressured client", async () => {
198
+ const fixture = await createGuardian()
199
+ const gatePath = path.join(fixture.root, "write-output")
200
+ const script = `const fs = require("node:fs"); const {once} = require("node:events"); (async () => { while (!fs.existsSync(${JSON.stringify(gatePath)})) await new Promise((resolve) => setTimeout(resolve, 5)); const line = "x".repeat(1024) + "\\n"; for (let index = 0; index < 8192; index += 1) if (!process.stdout.write(line)) await once(process.stdout, "drain"); })()`
201
+ const processInstance = fixture.client.process("backpressured-output", {
202
+ ...definition("backpressured-output"),
203
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(script)}`
204
+ })
205
+ const socket = fixture.client.socket
206
+
207
+ assert.ok(socket)
208
+ try {
209
+ await processInstance.start()
210
+ const pid = processInstance.status().pid
211
+
212
+ assert.ok(pid)
213
+ const finalStatus = fixture.client.waitForEvent("process")
214
+
215
+ socket.pause()
216
+ await fs.writeFile(gatePath, "write\n")
217
+ await waitForProcessExit(pid, 10000)
218
+ socket.resume()
219
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
220
+ let timeout
221
+
222
+ try {
223
+ await Promise.race([
224
+ finalStatus,
225
+ new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Guardian did not flush the final process status after backpressure")), 3000) })
226
+ ])
227
+ } finally {
228
+ clearTimeout(timeout)
229
+ }
230
+ assert.equal(processInstance.status().state, "failed")
231
+ } finally {
232
+ socket.resume()
233
+ await cleanupGuardian(fixture)
234
+ }
235
+ })
236
+
237
+ test("guardian resynchronizes retained logs after dropping output for a backpressured client", async () => {
238
+ const fixture = await createGuardian()
239
+ const gatePath = path.join(fixture.root, "write-retained-output")
240
+ const completedPath = path.join(fixture.root, "retained-output-complete")
241
+ const payload = "x".repeat(1024)
242
+ const finalLine = `8191:${payload}`
243
+ const script = `const fs = require("node:fs"); const {once} = require("node:events"); (async () => { while (!fs.existsSync(${JSON.stringify(gatePath)})) await new Promise((resolve) => setTimeout(resolve, 5)); for (let index = 0; index < 8192; index += 1) if (!process.stdout.write(index + ":" + ${JSON.stringify(payload)} + "\\n")) await once(process.stdout, "drain"); fs.writeFileSync(${JSON.stringify(completedPath)}, "complete\\n"); setInterval(() => {}, 1000); })()`
244
+ const processInstance = fixture.client.process("backpressured-retained-output", {
245
+ ...definition("backpressured-retained-output"),
246
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(script)}`
247
+ })
248
+ const socket = fixture.client.socket
249
+
250
+ assert.ok(socket)
251
+ try {
252
+ await processInstance.start()
253
+ socket.pause()
254
+ await fs.writeFile(gatePath, "write\n")
255
+ await waitForFileText(completedPath)
256
+ const resynchronized = fixture.client.waitForEvent("status")
257
+
258
+ socket.resume()
259
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
260
+ let timeout
261
+
262
+ try {
263
+ await Promise.race([
264
+ resynchronized,
265
+ new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Guardian did not resynchronize retained output after backpressure")), 3000) })
266
+ ])
267
+ } finally {
268
+ clearTimeout(timeout)
269
+ }
270
+ assert.equal(processInstance.status().logs.at(-1)?.line, finalLine)
271
+ } finally {
272
+ socket.resume()
273
+ await cleanupGuardian(fixture)
274
+ }
275
+ })
276
+
91
277
  test("guardian shutdown reports an exact owned process stop failure", async () => {
92
278
  const fixture = await createGuardian()
93
279
  const processInstance = fixture.client.process("broken-stop", {...definition("broken-stop"), stopSignal: "NOT_A_SIGNAL"})
@@ -153,6 +339,403 @@ test("successful guardian shutdown closes an authenticated waiting contender bef
153
339
  }
154
340
  })
155
341
 
342
+ test("guardian restart uses the latest accepted command and exact environment", async () => {
343
+ const fixture = await createGuardian()
344
+ const markerPath = path.join(fixture.root, "restarts.jsonl")
345
+ const authority = {configDigest: "same", runtime: null}
346
+ const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
347
+
348
+ try {
349
+ await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "old", 250))
350
+ fixture.client.disconnect()
351
+ await replacement.connect()
352
+ await replacement.claimOwner(250, authority)
353
+ await replacement.publishOwnerState(recoveryOwnerState(authority, markerPath, "new", 40))
354
+ replacement.disconnect()
355
+
356
+ const [restart] = await waitForRestartRecords(markerPath, 1)
357
+
358
+ assert.deepEqual({home: restart.home, marker: restart.marker}, {home: null, marker: "new"})
359
+ } finally {
360
+ await reconnectAndShutdownGuardian(fixture, authority)
361
+ }
362
+ })
363
+
364
+ test("guardian rearms recovery after an ownerless replacement aborts", async () => {
365
+ const fixture = await createGuardian()
366
+ const markerPath = path.join(fixture.root, "replacement-abort.jsonl")
367
+ const authority = {configDigest: "same", runtime: null}
368
+ const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
369
+
370
+ try {
371
+ await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "accepted", 40))
372
+ fixture.client.disconnect()
373
+ await replacement.connect()
374
+ const prepared = await replacement.prepareOwnerReplacement(authority, authority)
375
+
376
+ await new Promise((resolve) => setTimeout(resolve, 80))
377
+ await replacement.abortOwnerReplacement(prepared.replacementId)
378
+ assert.deepEqual((await waitForRestartRecords(markerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "accepted"}])
379
+ } finally {
380
+ replacement.disconnect()
381
+ await reconnectAndShutdownGuardian(fixture, authority)
382
+ }
383
+ })
384
+
385
+ test("guardian retry backoff remains nonzero when reconnect grace is zero", async () => {
386
+ const fixture = await createGuardian()
387
+ const markerPath = path.join(fixture.root, "retry-backoff.jsonl")
388
+ const authority = {configDigest: "same", runtime: null}
389
+
390
+ try {
391
+ await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "retry", 0))
392
+ fixture.client.disconnect()
393
+ const records = await waitForRestartRecords(markerPath, 2)
394
+
395
+ assert.equal(typeof records[0]?.at, "number")
396
+ assert.equal(typeof records[1]?.at, "number")
397
+ assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `failed owner recovery retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
398
+ } finally {
399
+ await reconnectAndShutdownGuardian(fixture, authority)
400
+ }
401
+ })
402
+
403
+ test("a retired guardian-started owner does not block recovery of its replacement", async () => {
404
+ const fixture = await createGuardian()
405
+ const firstMarkerPath = path.join(fixture.root, "first-owner.pid")
406
+ const secondMarkerPath = path.join(fixture.root, "second-owner.jsonl")
407
+ const authority = {configDigest: "same", runtime: null}
408
+ const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
409
+ let firstOwnerPid
410
+
411
+ try {
412
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, firstMarkerPath))
413
+ fixture.client.disconnect()
414
+ firstOwnerPid = Number((await waitForFileText(firstMarkerPath)).trim())
415
+ await replacement.connect()
416
+ const prepared = await replacement.prepareOwnerReplacement(authority, authority)
417
+ const committed = replacement.waitForEvent("replacement-committed")
418
+
419
+ await replacement.stageOwnerReplacement(prepared.replacementId, recoveryOwnerState(authority, secondMarkerPath, "replacement", 20))
420
+ process.kill(firstOwnerPid, "SIGUSR1")
421
+ await committed
422
+ replacement.disconnect()
423
+
424
+ assert.deepEqual((await waitForRestartRecords(secondMarkerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "replacement"}])
425
+ } finally {
426
+ replacement.disconnect()
427
+ if (firstOwnerPid) {
428
+ try { process.kill(firstOwnerPid, "SIGKILL") } catch (_error) { /* Exact retired fixture owner already exited. */ }
429
+ }
430
+ await reconnectAndShutdownGuardian(fixture, authority)
431
+ }
432
+ })
433
+
434
+ test("guardian terminates a restart attempt that never claims ownership", async () => {
435
+ const fixture = await createGuardian()
436
+ const markerPath = path.join(fixture.root, "hung-owner.pid")
437
+ const authority = {configDigest: "same", runtime: null}
438
+ let hungPid
439
+
440
+ try {
441
+ await fixture.client.publishOwnerState({
442
+ authority,
443
+ recovery: {
444
+ command: {
445
+ args: ["-e", "require('node:fs').writeFileSync(process.argv[1], String(process.pid)); setInterval(() => {}, 1000)", markerPath],
446
+ cwd: fixture.root,
447
+ env: {},
448
+ executable: process.execPath
449
+ },
450
+ reconnectGraceMs: 0,
451
+ startupTimeoutMs: 1000
452
+ },
453
+ snapshot: {activeReleaseId: null}
454
+ })
455
+ fixture.client.disconnect()
456
+ hungPid = Number((await waitForFileText(markerPath)).trim())
457
+
458
+ await waitForProcessExit(hungPid)
459
+ } finally {
460
+ if (hungPid) {
461
+ try { process.kill(hungPid, "SIGKILL") } catch (_error) { /* Exact hung fixture owner already exited. */ }
462
+ }
463
+ await reconnectAndShutdownGuardian(fixture, authority)
464
+ }
465
+ })
466
+
467
+ test("guardian terminates descendants when a restart leader exits before claiming ownership", async () => {
468
+ const fixture = await createGuardian()
469
+ const descendantPath = path.join(fixture.root, "early-exit-descendant.pid")
470
+ const authority = {configDigest: "same", runtime: null}
471
+ let descendantPid
472
+
473
+ try {
474
+ await fixture.client.publishOwnerState({
475
+ authority,
476
+ recovery: {
477
+ command: {
478
+ args: ["-e", `const {spawn} = require("node:child_process"); const fs = require("node:fs"); const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {stdio: "ignore"}); fs.writeFileSync(${JSON.stringify(descendantPath)}, String(child.pid)); child.unref()`],
479
+ cwd: fixture.root,
480
+ env: {},
481
+ executable: process.execPath
482
+ },
483
+ reconnectGraceMs: 0,
484
+ startupTimeoutMs: 1000
485
+ },
486
+ snapshot: {activeReleaseId: null}
487
+ })
488
+ fixture.client.disconnect()
489
+ descendantPid = Number((await waitForFileText(descendantPath)).trim())
490
+
491
+ await waitForProcessExit(descendantPid)
492
+ } finally {
493
+ if (descendantPid) {
494
+ try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
495
+ }
496
+ await reconnectAndShutdownGuardian(fixture, authority)
497
+ }
498
+ })
499
+
500
+ test("guardian terminates a restarted owner process group that claims but never becomes ready", async () => {
501
+ const fixture = await createGuardian()
502
+ const descendantPath = path.join(fixture.root, "claimed-hung-descendant.pid")
503
+ const markerPath = path.join(fixture.root, "claimed-hung-owner.pid")
504
+ const authority = {configDigest: "same", runtime: null}
505
+ let descendantPid
506
+ let ownerPid
507
+
508
+ try {
509
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
510
+ descendantPath,
511
+ ready: false,
512
+ startupTimeoutMs: 100
513
+ }))
514
+ fixture.client.disconnect()
515
+ ownerPid = Number((await waitForFileText(markerPath)).trim())
516
+ descendantPid = Number((await waitForFileText(descendantPath)).trim())
517
+
518
+ await Promise.all([waitForProcessExit(ownerPid), waitForProcessExit(descendantPid)])
519
+ } finally {
520
+ if (ownerPid) killExactProcessGroup(ownerPid)
521
+ if (descendantPid) {
522
+ try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
523
+ }
524
+ await reconnectAndShutdownGuardian(fixture, authority)
525
+ }
526
+ })
527
+
528
+ test("guardian backs off when a restarted owner exits after claiming but before readiness", async () => {
529
+ const fixture = await createGuardian()
530
+ const descendantPath = path.join(fixture.root, "post-claim-descendant.pid")
531
+ const markerPath = path.join(fixture.root, "post-claim-exit.pid")
532
+ const startedLogPath = path.join(fixture.root, "post-claim-starts.jsonl")
533
+ const authority = {configDigest: "same", runtime: null}
534
+ let descendantPid
535
+
536
+ try {
537
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
538
+ descendantPath,
539
+ exitAfterClaim: true,
540
+ startedLogPath
541
+ }))
542
+ fixture.client.disconnect()
543
+ descendantPid = Number((await waitForFileText(descendantPath)).trim())
544
+ const records = await waitForRestartRecords(startedLogPath, 2)
545
+
546
+ await waitForProcessExit(descendantPid)
547
+ assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `post-claim failure retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
548
+ } finally {
549
+ if (descendantPid) {
550
+ try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
551
+ }
552
+ await reconnectAndShutdownGuardian(fixture, authority)
553
+ }
554
+ })
555
+
556
+ test("guardian preserves restart backoff when an unready owner disconnect aborts a prepared replacement", async () => {
557
+ const fixture = await createGuardian()
558
+ const markerPath = path.join(fixture.root, "prepared-post-claim-exit.pid")
559
+ const startedLogPath = path.join(fixture.root, "prepared-post-claim-starts.jsonl")
560
+ const authority = {configDigest: "same", runtime: null}
561
+ const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
562
+
563
+ try {
564
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
565
+ ready: false,
566
+ startedLogPath,
567
+ startupTimeoutMs: 150
568
+ }))
569
+ fixture.client.disconnect()
570
+ await waitForFileText(markerPath)
571
+ await replacement.connect()
572
+ const aborted = replacement.waitForEvent("replacement-aborted")
573
+
574
+ await replacement.prepareOwnerReplacement(authority, authority)
575
+ await aborted
576
+ const records = await waitForRestartRecords(startedLogPath, 2)
577
+
578
+ assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `replacement abort retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
579
+ } finally {
580
+ replacement.disconnect()
581
+ await reconnectAndShutdownGuardian(fixture, authority)
582
+ }
583
+ })
584
+
585
+ test("replacement commit preserves a claimed guardian restart child through listener retirement", async () => {
586
+ const fixture = await createGuardian()
587
+ const claimedPath = path.join(fixture.root, "claimed-restart-owner.pid")
588
+ const committedPath = path.join(fixture.root, "claimed-restart-owner-committed.txt")
589
+ const preparedPath = path.join(fixture.root, "claimed-restart-owner-prepared.txt")
590
+ const authority = {configDigest: "old", runtime: null}
591
+ const nextAuthority = {configDigest: "new", runtime: null}
592
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
593
+ /** @type {number | undefined} */
594
+ let recoveredOwnerPid
595
+
596
+ try {
597
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, claimedPath, {
598
+ ready: false,
599
+ replacementCommittedPath: committedPath,
600
+ replacementPreparedPath: preparedPath,
601
+ startupTimeoutMs: 5000
602
+ }))
603
+ fixture.client.disconnect()
604
+ const incumbentPid = Number((await waitForFileText(claimedPath)).trim())
605
+
606
+ recoveredOwnerPid = incumbentPid
607
+ await candidate.connect()
608
+ const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
609
+
610
+ await waitForFileText(preparedPath, new RegExp(prepared.replacementId))
611
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: null}})
612
+ const published = candidate.waitForEvent("replacement-committed")
613
+
614
+ process.kill(incumbentPid, "SIGUSR2")
615
+ await Promise.race([waitForFileText(committedPath, new RegExp(prepared.replacementId)), published])
616
+ await new Promise((resolve) => setTimeout(resolve, 50))
617
+ assert.doesNotThrow(() => process.kill(incumbentPid, 0), "claimed incumbent must survive replacement commit until its listeners retire")
618
+
619
+ process.kill(incumbentPid, "SIGUSR1")
620
+ await published
621
+ await candidate.shutdown()
622
+ await fixture.client.guardianExit()
623
+ } finally {
624
+ candidate.disconnect()
625
+ if (recoveredOwnerPid) {
626
+ try { process.kill(-recoveredOwnerPid, "SIGKILL") } catch (_error) { /* Exact recovered fixture owner already exited. */ }
627
+ }
628
+ await cleanupGuardian(fixture)
629
+ }
630
+ })
631
+
632
+ test("ownerless replacement commit kills a superseded restart candidate", async () => {
633
+ const fixture = await createGuardian()
634
+ const delayedClaimPath = path.join(fixture.root, "delayed-claim.pid")
635
+ const firstMarkerPath = path.join(fixture.root, "delayed-owner-started.pid")
636
+ const secondMarkerPath = path.join(fixture.root, "committed-replacement.jsonl")
637
+ const authority = {configDigest: "old", runtime: null}
638
+ const nextAuthority = {configDigest: "new", runtime: null}
639
+ const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
640
+ let delayedOwnerPid
641
+ let staged = false
642
+
643
+ try {
644
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, delayedClaimPath, {claimDelayMs: 500, startedPath: firstMarkerPath}))
645
+ fixture.client.disconnect()
646
+ delayedOwnerPid = Number((await waitForFileText(firstMarkerPath)).trim())
647
+ await replacement.connect()
648
+ const prepared = await replacement.prepareOwnerReplacement(authority, nextAuthority)
649
+
650
+ await waitForProcessExit(delayedOwnerPid)
651
+ const committed = replacement.waitForEvent("replacement-committed")
652
+
653
+ await replacement.stageOwnerReplacement(prepared.replacementId, recoveryOwnerState(nextAuthority, secondMarkerPath, "committed", 20))
654
+ staged = true
655
+ await committed
656
+ replacement.disconnect()
657
+
658
+ assert.deepEqual((await waitForRestartRecords(secondMarkerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "committed"}])
659
+ } finally {
660
+ replacement.disconnect()
661
+ if (delayedOwnerPid) {
662
+ try { process.kill(delayedOwnerPid, "SIGKILL") } catch (_error) { /* Exact superseded fixture owner already exited. */ }
663
+ }
664
+ await reconnectAndShutdownGuardian(fixture, staged ? nextAuthority : authority)
665
+ }
666
+ })
667
+
668
+ test("guardian logs an asynchronous daemon spawn failure before retrying", async () => {
669
+ const fixture = await createGuardian()
670
+ const logPath = path.join(fixture.root, "daemon.log")
671
+ const authority = {configDigest: "same", runtime: null}
672
+ const privateArgs = "private-argument-value"
673
+ const privateEnvironment = "private-environment-value"
674
+ const privateExecutable = path.join(fixture.root, "private-runtime", "missing-rollbridge")
675
+
676
+ try {
677
+ await fixture.client.publishOwnerState({
678
+ authority,
679
+ recovery: {
680
+ command: {
681
+ args: [privateArgs],
682
+ cwd: fixture.root,
683
+ env: {PRIVATE_ENVIRONMENT: privateEnvironment},
684
+ executable: privateExecutable,
685
+ logPath
686
+ },
687
+ reconnectGraceMs: 0,
688
+ startupTimeoutMs: 1000
689
+ },
690
+ snapshot: {activeReleaseId: null}
691
+ })
692
+ fixture.client.disconnect()
693
+ const diagnosticPattern = /"code":"ENOENT".*"message":"guardian failed to restart daemon"/
694
+ const diagnostic = await waitForFileText(logPath, diagnosticPattern)
695
+
696
+ assert.match(diagnostic, diagnosticPattern)
697
+ for (const privateValue of [privateArgs, privateEnvironment, privateExecutable, fixture.root, logPath]) {
698
+ assert.ok(!diagnostic.includes(privateValue), `guardian diagnostic exposed ${privateValue}`)
699
+ }
700
+ } finally {
701
+ await reconnectAndShutdownGuardian(fixture, authority)
702
+ }
703
+ })
704
+
705
+ test("guardian publishes the authenticated ready owner's PID file", async () => {
706
+ const fixture = await createGuardian()
707
+ const pidPath = path.join(fixture.root, "run", "daemon.pid")
708
+ const victimPath = path.join(fixture.root, "victim")
709
+ const authority = {configDigest: "same", runtime: null}
710
+
711
+ try {
712
+ await fixture.client.publishOwnerState({
713
+ authority,
714
+ recovery: {
715
+ command: {
716
+ args: [],
717
+ cwd: fixture.root,
718
+ env: {},
719
+ executable: process.execPath,
720
+ pidPath
721
+ },
722
+ reconnectGraceMs: 10,
723
+ startupTimeoutMs: 1000
724
+ },
725
+ snapshot: {activeReleaseId: null}
726
+ })
727
+ await fs.mkdir(path.dirname(pidPath), {recursive: true})
728
+ await fs.writeFile(victimPath, "unchanged\n")
729
+ await fs.symlink(victimPath, pidPath)
730
+ await fixture.client.ownerReady()
731
+
732
+ assert.equal(await fs.readFile(pidPath, "utf8"), `${process.pid}\n`)
733
+ assert.equal(await fs.readFile(victimPath, "utf8"), "unchanged\n")
734
+ } finally {
735
+ await cleanupGuardian(fixture)
736
+ }
737
+ })
738
+
156
739
  test("replacement commit notification waits for incumbent listener retirement", async () => {
157
740
  const fixture = await createGuardian()
158
741
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
@@ -170,6 +753,10 @@ test("replacement commit notification waits for incumbent listener retirement",
170
753
  await fixture.client.commitOwnerReplacement(prepared.replacementId)
171
754
  await new Promise((resolve) => setImmediate(resolve))
172
755
  assert.equal(committed, false, "candidate publication must remain fenced while incumbent listener retirement is delayed")
756
+ const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
757
+
758
+ await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
759
+ await listenersRetired
173
760
  await fixture.client.request({command: "finalize-owner-replacement", replacementId: prepared.replacementId})
174
761
  await notification
175
762
  assert.equal(committed, true)
@@ -181,6 +768,40 @@ test("replacement commit notification waits for incumbent listener retirement",
181
768
  }
182
769
  })
183
770
 
771
+ test("completed direct listener retirement finalizes when the incumbent disconnects", {timeout: 3000}, async () => {
772
+ const fixture = await createGuardian()
773
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
774
+ const authority = {configDigest: "incumbent", runtime: null}
775
+ const nextAuthority = {configDigest: "candidate", runtime: null}
776
+
777
+ try {
778
+ await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v1"}})
779
+ await candidate.connect()
780
+ const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
781
+
782
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})
783
+ const committed = candidate.waitForEvent("replacement-committed")
784
+
785
+ await fixture.client.commitOwnerReplacement(prepared.replacementId)
786
+ await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
787
+ fixture.client.disconnect()
788
+ await committed
789
+ await candidate.finalizeOwnerReplacement(prepared.replacementId)
790
+ assert.deepEqual(await candidate.replacementStatus(), {
791
+ committedReplacementId: prepared.replacementId,
792
+ ownerClaimed: true,
793
+ retirementFailed: false,
794
+ retirementPending: false,
795
+ retirementReady: false
796
+ })
797
+ await candidate.shutdown()
798
+ await fixture.client.guardianExit()
799
+ } finally {
800
+ candidate.disconnect()
801
+ await cleanupGuardian(fixture)
802
+ }
803
+ })
804
+
184
805
  test("replacement staging rejects owner state published after prepare", async () => {
185
806
  const fixture = await createGuardian()
186
807
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
@@ -210,6 +831,115 @@ test("replacement staging rejects owner state published after prepare", async ()
210
831
  }
211
832
  })
212
833
 
834
+ test("staged replacement receives cleared local sources when the incumbent disconnects", {timeout: 3000}, async () => {
835
+ const fixture = await createGuardian()
836
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
837
+ const authority = {configDigest: "incumbent", runtime: null}
838
+ const nextAuthority = {configDigest: "candidate", runtime: null}
839
+ const snapshot = {activeReleaseId: "v1"}
840
+
841
+ try {
842
+ await fixture.client.publishOwnerState({
843
+ authority,
844
+ listenerConnectionSources: {"incumbent-local": {v1: {http: 0, websocket: 1}}},
845
+ listenerSourceId: "incumbent-local",
846
+ snapshot
847
+ })
848
+ await candidate.connect()
849
+ const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
850
+
851
+ await candidate.stageOwnerReplacement(prepared.replacementId, {
852
+ authority: nextAuthority,
853
+ listenerConnectionSources: {"incumbent-local": {v1: {http: 0, websocket: 1}}},
854
+ listenerSourceId: "candidate-local",
855
+ snapshot
856
+ })
857
+ const cleared = candidate.waitForEvent("owner-connection-state")
858
+ const committed = candidate.waitForEvent("replacement-committed")
859
+
860
+ fixture.client.disconnect()
861
+ assert.deepEqual(await cleared, {
862
+ connections: {http: 0, websocket: 0},
863
+ event: "owner-connection-state",
864
+ releaseId: "v1",
865
+ sourceId: "incumbent-local"
866
+ })
867
+ await committed
868
+ const state = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await candidate.ownerState())
869
+
870
+ assert.deepEqual(state.listenerConnectionSources, {})
871
+ await candidate.shutdown()
872
+ await fixture.client.guardianExit()
873
+ } finally {
874
+ candidate.disconnect()
875
+ await cleanupGuardian(fixture)
876
+ }
877
+ })
878
+
879
+ test("staged successor state receives tombstones when an older completed listener disconnects", {timeout: 3000}, async () => {
880
+ const fixture = await createGuardian()
881
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
882
+ const successor = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
883
+ const authority = {configDigest: "owner", runtime: null}
884
+ const snapshot = {activeReleaseId: "v1"}
885
+
886
+ try {
887
+ await fixture.client.publishOwnerState({authority, snapshot})
888
+ await candidate.connect()
889
+ const first = await candidate.prepareOwnerReplacement(authority, authority)
890
+
891
+ await candidate.stageOwnerReplacement(first.replacementId, {authority, listenerSourceId: "candidate-local", snapshot})
892
+ const firstCommitted = candidate.waitForEvent("replacement-committed")
893
+
894
+ await fixture.client.commitOwnerReplacement(first.replacementId)
895
+ const sourcePublished = candidate.waitForEvent("owner-connection-state")
896
+
897
+ await fixture.client.publishOwnerConnectionState(first.replacementId, "retired-local", "v1", {http: 0, websocket: 1}, true)
898
+ await sourcePublished
899
+ const firstListenersRetired = candidate.waitForEvent("replacement-listeners-retired")
900
+
901
+ await fixture.client.completeOwnerListenerRetirement(first.replacementId)
902
+ await firstListenersRetired
903
+ await candidate.finalizeOwnerReplacement(first.replacementId)
904
+ await firstCommitted
905
+
906
+ await successor.connect()
907
+ const second = await successor.prepareOwnerReplacement(authority, authority)
908
+
909
+ await successor.stageOwnerReplacement(second.replacementId, {
910
+ authority,
911
+ listenerConnectionSources: {"retired-local": {v1: {http: 0, websocket: 1}}},
912
+ listenerSourceId: "successor-local",
913
+ snapshot
914
+ })
915
+ const sourceCleared = candidate.waitForEvent("owner-connection-state")
916
+ const stagedSourceCleared = successor.waitForEvent("owner-connection-state")
917
+
918
+ fixture.client.disconnect()
919
+ const tombstone = {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "retired-local"}
920
+
921
+ assert.deepEqual(await Promise.all([sourceCleared, stagedSourceCleared]), [tombstone, tombstone])
922
+
923
+ await candidate.commitOwnerReplacement(second.replacementId)
924
+ const successorState = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await successor.ownerState())
925
+
926
+ assert.deepEqual(successorState.listenerConnectionSources, {})
927
+ const secondCommitted = successor.waitForEvent("replacement-committed")
928
+ const secondListenersRetired = successor.waitForEvent("replacement-listeners-retired")
929
+
930
+ await candidate.completeOwnerListenerRetirement(second.replacementId)
931
+ await secondListenersRetired
932
+ await successor.finalizeOwnerReplacement(second.replacementId)
933
+ await secondCommitted
934
+ await successor.shutdown()
935
+ await fixture.client.guardianExit()
936
+ } finally {
937
+ successor.disconnect()
938
+ candidate.disconnect()
939
+ await cleanupGuardian(fixture)
940
+ }
941
+ })
942
+
213
943
  test("replacement abort notifies both the candidate and committed owner", async () => {
214
944
  const fixture = await createGuardian()
215
945
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
@@ -298,76 +1028,365 @@ test("guardian owner-replacement capability classification is explicit and fail
298
1028
  }
299
1029
  })
300
1030
 
301
- test("reserved process recovery rejects a reconstructed definition with different provenance", async () => {
1031
+ test("reserved process recovery rejects a reconstructed definition with different provenance", async () => {
1032
+ const fixture = await createGuardian()
1033
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1034
+ const processKey = "release:v1:worker"
1035
+
1036
+ try {
1037
+ await fixture.client.process(processKey, definition("worker")).recover()
1038
+ const [registration] = await fixture.client.inventory()
1039
+
1040
+ assert.ok(registration)
1041
+ await candidate.connect()
1042
+ candidate.reserveProcessRecovery(processKey, registration.provenance)
1043
+ await assert.rejects(
1044
+ () => candidate.process(processKey, definition("different-worker")).recover(),
1045
+ /provenance mismatch for reserved process/
1046
+ )
1047
+ assert.deepEqual((await fixture.client.inventory()).map(({key}) => key), [processKey])
1048
+ } finally {
1049
+ candidate.disconnect()
1050
+ await cleanupGuardian(fixture)
1051
+ }
1052
+ })
1053
+
1054
+ test("retired owner replacement rejects a registered process absent from committed owner state", async () => {
1055
+ const fixture = await createGuardian()
1056
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1057
+ const committedProcessKey = "release:v1:worker"
1058
+ const candidateProcessKey = "release:candidate:worker"
1059
+ const authority = {configDigest: "owner", runtime: null}
1060
+ const snapshot = {
1061
+ activeReleaseId: "v1",
1062
+ control: {path: path.join(fixture.root, "rollbridge.sock")},
1063
+ releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
1064
+ services: [],
1065
+ singletons: []
1066
+ }
1067
+ const candidateSnapshot = {
1068
+ ...snapshot,
1069
+ releases: [...snapshot.releases, {processes: [{id: "worker"}], releaseId: "candidate"}]
1070
+ }
1071
+
1072
+ try {
1073
+ await fixture.client.process(committedProcessKey, definition("worker")).recover()
1074
+ await fixture.client.process(candidateProcessKey, definition("candidate-worker")).recover()
1075
+ await fixture.client.publishOwnerState({authority, snapshot})
1076
+ await candidate.connect()
1077
+ const prepared = await candidate.prepareOwnerReplacement(authority, authority)
1078
+
1079
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot: candidateSnapshot})
1080
+ await assert.rejects(
1081
+ () => candidate.commitRetiredOwnerReplacement(prepared.replacementId, candidateProcessKey),
1082
+ /process .* does not belong to the committed owner/
1083
+ )
1084
+ } finally {
1085
+ candidate.disconnect()
1086
+ await cleanupGuardian(fixture)
1087
+ }
1088
+ })
1089
+
1090
+ test("retired owner replacement requires unchanged authority and the exact control path absent", async () => {
1091
+ const fixture = await createGuardian()
1092
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1093
+ const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1094
+ const controlPath = path.join(fixture.root, "rollbridge.sock")
1095
+ const processKey = "release:v1:worker"
1096
+ const authority = {configDigest: "incumbent", runtime: null}
1097
+ const nextAuthority = {configDigest: "candidate", runtime: null}
1098
+ const snapshot = {
1099
+ activeReleaseId: "v1",
1100
+ control: {path: controlPath},
1101
+ releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
1102
+ services: [],
1103
+ singletons: []
1104
+ }
1105
+
1106
+ try {
1107
+ await fixture.client.process(processKey, definition("worker")).recover()
1108
+ await fixture.client.publishOwnerState({authority, snapshot})
1109
+ await candidate.connect()
1110
+ const changed = await candidate.prepareOwnerReplacement(authority, nextAuthority)
1111
+
1112
+ await candidate.stageOwnerReplacement(changed.replacementId, {authority: nextAuthority, snapshot})
1113
+ await assert.rejects(() => candidate.commitRetiredOwnerReplacement(changed.replacementId, processKey), /unchanged owner authority/)
1114
+ await candidate.abortOwnerReplacement(changed.replacementId)
1115
+
1116
+ const occupied = await candidate.prepareOwnerReplacement(authority, authority)
1117
+
1118
+ await candidate.stageOwnerReplacement(occupied.replacementId, {authority, snapshot})
1119
+ await fs.writeFile(controlPath, "occupied\n")
1120
+ await assert.rejects(() => candidate.commitRetiredOwnerReplacement(occupied.replacementId, processKey), /control socket .* still exists/)
1121
+ await candidate.abortOwnerReplacement(occupied.replacementId)
1122
+
1123
+ await fs.rm(controlPath)
1124
+ const ready = await candidate.prepareOwnerReplacement(authority, authority)
1125
+
1126
+ await candidate.stageOwnerReplacement(ready.replacementId, {authority, snapshot})
1127
+ await contender.connect()
1128
+ await assert.rejects(
1129
+ () => contender.request({command: "commit-retired-owner-replacement", key: processKey, replacementId: ready.replacementId}),
1130
+ /not the prepared candidate/
1131
+ )
1132
+ await assert.rejects(
1133
+ () => candidate.commitRetiredOwnerReplacement("stale-replacement", processKey),
1134
+ /not the prepared candidate/
1135
+ )
1136
+ await assert.rejects(
1137
+ () => candidate.commitRetiredOwnerReplacement(ready.replacementId, "release:v1:wrong"),
1138
+ /process .* is not registered/
1139
+ )
1140
+ const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
1141
+
1142
+ void handoffRequested.catch(() => undefined)
1143
+
1144
+ await candidate.prepareRetiredOwnerListenerHandoff(ready.replacementId, processKey)
1145
+ await handoffRequested
1146
+ await assert.rejects(
1147
+ () => contender.prepareOwnerReplacement(authority, authority),
1148
+ /listener retirement is pending/
1149
+ )
1150
+ const committed = candidate.waitForEvent("replacement-committed")
1151
+ const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
1152
+ const connectionState = candidate.waitForEvent("owner-connection-state")
1153
+
1154
+ await fixture.client.publishOwnerConnectionState(ready.replacementId, "listener-a", "v1", {http: 1, websocket: 2}, true)
1155
+ assert.deepEqual(await connectionState, {connections: {http: 1, websocket: 2}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
1156
+ await fixture.client.completeOwnerListenerRetirement(ready.replacementId)
1157
+ await listenersRetired
1158
+ const retirementRequested = fixture.client.waitForEvent("replacement-retirement-requested")
1159
+
1160
+ await candidate.commitRetiredOwnerReplacement(ready.replacementId, processKey)
1161
+ await retirementRequested
1162
+ await candidate.finalizeOwnerReplacement(ready.replacementId)
1163
+ await committed
1164
+ await candidate.shutdown()
1165
+ await fixture.client.guardianExit()
1166
+ } finally {
1167
+ contender.disconnect()
1168
+ candidate.disconnect()
1169
+ await cleanupGuardian(fixture)
1170
+ }
1171
+ })
1172
+
1173
+ test("completed listener retirement survives owner recovery and clears a crashed local source", async () => {
1174
+ const fixture = await createGuardian()
1175
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1176
+ const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1177
+ const controlPath = path.join(fixture.root, "rollbridge.sock")
1178
+ const processKey = "release:v1:worker"
1179
+ const authority = {configDigest: "owner", runtime: null}
1180
+ const snapshot = {
1181
+ activeReleaseId: "v1",
1182
+ control: {path: controlPath},
1183
+ releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
1184
+ services: [],
1185
+ singletons: []
1186
+ }
1187
+
1188
+ try {
1189
+ await fixture.client.process(processKey, definition("worker")).recover()
1190
+ await fixture.client.publishOwnerState({authority, snapshot})
1191
+ await candidate.connect()
1192
+ const prepared = await candidate.prepareOwnerReplacement(authority, authority)
1193
+
1194
+ await candidate.stageOwnerReplacement(prepared.replacementId, {
1195
+ authority,
1196
+ listenerConnectionSources: {"candidate-local": {v1: {http: 0, websocket: 1}}},
1197
+ listenerSourceId: "candidate-local",
1198
+ snapshot
1199
+ })
1200
+ const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
1201
+
1202
+ void handoffRequested.catch(() => undefined)
1203
+
1204
+ await candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
1205
+ await handoffRequested
1206
+ const initial = candidate.waitForEvent("owner-connection-state")
1207
+
1208
+ await fixture.client.publishOwnerConnectionState(prepared.replacementId, "listener-a", "v1", {http: 0, websocket: 1}, true)
1209
+ assert.deepEqual(await initial, {connections: {http: 0, websocket: 1}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
1210
+ const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
1211
+
1212
+ await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
1213
+ await listenersRetired
1214
+ await candidate.commitRetiredOwnerReplacement(prepared.replacementId, processKey)
1215
+ candidate.disconnect()
1216
+ await recovered.connect()
1217
+ await recovered.claimOwner(1000, authority)
1218
+ const stateAfterCandidateCrash = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
1219
+
1220
+ assert.equal(stateAfterCandidateCrash.listenerConnectionSources?.["candidate-local"], undefined)
1221
+ assert.deepEqual(stateAfterCandidateCrash.listenerConnectionSources?.["listener-a"], {v1: {http: 0, websocket: 1}})
1222
+ assert.deepEqual(await recovered.replacementStatus(), {
1223
+ committedReplacementId: prepared.replacementId,
1224
+ ownerClaimed: true,
1225
+ retirementFailed: false,
1226
+ retirementPending: true,
1227
+ retirementReady: true
1228
+ })
1229
+ await recovered.finalizeOwnerReplacement(prepared.replacementId)
1230
+ const cleared = recovered.waitForEvent("owner-connection-state")
1231
+
1232
+ fixture.client.disconnect()
1233
+ assert.deepEqual(await Promise.race([
1234
+ cleared,
1235
+ new Promise((_, reject) => {
1236
+ const timer = setTimeout(() => reject(new Error("Recovered owner did not receive the retired source tombstone")), 500)
1237
+
1238
+ timer.unref()
1239
+ })
1240
+ ]), {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
1241
+ const recoveredOwnerState = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
1242
+
1243
+ assert.deepEqual(recoveredOwnerState.listenerConnectionSources, {})
1244
+ await recovered.publishOwnerState({
1245
+ authority,
1246
+ listenerConnectionSources: {"listener-a": {v1: {http: 0, websocket: 1}}},
1247
+ listenerSourceId: "recovered-local",
1248
+ snapshot
1249
+ })
1250
+ const afterStalePublication = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
1251
+
1252
+ assert.deepEqual(afterStalePublication.listenerConnectionSources, {})
1253
+ const recoveredProcess = recovered.process(processKey, definition("worker"))
1254
+
1255
+ await recoveredProcess.recover()
1256
+ await recoveredProcess.updateDefinition({...definition("worker"), env: {REVISION: "stale"}}, {
1257
+ authority,
1258
+ listenerConnectionSources: {"listener-a": {v1: {http: 0, websocket: 1}}},
1259
+ listenerSourceId: "recovered-local",
1260
+ snapshot
1261
+ })
1262
+ const afterStaleProcessUpdate = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
1263
+
1264
+ assert.deepEqual(afterStaleProcessUpdate.listenerConnectionSources, {})
1265
+ await recovered.shutdown()
1266
+ await fixture.client.guardianExit()
1267
+ } finally {
1268
+ recovered.disconnect()
1269
+ candidate.disconnect()
1270
+ await cleanupGuardian(fixture)
1271
+ }
1272
+ })
1273
+
1274
+ test("direct retired-listener source relay survives committed owner recovery", async () => {
302
1275
  const fixture = await createGuardian()
303
1276
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
304
- const processKey = "release:v1:worker"
1277
+ const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1278
+ const authority = {configDigest: "owner", runtime: null}
1279
+ const snapshot = {activeReleaseId: "v1", control: {path: path.join(fixture.root, "rollbridge.sock")}, releases: [], services: [], singletons: []}
305
1280
 
306
1281
  try {
307
- await fixture.client.process(processKey, definition("worker")).recover()
308
- const [registration] = await fixture.client.inventory()
309
-
310
- assert.ok(registration)
1282
+ await fixture.client.publishOwnerState({authority, snapshot})
311
1283
  await candidate.connect()
312
- candidate.reserveProcessRecovery(processKey, registration.provenance)
313
- await assert.rejects(
314
- () => candidate.process(processKey, definition("different-worker")).recover(),
315
- /provenance mismatch for reserved process/
316
- )
317
- assert.deepEqual((await fixture.client.inventory()).map(({key}) => key), [processKey])
1284
+ const prepared = await candidate.prepareOwnerReplacement(authority, authority)
1285
+
1286
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority, listenerSourceId: "candidate-local", snapshot})
1287
+ await fixture.client.commitOwnerReplacement(prepared.replacementId)
1288
+ const initial = candidate.waitForEvent("owner-connection-state")
1289
+
1290
+ await fixture.client.publishOwnerConnectionState(prepared.replacementId, "direct-listener", "v1", {http: 0, websocket: 1}, true)
1291
+ assert.deepEqual(await initial, {connections: {http: 0, websocket: 1}, event: "owner-connection-state", releaseId: "v1", sourceId: "direct-listener"})
1292
+ const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
1293
+
1294
+ await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
1295
+ await listenersRetired
1296
+ await fixture.client.finalizeOwnerReplacement(prepared.replacementId)
1297
+ candidate.disconnect()
1298
+ await recovered.connect()
1299
+ await recovered.claimOwner(1000, authority)
1300
+ const cleared = recovered.waitForEvent("owner-connection-state")
1301
+
1302
+ await fixture.client.publishOwnerConnectionState(prepared.replacementId, "direct-listener", "v1", {http: 0, websocket: 0}, true)
1303
+ assert.deepEqual(await cleared, {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "direct-listener"})
1304
+ await recovered.shutdown()
1305
+ await fixture.client.guardianExit()
318
1306
  } finally {
1307
+ recovered.disconnect()
319
1308
  candidate.disconnect()
320
1309
  await cleanupGuardian(fixture)
321
1310
  }
322
1311
  })
323
1312
 
324
- test("retired owner replacement rejects a registered process absent from committed owner state", async () => {
1313
+ test("incumbent listener disconnect before state completion aborts without committing the candidate", async () => {
325
1314
  const fixture = await createGuardian()
326
1315
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
327
- const committedProcessKey = "release:v1:worker"
328
- const candidateProcessKey = "release:candidate:worker"
1316
+ const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1317
+ const controlPath = path.join(fixture.root, "rollbridge.sock")
1318
+ const processKey = "release:v1:worker"
329
1319
  const authority = {configDigest: "owner", runtime: null}
330
1320
  const snapshot = {
331
1321
  activeReleaseId: "v1",
332
- control: {path: path.join(fixture.root, "rollbridge.sock")},
1322
+ control: {path: controlPath},
333
1323
  releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
334
1324
  services: [],
335
1325
  singletons: []
336
1326
  }
337
- const candidateSnapshot = {
338
- ...snapshot,
339
- releases: [...snapshot.releases, {processes: [{id: "worker"}], releaseId: "candidate"}]
340
- }
341
1327
 
342
1328
  try {
343
- await fixture.client.process(committedProcessKey, definition("worker")).recover()
344
- await fixture.client.process(candidateProcessKey, definition("candidate-worker")).recover()
1329
+ await fixture.client.process(processKey, definition("worker")).recover()
345
1330
  await fixture.client.publishOwnerState({authority, snapshot})
346
1331
  await candidate.connect()
347
1332
  const prepared = await candidate.prepareOwnerReplacement(authority, authority)
348
1333
 
349
- await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot: candidateSnapshot})
350
- await assert.rejects(
351
- () => candidate.commitRetiredOwnerReplacement(prepared.replacementId, candidateProcessKey),
352
- /process .* does not belong to the committed owner/
353
- )
1334
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot})
1335
+ const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
1336
+ const failed = candidate.waitForEvent("replacement-retirement-failed")
1337
+ const aborted = candidate.waitForEvent("replacement-aborted")
1338
+ let tombstones = 0
1339
+
1340
+ candidate.onEvent("owner-connection-state", (event) => {
1341
+ if (event.sourceId === "incumbent-local" && event.releaseId === "v1" && event.connections && typeof event.connections === "object" && !Array.isArray(event.connections) && event.connections.http === 0 && event.connections.websocket === 0) {
1342
+ tombstones += 1
1343
+ }
1344
+ })
1345
+
1346
+ await candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
1347
+ await handoffRequested
1348
+ const sourcePublished = candidate.waitForEvent("owner-connection-state")
1349
+
1350
+ await fixture.client.publishOwnerConnectionState(prepared.replacementId, "incumbent-local", "v1", {http: 0, websocket: 1}, true)
1351
+ await sourcePublished
1352
+ fixture.client.disconnect()
1353
+ assert.deepEqual(await failed, {
1354
+ event: "replacement-retirement-failed",
1355
+ reason: "Incumbent listener disconnected during the prepared handoff",
1356
+ replacementId: prepared.replacementId
1357
+ })
1358
+ assert.deepEqual(await aborted, {
1359
+ event: "replacement-aborted",
1360
+ reason: "Incumbent listener disconnected during the prepared handoff"
1361
+ })
1362
+ assert.equal(tombstones, 1)
1363
+ assert.deepEqual(await candidate.replacementStatus(), {
1364
+ committedReplacementId: null,
1365
+ ownerClaimed: false,
1366
+ retirementFailed: false,
1367
+ retirementPending: false,
1368
+ retirementReady: false
1369
+ })
1370
+ await recovered.connect()
1371
+ await recovered.claimOwner(1000, authority)
1372
+ await recovered.shutdown()
1373
+ await fixture.client.guardianExit()
354
1374
  } finally {
1375
+ recovered.disconnect()
355
1376
  candidate.disconnect()
356
1377
  await cleanupGuardian(fixture)
357
1378
  }
358
1379
  })
359
1380
 
360
- test("retired owner replacement requires unchanged authority and the exact control path absent", async () => {
1381
+ test("replacement abort during listener handoff validation leaves the incumbent available", async () => {
361
1382
  const fixture = await createGuardian()
362
1383
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
363
1384
  const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
364
- const controlPath = path.join(fixture.root, "rollbridge.sock")
365
1385
  const processKey = "release:v1:worker"
366
- const authority = {configDigest: "incumbent", runtime: null}
367
- const nextAuthority = {configDigest: "candidate", runtime: null}
1386
+ const authority = {configDigest: "owner", runtime: null}
368
1387
  const snapshot = {
369
1388
  activeReleaseId: "v1",
370
- control: {path: controlPath},
1389
+ control: {path: path.join(fixture.root, "rollbridge.sock")},
371
1390
  releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
372
1391
  services: [],
373
1392
  singletons: []
@@ -377,45 +1396,68 @@ test("retired owner replacement requires unchanged authority and the exact contr
377
1396
  await fixture.client.process(processKey, definition("worker")).recover()
378
1397
  await fixture.client.publishOwnerState({authority, snapshot})
379
1398
  await candidate.connect()
380
- const changed = await candidate.prepareOwnerReplacement(authority, nextAuthority)
1399
+ const prepared = await candidate.prepareOwnerReplacement(authority, authority)
381
1400
 
382
- await candidate.stageOwnerReplacement(changed.replacementId, {authority: nextAuthority, snapshot})
383
- await assert.rejects(() => candidate.commitRetiredOwnerReplacement(changed.replacementId, processKey), /unchanged owner authority/)
384
- await candidate.abortOwnerReplacement(changed.replacementId)
1401
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot})
1402
+ const abandonedHandoff = candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
385
1403
 
386
- const occupied = await candidate.prepareOwnerReplacement(authority, authority)
1404
+ await candidate.abortOwnerReplacement(prepared.replacementId)
1405
+ await assert.rejects(() => abandonedHandoff, /prepared candidate/)
1406
+ await contender.connect()
1407
+ const fresh = await contender.prepareOwnerReplacement(authority, authority)
387
1408
 
388
- await candidate.stageOwnerReplacement(occupied.replacementId, {authority, snapshot})
389
- await fs.writeFile(controlPath, "occupied\n")
390
- await assert.rejects(() => candidate.commitRetiredOwnerReplacement(occupied.replacementId, processKey), /control socket .* still exists/)
391
- await candidate.abortOwnerReplacement(occupied.replacementId)
1409
+ await contender.abortOwnerReplacement(fresh.replacementId)
1410
+ await fixture.client.shutdown()
1411
+ await fixture.client.guardianExit()
1412
+ } finally {
1413
+ contender.disconnect()
1414
+ candidate.disconnect()
1415
+ await cleanupGuardian(fixture)
1416
+ }
1417
+ })
392
1418
 
393
- await fs.rm(controlPath)
394
- const ready = await candidate.prepareOwnerReplacement(authority, authority)
1419
+ test("explicit replacement abort notifies the incumbent owner", async () => {
1420
+ const fixture = await createGuardian()
1421
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1422
+ const authority = {configDigest: "incumbent", runtime: null}
1423
+ const nextAuthority = {configDigest: "candidate", runtime: null}
395
1424
 
396
- await candidate.stageOwnerReplacement(ready.replacementId, {authority, snapshot})
1425
+ try {
1426
+ await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v1"}})
1427
+ await candidate.connect()
1428
+ const aborted = fixture.client.waitForEvent("replacement-aborted")
1429
+ const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
1430
+
1431
+ await candidate.abortOwnerReplacement(prepared.replacementId)
1432
+ await aborted
1433
+ await fixture.client.shutdown()
1434
+ await fixture.client.guardianExit()
1435
+ } finally {
1436
+ candidate.disconnect()
1437
+ await cleanupGuardian(fixture)
1438
+ }
1439
+ })
1440
+
1441
+ test("queued owner claim is revalidated against the latest committed authority", async () => {
1442
+ const fixture = await createGuardian()
1443
+ const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1444
+ const authority = {configDigest: "old", runtime: null}
1445
+ const nextAuthority = {configDigest: "new", runtime: null}
1446
+
1447
+ try {
1448
+ await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: null}})
397
1449
  await contender.connect()
398
- await assert.rejects(
399
- () => contender.request({command: "commit-retired-owner-replacement", key: processKey, replacementId: ready.replacementId}),
400
- /not the prepared candidate/
401
- )
402
- await assert.rejects(
403
- () => candidate.commitRetiredOwnerReplacement("stale-replacement", processKey),
404
- /not the prepared candidate/
405
- )
406
- await assert.rejects(
407
- () => candidate.commitRetiredOwnerReplacement(ready.replacementId, "release:v1:wrong"),
408
- /process .* is not registered/
409
- )
410
- const committed = candidate.waitForEvent("replacement-committed")
1450
+ const claim = contender.claimOwner(500, authority)
411
1451
 
412
- await candidate.commitRetiredOwnerReplacement(ready.replacementId, processKey)
413
- await committed
414
- await candidate.shutdown()
1452
+ await new Promise((resolve) => setTimeout(resolve, 20))
1453
+ await fixture.client.publishOwnerState({authority: nextAuthority, snapshot: {activeReleaseId: null}})
1454
+ fixture.client.disconnect()
1455
+ await assert.rejects(claim, /authority changed while the claim was queued/)
1456
+ await contender.claimOwner(500, nextAuthority)
1457
+ await contender.shutdown()
415
1458
  await fixture.client.guardianExit()
416
1459
  } finally {
417
1460
  contender.disconnect()
418
- candidate.disconnect()
419
1461
  await cleanupGuardian(fixture)
420
1462
  }
421
1463
  })
@@ -429,6 +1471,7 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
429
1471
 
430
1472
  try {
431
1473
  await legacyProcess.start()
1474
+ await assert.rejects(() => fixture.client.capabilities(), /Guardian capabilities requires a process key/)
432
1475
  legacyPid = legacyProcess.status().pid
433
1476
  const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
434
1477
  const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
@@ -469,6 +1512,190 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
469
1512
  }
470
1513
  })
471
1514
 
1515
+ test("split guardian rejects an owner-state update when its nested legacy definition update fails", async () => {
1516
+ const fixture = await createLegacyGuardian()
1517
+ const processDefinition = definition("legacy-worker")
1518
+ const legacyProcess = fixture.client.process("release:v1:legacy-worker", processDefinition)
1519
+ const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
1520
+ const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
1521
+ const ownerState = {
1522
+ authority,
1523
+ snapshot: {activeReleaseId: "v1", releases: [{processes: [{id: "legacy-worker"}], releaseId: "v1"}], services: [], singletons: []}
1524
+ }
1525
+ let upgraded
1526
+
1527
+ try {
1528
+ await legacyProcess.start()
1529
+ upgraded = await fixture.client.upgradeLegacyGuardian({
1530
+ ownerState,
1531
+ socketPath: path.join(fixture.root, "guardian-v2.sock"),
1532
+ token: "candidate-guardian-capability"
1533
+ })
1534
+ const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
1535
+ const committedOwnerState = {authority: nextAuthority, snapshot: ownerState.snapshot}
1536
+
1537
+ await upgraded.stageOwnerReplacement(prepared.replacementId, committedOwnerState)
1538
+ const restored = upgraded.process("release:v1:legacy-worker", processDefinition)
1539
+
1540
+ await restored.recover()
1541
+ await legacyProcess.updateDefinition({...processDefinition, env: {REVISION: "external"}})
1542
+ await assert.rejects(
1543
+ () => restored.updateDefinition({...processDefinition, env: {REVISION: "candidate"}}, {authority: nextAuthority, snapshot: {...ownerState.snapshot, serviceReleaseIds: {service: "v2"}}}),
1544
+ /provenance mismatch/
1545
+ )
1546
+ assert.deepEqual(await upgraded.ownerState(), committedOwnerState)
1547
+ } finally {
1548
+ await legacyProcess.stop().catch(() => {})
1549
+ upgraded?.disconnect()
1550
+ fixture.client.disconnect()
1551
+ if (upgraded?.pid) killExactProcessGroup(upgraded.pid)
1552
+ if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
1553
+ await fs.rm(fixture.root, {force: true, recursive: true})
1554
+ }
1555
+ })
1556
+
1557
+ test("split guardian defers owner handoff until a nested legacy definition update commits", async () => {
1558
+ const fixture = await createLegacyGuardian()
1559
+ const processDefinition = definition("legacy-worker")
1560
+ const legacyProcess = fixture.client.process("release:v1:legacy-worker", processDefinition)
1561
+ const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
1562
+ const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
1563
+ const ownerState = {
1564
+ authority,
1565
+ snapshot: {activeReleaseId: "v1", releases: [{processes: [{id: "legacy-worker"}], releaseId: "v1"}], services: [], singletons: []}
1566
+ }
1567
+ const socketPath = path.join(fixture.root, "guardian-v2.sock")
1568
+ const token = "candidate-guardian-capability"
1569
+ const gatePath = path.join(fixture.root, "legacy-update.allow")
1570
+ const committedOwnerState = {
1571
+ authority: nextAuthority,
1572
+ listenerConnectionSources: {"updating-owner": {v1: {http: 0, websocket: 1}}},
1573
+ listenerSourceId: "updating-owner",
1574
+ snapshot: {...ownerState.snapshot, update: "committed"}
1575
+ }
1576
+ const contender = new GuardianClient({socketPath, token})
1577
+ let upgraded
1578
+
1579
+ try {
1580
+ await legacyProcess.start()
1581
+ upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, socketPath, token})
1582
+ const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
1583
+
1584
+ await upgraded.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: ownerState.snapshot})
1585
+ const restored = upgraded.process("release:v1:legacy-worker", processDefinition)
1586
+
1587
+ await restored.recover()
1588
+ await contender.connect()
1589
+ const updateResult = restored.updateDefinition({...processDefinition, env: {ROLLBRIDGE_TEST_UPDATE_GATE: gatePath}}, committedOwnerState)
1590
+ .then(() => undefined, (error) => error)
1591
+
1592
+ await waitForFileText(`${gatePath}.waiting`)
1593
+ const claim = contender.claimOwner(1000, nextAuthority)
1594
+ let claimSettled = false
1595
+
1596
+ void claim.finally(() => { claimSettled = true })
1597
+ upgraded.disconnect()
1598
+ await new Promise((resolve) => setTimeout(resolve, 50))
1599
+ assert.equal(claimSettled, false, "owner handoff must wait for the nested definition update")
1600
+
1601
+ await fs.writeFile(gatePath, "allow\n")
1602
+ await claim
1603
+ const updateError = await updateResult
1604
+
1605
+ assert.match(String(updateError), /connection closed while awaiting update/)
1606
+ assert.deepEqual(await contender.ownerState(), {...committedOwnerState, listenerConnectionSources: {}})
1607
+ } finally {
1608
+ await fs.writeFile(gatePath, "allow\n").catch(() => {})
1609
+ await contender.shutdown().catch(() => {})
1610
+ contender.disconnect()
1611
+ upgraded?.disconnect()
1612
+ await legacyProcess.stop().catch(() => {})
1613
+ fixture.client.disconnect()
1614
+ if (upgraded?.pid) killExactProcessGroup(upgraded.pid)
1615
+ if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
1616
+ await fs.rm(fixture.root, {force: true, recursive: true})
1617
+ }
1618
+ })
1619
+
1620
+ test("a disconnected legacy upgrade candidate does not strand its bridge guardian", async () => {
1621
+ const fixture = await createLegacyGuardian()
1622
+ const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
1623
+ const ownerState = {
1624
+ authority,
1625
+ snapshot: {activeReleaseId: null, releases: [], services: [], singletons: []}
1626
+ }
1627
+ let upgraded
1628
+ let bridgeExited = false
1629
+
1630
+ try {
1631
+ upgraded = await fixture.client.upgradeLegacyGuardian({
1632
+ ownerState,
1633
+ socketPath: path.join(fixture.root, "guardian-v2.sock"),
1634
+ token: "candidate-guardian-capability"
1635
+ })
1636
+ await upgraded.prepareOwnerReplacement(authority, {...authority, runtime: {...authority.runtime, digest: "candidate-runtime"}})
1637
+ upgraded.disconnect()
1638
+ let timeout = /** @type {ReturnType<typeof setTimeout> | undefined} */ (undefined)
1639
+
1640
+ await Promise.race([
1641
+ upgraded.guardianExit(),
1642
+ new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Legacy upgrade bridge guardian did not exit after candidate disconnect")), 1000) })
1643
+ ]).finally(() => { if (timeout) clearTimeout(timeout) })
1644
+ bridgeExited = true
1645
+ assert.equal(fixture.child.exitCode, null, "the pre-split guardian must remain available after bridge abandonment")
1646
+ } finally {
1647
+ upgraded?.disconnect()
1648
+ if (!bridgeExited && upgraded?.pid) killExactProcessGroup(upgraded.pid)
1649
+ await cleanupGuardian({client: fixture.client, root: fixture.root})
1650
+ }
1651
+ })
1652
+
1653
+ test("a legacy bridge remains discoverable when its candidate disconnects after the disruptive boundary", async () => {
1654
+ const fixture = await createLegacyGuardian()
1655
+ const statePath = path.join(fixture.root, "state.json")
1656
+ const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
1657
+ const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime"}}
1658
+ const ownerState = {
1659
+ authority,
1660
+ config: {statePath},
1661
+ snapshot: {activeReleaseId: null, control: {path: path.join(fixture.root, "control.sock")}, releases: [], services: [], singletons: []}
1662
+ }
1663
+ let upgraded
1664
+ let replacement
1665
+
1666
+ try {
1667
+ const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
1668
+
1669
+ upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
1670
+ const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
1671
+ const recoverySnapshot = {
1672
+ ...ownerState.snapshot,
1673
+ recovery: {configDigest: authority.configDigest, format: 1, guardian: {...identity, pid: upgraded.pid}, reconnectGraceMs: 3000}
1674
+ }
1675
+
1676
+ await fs.writeFile(statePath, `${JSON.stringify(ownerState.snapshot, null, 2)}\n`)
1677
+ await upgraded.beginLegacyOwnerClaim(prepared.replacementId, 1000, statePath, recoverySnapshot)
1678
+ fixture.client.disconnect()
1679
+ await upgraded.completeLegacyOwnerClaim(prepared.replacementId)
1680
+ assert.deepEqual(JSON.parse(await fs.readFile(statePath, "utf8")), recoverySnapshot)
1681
+ upgraded.disconnect()
1682
+
1683
+ replacement = new GuardianClient({...identity, pid: upgraded.pid})
1684
+ await replacement.connect()
1685
+ const resumed = await replacement.prepareOwnerReplacement(authority, nextAuthority)
1686
+ const committed = replacement.waitForEvent("replacement-committed")
1687
+
1688
+ assert.deepEqual(await replacement.stageOwnerReplacement(resumed.replacementId, {authority: nextAuthority, config: {statePath}, snapshot: ownerState.snapshot}), {committed: true})
1689
+ await committed
1690
+ await replacement.shutdown()
1691
+ await upgraded.guardianExit()
1692
+ } finally {
1693
+ upgraded?.disconnect()
1694
+ replacement?.disconnect()
1695
+ await cleanupGuardian({client: fixture.client, root: fixture.root})
1696
+ }
1697
+ })
1698
+
472
1699
  /** @returns {Promise<{client: GuardianClient, root: string, token: string}>} Started guardian fixture. */
473
1700
  async function createGuardian() {
474
1701
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-guardian-client-"))
@@ -524,9 +1751,139 @@ async function cleanupGuardian(fixture) {
524
1751
  await fs.rm(fixture.root, {force: true, recursive: true})
525
1752
  }
526
1753
 
1754
+ /**
1755
+ * @param {Record<string, import("../src/json.js").JsonValue>} authority - Recovery authority.
1756
+ * @param {string} markerPath - Restart marker path.
1757
+ * @param {string} marker - Expected accepted command marker.
1758
+ * @param {number} reconnectGraceMs - Initial reconnect grace.
1759
+ * @returns {Record<string, import("../src/json.js").JsonValue>} Guardian owner state.
1760
+ */
1761
+ function recoveryOwnerState(authority, markerPath, marker, reconnectGraceMs) {
1762
+ return {
1763
+ authority,
1764
+ recovery: {
1765
+ command: {
1766
+ args: ["-e", "require('node:fs').appendFileSync(process.argv[1], JSON.stringify({at: Date.now(), home: process.env.HOME ?? null, marker: process.env.RESTART_MARKER}) + '\\n')", markerPath],
1767
+ cwd: path.dirname(markerPath),
1768
+ env: {RESTART_MARKER: marker},
1769
+ executable: process.execPath
1770
+ },
1771
+ reconnectGraceMs,
1772
+ startupTimeoutMs: 1000
1773
+ },
1774
+ snapshot: {activeReleaseId: null}
1775
+ }
1776
+ }
1777
+
1778
+ /**
1779
+ * @param {{client: GuardianClient, root: string, token: string}} fixture - Guardian fixture.
1780
+ * @param {Record<string, import("../src/json.js").JsonValue>} authority - Recovery authority.
1781
+ * @param {string} markerPath - Claim marker path.
1782
+ * @param {{claimDelayMs?: number, descendantPath?: string, exitAfterClaim?: boolean, ready?: boolean, replacementCommittedPath?: string, replacementPreparedPath?: string, startedLogPath?: string, startedPath?: string, startupTimeoutMs?: number}} [options] - Optional recovery behavior.
1783
+ * @returns {Record<string, import("../src/json.js").JsonValue>} Guardian owner state.
1784
+ */
1785
+ function claimingRecoveryOwnerState(fixture, authority, markerPath, options = {}) {
1786
+ return {
1787
+ authority,
1788
+ recovery: {
1789
+ command: {
1790
+ args: [recoveryOwnerPath],
1791
+ cwd: fixture.root,
1792
+ env: {
1793
+ GUARDIAN_AUTHORITY: JSON.stringify(authority),
1794
+ GUARDIAN_CLAIM_DELAY_MS: String(options.claimDelayMs || 0),
1795
+ ...(options.descendantPath ? {GUARDIAN_DESCENDANT_PATH: options.descendantPath} : {}),
1796
+ ...(options.exitAfterClaim ? {GUARDIAN_EXIT_AFTER_CLAIM: "1"} : {}),
1797
+ GUARDIAN_MARKER_PATH: markerPath,
1798
+ ...(options.replacementCommittedPath ? {GUARDIAN_REPLACEMENT_COMMITTED_PATH: options.replacementCommittedPath} : {}),
1799
+ ...(options.replacementPreparedPath ? {GUARDIAN_REPLACEMENT_PREPARED_PATH: options.replacementPreparedPath} : {}),
1800
+ ...(options.ready === false ? {GUARDIAN_SKIP_READY: "1"} : {}),
1801
+ GUARDIAN_SOCKET_PATH: fixture.client.socketPath,
1802
+ ...(options.startedLogPath ? {GUARDIAN_STARTED_LOG_PATH: options.startedLogPath} : {}),
1803
+ ...(options.startedPath ? {GUARDIAN_STARTED_PATH: options.startedPath} : {}),
1804
+ GUARDIAN_TOKEN: fixture.token
1805
+ },
1806
+ executable: process.execPath
1807
+ },
1808
+ reconnectGraceMs: 10,
1809
+ startupTimeoutMs: options.startupTimeoutMs || 1000
1810
+ },
1811
+ snapshot: {activeReleaseId: null}
1812
+ }
1813
+ }
1814
+
1815
+ /**
1816
+ * @param {string} markerPath - Restart marker path.
1817
+ * @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Restart records.
1818
+ */
1819
+ async function restartRecords(markerPath) {
1820
+ try {
1821
+ return (await fs.readFile(markerPath, "utf8")).trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))
1822
+ } catch (error) {
1823
+ if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return []
1824
+ throw error
1825
+ }
1826
+ }
1827
+
1828
+ /**
1829
+ * @param {string} markerPath - Restart marker path.
1830
+ * @param {number} count - Required record count.
1831
+ * @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Restart records.
1832
+ */
1833
+ async function waitForRestartRecords(markerPath, count) {
1834
+ const deadline = Date.now() + 3000
1835
+
1836
+ while (Date.now() < deadline) {
1837
+ const records = await restartRecords(markerPath)
1838
+
1839
+ if (records.length >= count) return records
1840
+ await new Promise((resolve) => setTimeout(resolve, 10))
1841
+ }
1842
+ throw new Error(`Timed out waiting for ${count} guardian restart record${count === 1 ? "" : "s"}`)
1843
+ }
1844
+
1845
+ /**
1846
+ * @param {string} filePath - File to read after creation.
1847
+ * @param {RegExp} [expected] - Content that must be present before returning.
1848
+ * @returns {Promise<string>} Non-empty accepted file contents.
1849
+ */
1850
+ async function waitForFileText(filePath, expected) {
1851
+ const deadline = Date.now() + 3000
1852
+
1853
+ while (Date.now() < deadline) {
1854
+ try {
1855
+ const contents = await fs.readFile(filePath, "utf8")
1856
+
1857
+ if (contents && (!expected || expected.test(contents))) return contents
1858
+ } catch (error) {
1859
+ if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
1860
+ }
1861
+ await new Promise((resolve) => setTimeout(resolve, 10))
1862
+ }
1863
+ throw new Error(`Timed out waiting for ${filePath}`)
1864
+ }
1865
+
1866
+ /**
1867
+ * @param {{client: GuardianClient, root: string, token: string}} fixture - Guardian fixture.
1868
+ * @param {Record<string, import("../src/json.js").JsonValue>} authority - Committed authority.
1869
+ */
1870
+ async function reconnectAndShutdownGuardian(fixture, authority) {
1871
+ const cleanup = new GuardianClient({pid: fixture.client.pid, socketPath: fixture.client.socketPath, token: fixture.token})
1872
+
1873
+ try {
1874
+ await cleanup.connect()
1875
+ await cleanup.claimOwner(500, authority)
1876
+ await cleanup.shutdown()
1877
+ } finally {
1878
+ cleanup.disconnect()
1879
+ await fixture.client.guardianExit().catch(() => {})
1880
+ await fs.rm(fixture.root, {force: true, recursive: true})
1881
+ }
1882
+ }
1883
+
527
1884
  /**
528
1885
  * @param {string} id - Process id.
529
- * @returns {Parameters<GuardianClient["process"]>[1]} Managed process definition.
1886
+ * @returns {Parameters<GuardianClient["process"]>[1] & import("../src/managed-process.js").ManagedProcessDefinition} Managed process definition.
530
1887
  */
531
1888
  function definition(id) {
532
1889
  return {
@@ -536,6 +1893,7 @@ function definition(id) {
536
1893
  id,
537
1894
  lifecycle: {drainTimeoutMs: 0},
538
1895
  logger: () => {},
1896
+ memory: undefined,
539
1897
  outputLines: 10,
540
1898
  restart: {backoffFactor: 1, maxDelayMs: 0, maxRestarts: 0, windowMs: 0},
541
1899
  restartDelayMs: 0,