rollbridge 0.1.39 → 0.1.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,13 +9,16 @@ import path from "node:path"
9
9
  import test from "node:test"
10
10
  import {fileURLToPath} from "node:url"
11
11
  import GuardianClient from "../src/guardian-client.js"
12
+ import {waitForProcessExit} from "./support/process.js"
12
13
 
13
14
  const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
15
+ const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
14
16
 
15
17
  test("guardian bootstrap capability is absent from process argv", async () => {
16
18
  const fixture = await createGuardian()
17
19
 
18
20
  try {
21
+ assert.deepEqual(await fixture.client.capabilities(), {daemonRecovery: 1})
19
22
  const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
20
23
  const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
21
24
  const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
@@ -67,6 +70,25 @@ test("guardian runs a strict activation lifecycle command for the exact register
67
70
  }
68
71
  })
69
72
 
73
+ test("guardian atomically updates process provenance with private owner state", async () => {
74
+ const fixture = await createGuardian()
75
+ const processInstance = fixture.client.process("service", definition("service"))
76
+ const previousOwnerState = {authority: null, serviceReleaseIds: {service: "v1"}}
77
+ const nextOwnerState = {authority: null, serviceReleaseIds: {service: "v2"}}
78
+
79
+ try {
80
+ await fixture.client.publishOwnerState(previousOwnerState)
81
+ await processInstance.start()
82
+ const previousProvenance = (await fixture.client.inventory())[0]?.provenance
83
+
84
+ await processInstance.updateDefinition({...definition("service"), env: {RELEASE: "v2"}}, nextOwnerState)
85
+ assert.deepEqual(await fixture.client.ownerState(), nextOwnerState)
86
+ assert.notEqual((await fixture.client.inventory())[0]?.provenance, previousProvenance)
87
+ } finally {
88
+ await cleanupGuardian(fixture)
89
+ }
90
+ })
91
+
70
92
  test("guardian forwards each retained output line to its exact process proxy", async () => {
71
93
  const fixture = await createGuardian()
72
94
  const marker = "guardian-output-ready"
@@ -75,19 +97,102 @@ test("guardian forwards each retained output line to its exact process proxy", a
75
97
  command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`console.log(${JSON.stringify(marker)})`)}`
76
98
  })
77
99
  const logged = once(processInstance, "log")
100
+ const forwarded = fixture.client.waitForEvent("process-log")
78
101
  const exitedFirst = once(processInstance, "exit").then(() => { throw new Error("Guardian process exited before forwarding retained output") })
79
102
 
80
103
  try {
81
104
  await processInstance.start()
82
105
  const [entry] = await Promise.race([logged, exitedFirst])
106
+ const event = await forwarded
83
107
 
84
108
  assert.equal(entry.line, marker)
109
+ assert.equal(event.status, undefined, "log events must not resend the complete retained process status")
85
110
  assert.ok(processInstance.status().logs.some((candidate) => candidate.line === marker))
86
111
  } finally {
87
112
  await cleanupGuardian(fixture)
88
113
  }
89
114
  })
90
115
 
116
+ test("guardian delivers the final process status after dropping logs for a backpressured client", async () => {
117
+ const fixture = await createGuardian()
118
+ const gatePath = path.join(fixture.root, "write-output")
119
+ const script = `const fs = require("node:fs"); const {once} = require("node:events"); (async () => { while (!fs.existsSync(${JSON.stringify(gatePath)})) await new Promise((resolve) => setTimeout(resolve, 5)); const line = "x".repeat(1024) + "\\n"; for (let index = 0; index < 8192; index += 1) if (!process.stdout.write(line)) await once(process.stdout, "drain"); })()`
120
+ const processInstance = fixture.client.process("backpressured-output", {
121
+ ...definition("backpressured-output"),
122
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(script)}`
123
+ })
124
+ const socket = fixture.client.socket
125
+
126
+ assert.ok(socket)
127
+ try {
128
+ await processInstance.start()
129
+ const pid = processInstance.status().pid
130
+
131
+ assert.ok(pid)
132
+ const finalStatus = fixture.client.waitForEvent("process")
133
+
134
+ socket.pause()
135
+ await fs.writeFile(gatePath, "write\n")
136
+ await waitForProcessExit(pid, 10000)
137
+ socket.resume()
138
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
139
+ let timeout
140
+
141
+ try {
142
+ await Promise.race([
143
+ finalStatus,
144
+ new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Guardian did not flush the final process status after backpressure")), 3000) })
145
+ ])
146
+ } finally {
147
+ clearTimeout(timeout)
148
+ }
149
+ assert.equal(processInstance.status().state, "failed")
150
+ } finally {
151
+ socket.resume()
152
+ await cleanupGuardian(fixture)
153
+ }
154
+ })
155
+
156
+ test("guardian resynchronizes retained logs after dropping output for a backpressured client", async () => {
157
+ const fixture = await createGuardian()
158
+ const gatePath = path.join(fixture.root, "write-retained-output")
159
+ const completedPath = path.join(fixture.root, "retained-output-complete")
160
+ const payload = "x".repeat(1024)
161
+ const finalLine = `8191:${payload}`
162
+ const script = `const fs = require("node:fs"); const {once} = require("node:events"); (async () => { while (!fs.existsSync(${JSON.stringify(gatePath)})) await new Promise((resolve) => setTimeout(resolve, 5)); for (let index = 0; index < 8192; index += 1) if (!process.stdout.write(index + ":" + ${JSON.stringify(payload)} + "\\n")) await once(process.stdout, "drain"); fs.writeFileSync(${JSON.stringify(completedPath)}, "complete\\n"); setInterval(() => {}, 1000); })()`
163
+ const processInstance = fixture.client.process("backpressured-retained-output", {
164
+ ...definition("backpressured-retained-output"),
165
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(script)}`
166
+ })
167
+ const socket = fixture.client.socket
168
+
169
+ assert.ok(socket)
170
+ try {
171
+ await processInstance.start()
172
+ socket.pause()
173
+ await fs.writeFile(gatePath, "write\n")
174
+ await waitForFileText(completedPath)
175
+ const resynchronized = fixture.client.waitForEvent("status")
176
+
177
+ socket.resume()
178
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
179
+ let timeout
180
+
181
+ try {
182
+ await Promise.race([
183
+ resynchronized,
184
+ new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Guardian did not resynchronize retained output after backpressure")), 3000) })
185
+ ])
186
+ } finally {
187
+ clearTimeout(timeout)
188
+ }
189
+ assert.equal(processInstance.status().logs.at(-1)?.line, finalLine)
190
+ } finally {
191
+ socket.resume()
192
+ await cleanupGuardian(fixture)
193
+ }
194
+ })
195
+
91
196
  test("guardian shutdown reports an exact owned process stop failure", async () => {
92
197
  const fixture = await createGuardian()
93
198
  const processInstance = fixture.client.process("broken-stop", {...definition("broken-stop"), stopSignal: "NOT_A_SIGNAL"})
@@ -153,6 +258,403 @@ test("successful guardian shutdown closes an authenticated waiting contender bef
153
258
  }
154
259
  })
155
260
 
261
+ test("guardian restart uses the latest accepted command and exact environment", async () => {
262
+ const fixture = await createGuardian()
263
+ const markerPath = path.join(fixture.root, "restarts.jsonl")
264
+ const authority = {configDigest: "same", runtime: null}
265
+ const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
266
+
267
+ try {
268
+ await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "old", 250))
269
+ fixture.client.disconnect()
270
+ await replacement.connect()
271
+ await replacement.claimOwner(250, authority)
272
+ await replacement.publishOwnerState(recoveryOwnerState(authority, markerPath, "new", 40))
273
+ replacement.disconnect()
274
+
275
+ const [restart] = await waitForRestartRecords(markerPath, 1)
276
+
277
+ assert.deepEqual({home: restart.home, marker: restart.marker}, {home: null, marker: "new"})
278
+ } finally {
279
+ await reconnectAndShutdownGuardian(fixture, authority)
280
+ }
281
+ })
282
+
283
+ test("guardian rearms recovery after an ownerless replacement aborts", async () => {
284
+ const fixture = await createGuardian()
285
+ const markerPath = path.join(fixture.root, "replacement-abort.jsonl")
286
+ const authority = {configDigest: "same", runtime: null}
287
+ const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
288
+
289
+ try {
290
+ await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "accepted", 40))
291
+ fixture.client.disconnect()
292
+ await replacement.connect()
293
+ const prepared = await replacement.prepareOwnerReplacement(authority, authority)
294
+
295
+ await new Promise((resolve) => setTimeout(resolve, 80))
296
+ await replacement.abortOwnerReplacement(prepared.replacementId)
297
+ assert.deepEqual((await waitForRestartRecords(markerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "accepted"}])
298
+ } finally {
299
+ replacement.disconnect()
300
+ await reconnectAndShutdownGuardian(fixture, authority)
301
+ }
302
+ })
303
+
304
+ test("guardian retry backoff remains nonzero when reconnect grace is zero", async () => {
305
+ const fixture = await createGuardian()
306
+ const markerPath = path.join(fixture.root, "retry-backoff.jsonl")
307
+ const authority = {configDigest: "same", runtime: null}
308
+
309
+ try {
310
+ await fixture.client.publishOwnerState(recoveryOwnerState(authority, markerPath, "retry", 0))
311
+ fixture.client.disconnect()
312
+ const records = await waitForRestartRecords(markerPath, 2)
313
+
314
+ assert.equal(typeof records[0]?.at, "number")
315
+ assert.equal(typeof records[1]?.at, "number")
316
+ assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `failed owner recovery retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
317
+ } finally {
318
+ await reconnectAndShutdownGuardian(fixture, authority)
319
+ }
320
+ })
321
+
322
+ test("a retired guardian-started owner does not block recovery of its replacement", async () => {
323
+ const fixture = await createGuardian()
324
+ const firstMarkerPath = path.join(fixture.root, "first-owner.pid")
325
+ const secondMarkerPath = path.join(fixture.root, "second-owner.jsonl")
326
+ const authority = {configDigest: "same", runtime: null}
327
+ const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
328
+ let firstOwnerPid
329
+
330
+ try {
331
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, firstMarkerPath))
332
+ fixture.client.disconnect()
333
+ firstOwnerPid = Number((await waitForFileText(firstMarkerPath)).trim())
334
+ await replacement.connect()
335
+ const prepared = await replacement.prepareOwnerReplacement(authority, authority)
336
+ const committed = replacement.waitForEvent("replacement-committed")
337
+
338
+ await replacement.stageOwnerReplacement(prepared.replacementId, recoveryOwnerState(authority, secondMarkerPath, "replacement", 20))
339
+ process.kill(firstOwnerPid, "SIGUSR1")
340
+ await committed
341
+ replacement.disconnect()
342
+
343
+ assert.deepEqual((await waitForRestartRecords(secondMarkerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "replacement"}])
344
+ } finally {
345
+ replacement.disconnect()
346
+ if (firstOwnerPid) {
347
+ try { process.kill(firstOwnerPid, "SIGKILL") } catch (_error) { /* Exact retired fixture owner already exited. */ }
348
+ }
349
+ await reconnectAndShutdownGuardian(fixture, authority)
350
+ }
351
+ })
352
+
353
+ test("guardian terminates a restart attempt that never claims ownership", async () => {
354
+ const fixture = await createGuardian()
355
+ const markerPath = path.join(fixture.root, "hung-owner.pid")
356
+ const authority = {configDigest: "same", runtime: null}
357
+ let hungPid
358
+
359
+ try {
360
+ await fixture.client.publishOwnerState({
361
+ authority,
362
+ recovery: {
363
+ command: {
364
+ args: ["-e", "require('node:fs').writeFileSync(process.argv[1], String(process.pid)); setInterval(() => {}, 1000)", markerPath],
365
+ cwd: fixture.root,
366
+ env: {},
367
+ executable: process.execPath
368
+ },
369
+ reconnectGraceMs: 0,
370
+ startupTimeoutMs: 1000
371
+ },
372
+ snapshot: {activeReleaseId: null}
373
+ })
374
+ fixture.client.disconnect()
375
+ hungPid = Number((await waitForFileText(markerPath)).trim())
376
+
377
+ await waitForProcessExit(hungPid)
378
+ } finally {
379
+ if (hungPid) {
380
+ try { process.kill(hungPid, "SIGKILL") } catch (_error) { /* Exact hung fixture owner already exited. */ }
381
+ }
382
+ await reconnectAndShutdownGuardian(fixture, authority)
383
+ }
384
+ })
385
+
386
+ test("guardian terminates descendants when a restart leader exits before claiming ownership", async () => {
387
+ const fixture = await createGuardian()
388
+ const descendantPath = path.join(fixture.root, "early-exit-descendant.pid")
389
+ const authority = {configDigest: "same", runtime: null}
390
+ let descendantPid
391
+
392
+ try {
393
+ await fixture.client.publishOwnerState({
394
+ authority,
395
+ recovery: {
396
+ command: {
397
+ args: ["-e", `const {spawn} = require("node:child_process"); const fs = require("node:fs"); const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {stdio: "ignore"}); fs.writeFileSync(${JSON.stringify(descendantPath)}, String(child.pid)); child.unref()`],
398
+ cwd: fixture.root,
399
+ env: {},
400
+ executable: process.execPath
401
+ },
402
+ reconnectGraceMs: 0,
403
+ startupTimeoutMs: 1000
404
+ },
405
+ snapshot: {activeReleaseId: null}
406
+ })
407
+ fixture.client.disconnect()
408
+ descendantPid = Number((await waitForFileText(descendantPath)).trim())
409
+
410
+ await waitForProcessExit(descendantPid)
411
+ } finally {
412
+ if (descendantPid) {
413
+ try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
414
+ }
415
+ await reconnectAndShutdownGuardian(fixture, authority)
416
+ }
417
+ })
418
+
419
+ test("guardian terminates a restarted owner process group that claims but never becomes ready", async () => {
420
+ const fixture = await createGuardian()
421
+ const descendantPath = path.join(fixture.root, "claimed-hung-descendant.pid")
422
+ const markerPath = path.join(fixture.root, "claimed-hung-owner.pid")
423
+ const authority = {configDigest: "same", runtime: null}
424
+ let descendantPid
425
+ let ownerPid
426
+
427
+ try {
428
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
429
+ descendantPath,
430
+ ready: false,
431
+ startupTimeoutMs: 100
432
+ }))
433
+ fixture.client.disconnect()
434
+ ownerPid = Number((await waitForFileText(markerPath)).trim())
435
+ descendantPid = Number((await waitForFileText(descendantPath)).trim())
436
+
437
+ await Promise.all([waitForProcessExit(ownerPid), waitForProcessExit(descendantPid)])
438
+ } finally {
439
+ if (ownerPid) killExactProcessGroup(ownerPid)
440
+ if (descendantPid) {
441
+ try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
442
+ }
443
+ await reconnectAndShutdownGuardian(fixture, authority)
444
+ }
445
+ })
446
+
447
+ test("guardian backs off when a restarted owner exits after claiming but before readiness", async () => {
448
+ const fixture = await createGuardian()
449
+ const descendantPath = path.join(fixture.root, "post-claim-descendant.pid")
450
+ const markerPath = path.join(fixture.root, "post-claim-exit.pid")
451
+ const startedLogPath = path.join(fixture.root, "post-claim-starts.jsonl")
452
+ const authority = {configDigest: "same", runtime: null}
453
+ let descendantPid
454
+
455
+ try {
456
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
457
+ descendantPath,
458
+ exitAfterClaim: true,
459
+ startedLogPath
460
+ }))
461
+ fixture.client.disconnect()
462
+ descendantPid = Number((await waitForFileText(descendantPath)).trim())
463
+ const records = await waitForRestartRecords(startedLogPath, 2)
464
+
465
+ await waitForProcessExit(descendantPid)
466
+ assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `post-claim failure retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
467
+ } finally {
468
+ if (descendantPid) {
469
+ try { process.kill(descendantPid, "SIGKILL") } catch (_error) { /* Exact descendant already exited. */ }
470
+ }
471
+ await reconnectAndShutdownGuardian(fixture, authority)
472
+ }
473
+ })
474
+
475
+ test("guardian preserves restart backoff when an unready owner disconnect aborts a prepared replacement", async () => {
476
+ const fixture = await createGuardian()
477
+ const markerPath = path.join(fixture.root, "prepared-post-claim-exit.pid")
478
+ const startedLogPath = path.join(fixture.root, "prepared-post-claim-starts.jsonl")
479
+ const authority = {configDigest: "same", runtime: null}
480
+ const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
481
+
482
+ try {
483
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, markerPath, {
484
+ ready: false,
485
+ startedLogPath,
486
+ startupTimeoutMs: 150
487
+ }))
488
+ fixture.client.disconnect()
489
+ await waitForFileText(markerPath)
490
+ await replacement.connect()
491
+ const aborted = replacement.waitForEvent("replacement-aborted")
492
+
493
+ await replacement.prepareOwnerReplacement(authority, authority)
494
+ await aborted
495
+ const records = await waitForRestartRecords(startedLogPath, 2)
496
+
497
+ assert.ok(Number(records[1].at) - Number(records[0].at) >= 900, `replacement abort retried after ${Number(records[1].at) - Number(records[0].at)}ms`)
498
+ } finally {
499
+ replacement.disconnect()
500
+ await reconnectAndShutdownGuardian(fixture, authority)
501
+ }
502
+ })
503
+
504
+ test("replacement commit preserves a claimed guardian restart child through listener retirement", async () => {
505
+ const fixture = await createGuardian()
506
+ const claimedPath = path.join(fixture.root, "claimed-restart-owner.pid")
507
+ const committedPath = path.join(fixture.root, "claimed-restart-owner-committed.txt")
508
+ const preparedPath = path.join(fixture.root, "claimed-restart-owner-prepared.txt")
509
+ const authority = {configDigest: "old", runtime: null}
510
+ const nextAuthority = {configDigest: "new", runtime: null}
511
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
512
+ /** @type {number | undefined} */
513
+ let recoveredOwnerPid
514
+
515
+ try {
516
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, claimedPath, {
517
+ ready: false,
518
+ replacementCommittedPath: committedPath,
519
+ replacementPreparedPath: preparedPath,
520
+ startupTimeoutMs: 5000
521
+ }))
522
+ fixture.client.disconnect()
523
+ const incumbentPid = Number((await waitForFileText(claimedPath)).trim())
524
+
525
+ recoveredOwnerPid = incumbentPid
526
+ await candidate.connect()
527
+ const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
528
+
529
+ await waitForFileText(preparedPath, new RegExp(prepared.replacementId))
530
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: null}})
531
+ const published = candidate.waitForEvent("replacement-committed")
532
+
533
+ process.kill(incumbentPid, "SIGUSR2")
534
+ await Promise.race([waitForFileText(committedPath, new RegExp(prepared.replacementId)), published])
535
+ await new Promise((resolve) => setTimeout(resolve, 50))
536
+ assert.doesNotThrow(() => process.kill(incumbentPid, 0), "claimed incumbent must survive replacement commit until its listeners retire")
537
+
538
+ process.kill(incumbentPid, "SIGUSR1")
539
+ await published
540
+ await candidate.shutdown()
541
+ await fixture.client.guardianExit()
542
+ } finally {
543
+ candidate.disconnect()
544
+ if (recoveredOwnerPid) {
545
+ try { process.kill(-recoveredOwnerPid, "SIGKILL") } catch (_error) { /* Exact recovered fixture owner already exited. */ }
546
+ }
547
+ await cleanupGuardian(fixture)
548
+ }
549
+ })
550
+
551
+ test("ownerless replacement commit kills a superseded restart candidate", async () => {
552
+ const fixture = await createGuardian()
553
+ const delayedClaimPath = path.join(fixture.root, "delayed-claim.pid")
554
+ const firstMarkerPath = path.join(fixture.root, "delayed-owner-started.pid")
555
+ const secondMarkerPath = path.join(fixture.root, "committed-replacement.jsonl")
556
+ const authority = {configDigest: "old", runtime: null}
557
+ const nextAuthority = {configDigest: "new", runtime: null}
558
+ const replacement = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
559
+ let delayedOwnerPid
560
+ let staged = false
561
+
562
+ try {
563
+ await fixture.client.publishOwnerState(claimingRecoveryOwnerState(fixture, authority, delayedClaimPath, {claimDelayMs: 500, startedPath: firstMarkerPath}))
564
+ fixture.client.disconnect()
565
+ delayedOwnerPid = Number((await waitForFileText(firstMarkerPath)).trim())
566
+ await replacement.connect()
567
+ const prepared = await replacement.prepareOwnerReplacement(authority, nextAuthority)
568
+
569
+ await waitForProcessExit(delayedOwnerPid)
570
+ const committed = replacement.waitForEvent("replacement-committed")
571
+
572
+ await replacement.stageOwnerReplacement(prepared.replacementId, recoveryOwnerState(nextAuthority, secondMarkerPath, "committed", 20))
573
+ staged = true
574
+ await committed
575
+ replacement.disconnect()
576
+
577
+ assert.deepEqual((await waitForRestartRecords(secondMarkerPath, 1)).map(({home, marker}) => ({home, marker})), [{home: null, marker: "committed"}])
578
+ } finally {
579
+ replacement.disconnect()
580
+ if (delayedOwnerPid) {
581
+ try { process.kill(delayedOwnerPid, "SIGKILL") } catch (_error) { /* Exact superseded fixture owner already exited. */ }
582
+ }
583
+ await reconnectAndShutdownGuardian(fixture, staged ? nextAuthority : authority)
584
+ }
585
+ })
586
+
587
+ test("guardian logs an asynchronous daemon spawn failure before retrying", async () => {
588
+ const fixture = await createGuardian()
589
+ const logPath = path.join(fixture.root, "daemon.log")
590
+ const authority = {configDigest: "same", runtime: null}
591
+ const privateArgs = "private-argument-value"
592
+ const privateEnvironment = "private-environment-value"
593
+ const privateExecutable = path.join(fixture.root, "private-runtime", "missing-rollbridge")
594
+
595
+ try {
596
+ await fixture.client.publishOwnerState({
597
+ authority,
598
+ recovery: {
599
+ command: {
600
+ args: [privateArgs],
601
+ cwd: fixture.root,
602
+ env: {PRIVATE_ENVIRONMENT: privateEnvironment},
603
+ executable: privateExecutable,
604
+ logPath
605
+ },
606
+ reconnectGraceMs: 0,
607
+ startupTimeoutMs: 1000
608
+ },
609
+ snapshot: {activeReleaseId: null}
610
+ })
611
+ fixture.client.disconnect()
612
+ const diagnosticPattern = /"code":"ENOENT".*"message":"guardian failed to restart daemon"/
613
+ const diagnostic = await waitForFileText(logPath, diagnosticPattern)
614
+
615
+ assert.match(diagnostic, diagnosticPattern)
616
+ for (const privateValue of [privateArgs, privateEnvironment, privateExecutable, fixture.root, logPath]) {
617
+ assert.ok(!diagnostic.includes(privateValue), `guardian diagnostic exposed ${privateValue}`)
618
+ }
619
+ } finally {
620
+ await reconnectAndShutdownGuardian(fixture, authority)
621
+ }
622
+ })
623
+
624
+ test("guardian publishes the authenticated ready owner's PID file", async () => {
625
+ const fixture = await createGuardian()
626
+ const pidPath = path.join(fixture.root, "run", "daemon.pid")
627
+ const victimPath = path.join(fixture.root, "victim")
628
+ const authority = {configDigest: "same", runtime: null}
629
+
630
+ try {
631
+ await fixture.client.publishOwnerState({
632
+ authority,
633
+ recovery: {
634
+ command: {
635
+ args: [],
636
+ cwd: fixture.root,
637
+ env: {},
638
+ executable: process.execPath,
639
+ pidPath
640
+ },
641
+ reconnectGraceMs: 10,
642
+ startupTimeoutMs: 1000
643
+ },
644
+ snapshot: {activeReleaseId: null}
645
+ })
646
+ await fs.mkdir(path.dirname(pidPath), {recursive: true})
647
+ await fs.writeFile(victimPath, "unchanged\n")
648
+ await fs.symlink(victimPath, pidPath)
649
+ await fixture.client.ownerReady()
650
+
651
+ assert.equal(await fs.readFile(pidPath, "utf8"), `${process.pid}\n`)
652
+ assert.equal(await fs.readFile(victimPath, "utf8"), "unchanged\n")
653
+ } finally {
654
+ await cleanupGuardian(fixture)
655
+ }
656
+ })
657
+
156
658
  test("replacement commit notification waits for incumbent listener retirement", async () => {
157
659
  const fixture = await createGuardian()
158
660
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
@@ -170,6 +672,10 @@ test("replacement commit notification waits for incumbent listener retirement",
170
672
  await fixture.client.commitOwnerReplacement(prepared.replacementId)
171
673
  await new Promise((resolve) => setImmediate(resolve))
172
674
  assert.equal(committed, false, "candidate publication must remain fenced while incumbent listener retirement is delayed")
675
+ const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
676
+
677
+ await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
678
+ await listenersRetired
173
679
  await fixture.client.request({command: "finalize-owner-replacement", replacementId: prepared.replacementId})
174
680
  await notification
175
681
  assert.equal(committed, true)
@@ -181,6 +687,40 @@ test("replacement commit notification waits for incumbent listener retirement",
181
687
  }
182
688
  })
183
689
 
690
+ test("completed direct listener retirement finalizes when the incumbent disconnects", {timeout: 3000}, async () => {
691
+ const fixture = await createGuardian()
692
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
693
+ const authority = {configDigest: "incumbent", runtime: null}
694
+ const nextAuthority = {configDigest: "candidate", runtime: null}
695
+
696
+ try {
697
+ await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v1"}})
698
+ await candidate.connect()
699
+ const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
700
+
701
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})
702
+ const committed = candidate.waitForEvent("replacement-committed")
703
+
704
+ await fixture.client.commitOwnerReplacement(prepared.replacementId)
705
+ await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
706
+ fixture.client.disconnect()
707
+ await committed
708
+ await candidate.finalizeOwnerReplacement(prepared.replacementId)
709
+ assert.deepEqual(await candidate.replacementStatus(), {
710
+ committedReplacementId: prepared.replacementId,
711
+ ownerClaimed: true,
712
+ retirementFailed: false,
713
+ retirementPending: false,
714
+ retirementReady: false
715
+ })
716
+ await candidate.shutdown()
717
+ await fixture.client.guardianExit()
718
+ } finally {
719
+ candidate.disconnect()
720
+ await cleanupGuardian(fixture)
721
+ }
722
+ })
723
+
184
724
  test("replacement staging rejects owner state published after prepare", async () => {
185
725
  const fixture = await createGuardian()
186
726
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
@@ -210,6 +750,115 @@ test("replacement staging rejects owner state published after prepare", async ()
210
750
  }
211
751
  })
212
752
 
753
+ test("staged replacement receives cleared local sources when the incumbent disconnects", {timeout: 3000}, async () => {
754
+ const fixture = await createGuardian()
755
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
756
+ const authority = {configDigest: "incumbent", runtime: null}
757
+ const nextAuthority = {configDigest: "candidate", runtime: null}
758
+ const snapshot = {activeReleaseId: "v1"}
759
+
760
+ try {
761
+ await fixture.client.publishOwnerState({
762
+ authority,
763
+ listenerConnectionSources: {"incumbent-local": {v1: {http: 0, websocket: 1}}},
764
+ listenerSourceId: "incumbent-local",
765
+ snapshot
766
+ })
767
+ await candidate.connect()
768
+ const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
769
+
770
+ await candidate.stageOwnerReplacement(prepared.replacementId, {
771
+ authority: nextAuthority,
772
+ listenerConnectionSources: {"incumbent-local": {v1: {http: 0, websocket: 1}}},
773
+ listenerSourceId: "candidate-local",
774
+ snapshot
775
+ })
776
+ const cleared = candidate.waitForEvent("owner-connection-state")
777
+ const committed = candidate.waitForEvent("replacement-committed")
778
+
779
+ fixture.client.disconnect()
780
+ assert.deepEqual(await cleared, {
781
+ connections: {http: 0, websocket: 0},
782
+ event: "owner-connection-state",
783
+ releaseId: "v1",
784
+ sourceId: "incumbent-local"
785
+ })
786
+ await committed
787
+ const state = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await candidate.ownerState())
788
+
789
+ assert.deepEqual(state.listenerConnectionSources, {})
790
+ await candidate.shutdown()
791
+ await fixture.client.guardianExit()
792
+ } finally {
793
+ candidate.disconnect()
794
+ await cleanupGuardian(fixture)
795
+ }
796
+ })
797
+
798
+ test("staged successor state receives tombstones when an older completed listener disconnects", {timeout: 3000}, async () => {
799
+ const fixture = await createGuardian()
800
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
801
+ const successor = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
802
+ const authority = {configDigest: "owner", runtime: null}
803
+ const snapshot = {activeReleaseId: "v1"}
804
+
805
+ try {
806
+ await fixture.client.publishOwnerState({authority, snapshot})
807
+ await candidate.connect()
808
+ const first = await candidate.prepareOwnerReplacement(authority, authority)
809
+
810
+ await candidate.stageOwnerReplacement(first.replacementId, {authority, listenerSourceId: "candidate-local", snapshot})
811
+ const firstCommitted = candidate.waitForEvent("replacement-committed")
812
+
813
+ await fixture.client.commitOwnerReplacement(first.replacementId)
814
+ const sourcePublished = candidate.waitForEvent("owner-connection-state")
815
+
816
+ await fixture.client.publishOwnerConnectionState(first.replacementId, "retired-local", "v1", {http: 0, websocket: 1}, true)
817
+ await sourcePublished
818
+ const firstListenersRetired = candidate.waitForEvent("replacement-listeners-retired")
819
+
820
+ await fixture.client.completeOwnerListenerRetirement(first.replacementId)
821
+ await firstListenersRetired
822
+ await candidate.finalizeOwnerReplacement(first.replacementId)
823
+ await firstCommitted
824
+
825
+ await successor.connect()
826
+ const second = await successor.prepareOwnerReplacement(authority, authority)
827
+
828
+ await successor.stageOwnerReplacement(second.replacementId, {
829
+ authority,
830
+ listenerConnectionSources: {"retired-local": {v1: {http: 0, websocket: 1}}},
831
+ listenerSourceId: "successor-local",
832
+ snapshot
833
+ })
834
+ const sourceCleared = candidate.waitForEvent("owner-connection-state")
835
+ const stagedSourceCleared = successor.waitForEvent("owner-connection-state")
836
+
837
+ fixture.client.disconnect()
838
+ const tombstone = {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "retired-local"}
839
+
840
+ assert.deepEqual(await Promise.all([sourceCleared, stagedSourceCleared]), [tombstone, tombstone])
841
+
842
+ await candidate.commitOwnerReplacement(second.replacementId)
843
+ const successorState = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await successor.ownerState())
844
+
845
+ assert.deepEqual(successorState.listenerConnectionSources, {})
846
+ const secondCommitted = successor.waitForEvent("replacement-committed")
847
+ const secondListenersRetired = successor.waitForEvent("replacement-listeners-retired")
848
+
849
+ await candidate.completeOwnerListenerRetirement(second.replacementId)
850
+ await secondListenersRetired
851
+ await successor.finalizeOwnerReplacement(second.replacementId)
852
+ await secondCommitted
853
+ await successor.shutdown()
854
+ await fixture.client.guardianExit()
855
+ } finally {
856
+ successor.disconnect()
857
+ candidate.disconnect()
858
+ await cleanupGuardian(fixture)
859
+ }
860
+ })
861
+
213
862
  test("replacement abort notifies both the candidate and committed owner", async () => {
214
863
  const fixture = await createGuardian()
215
864
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
@@ -298,76 +947,365 @@ test("guardian owner-replacement capability classification is explicit and fail
298
947
  }
299
948
  })
300
949
 
301
- test("reserved process recovery rejects a reconstructed definition with different provenance", async () => {
950
+ test("reserved process recovery rejects a reconstructed definition with different provenance", async () => {
951
+ const fixture = await createGuardian()
952
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
953
+ const processKey = "release:v1:worker"
954
+
955
+ try {
956
+ await fixture.client.process(processKey, definition("worker")).recover()
957
+ const [registration] = await fixture.client.inventory()
958
+
959
+ assert.ok(registration)
960
+ await candidate.connect()
961
+ candidate.reserveProcessRecovery(processKey, registration.provenance)
962
+ await assert.rejects(
963
+ () => candidate.process(processKey, definition("different-worker")).recover(),
964
+ /provenance mismatch for reserved process/
965
+ )
966
+ assert.deepEqual((await fixture.client.inventory()).map(({key}) => key), [processKey])
967
+ } finally {
968
+ candidate.disconnect()
969
+ await cleanupGuardian(fixture)
970
+ }
971
+ })
972
+
973
+ test("retired owner replacement rejects a registered process absent from committed owner state", async () => {
974
+ const fixture = await createGuardian()
975
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
976
+ const committedProcessKey = "release:v1:worker"
977
+ const candidateProcessKey = "release:candidate:worker"
978
+ const authority = {configDigest: "owner", runtime: null}
979
+ const snapshot = {
980
+ activeReleaseId: "v1",
981
+ control: {path: path.join(fixture.root, "rollbridge.sock")},
982
+ releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
983
+ services: [],
984
+ singletons: []
985
+ }
986
+ const candidateSnapshot = {
987
+ ...snapshot,
988
+ releases: [...snapshot.releases, {processes: [{id: "worker"}], releaseId: "candidate"}]
989
+ }
990
+
991
+ try {
992
+ await fixture.client.process(committedProcessKey, definition("worker")).recover()
993
+ await fixture.client.process(candidateProcessKey, definition("candidate-worker")).recover()
994
+ await fixture.client.publishOwnerState({authority, snapshot})
995
+ await candidate.connect()
996
+ const prepared = await candidate.prepareOwnerReplacement(authority, authority)
997
+
998
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot: candidateSnapshot})
999
+ await assert.rejects(
1000
+ () => candidate.commitRetiredOwnerReplacement(prepared.replacementId, candidateProcessKey),
1001
+ /process .* does not belong to the committed owner/
1002
+ )
1003
+ } finally {
1004
+ candidate.disconnect()
1005
+ await cleanupGuardian(fixture)
1006
+ }
1007
+ })
1008
+
1009
+ test("retired owner replacement requires unchanged authority and the exact control path absent", async () => {
1010
+ const fixture = await createGuardian()
1011
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1012
+ const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1013
+ const controlPath = path.join(fixture.root, "rollbridge.sock")
1014
+ const processKey = "release:v1:worker"
1015
+ const authority = {configDigest: "incumbent", runtime: null}
1016
+ const nextAuthority = {configDigest: "candidate", runtime: null}
1017
+ const snapshot = {
1018
+ activeReleaseId: "v1",
1019
+ control: {path: controlPath},
1020
+ releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
1021
+ services: [],
1022
+ singletons: []
1023
+ }
1024
+
1025
+ try {
1026
+ await fixture.client.process(processKey, definition("worker")).recover()
1027
+ await fixture.client.publishOwnerState({authority, snapshot})
1028
+ await candidate.connect()
1029
+ const changed = await candidate.prepareOwnerReplacement(authority, nextAuthority)
1030
+
1031
+ await candidate.stageOwnerReplacement(changed.replacementId, {authority: nextAuthority, snapshot})
1032
+ await assert.rejects(() => candidate.commitRetiredOwnerReplacement(changed.replacementId, processKey), /unchanged owner authority/)
1033
+ await candidate.abortOwnerReplacement(changed.replacementId)
1034
+
1035
+ const occupied = await candidate.prepareOwnerReplacement(authority, authority)
1036
+
1037
+ await candidate.stageOwnerReplacement(occupied.replacementId, {authority, snapshot})
1038
+ await fs.writeFile(controlPath, "occupied\n")
1039
+ await assert.rejects(() => candidate.commitRetiredOwnerReplacement(occupied.replacementId, processKey), /control socket .* still exists/)
1040
+ await candidate.abortOwnerReplacement(occupied.replacementId)
1041
+
1042
+ await fs.rm(controlPath)
1043
+ const ready = await candidate.prepareOwnerReplacement(authority, authority)
1044
+
1045
+ await candidate.stageOwnerReplacement(ready.replacementId, {authority, snapshot})
1046
+ await contender.connect()
1047
+ await assert.rejects(
1048
+ () => contender.request({command: "commit-retired-owner-replacement", key: processKey, replacementId: ready.replacementId}),
1049
+ /not the prepared candidate/
1050
+ )
1051
+ await assert.rejects(
1052
+ () => candidate.commitRetiredOwnerReplacement("stale-replacement", processKey),
1053
+ /not the prepared candidate/
1054
+ )
1055
+ await assert.rejects(
1056
+ () => candidate.commitRetiredOwnerReplacement(ready.replacementId, "release:v1:wrong"),
1057
+ /process .* is not registered/
1058
+ )
1059
+ const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
1060
+
1061
+ void handoffRequested.catch(() => undefined)
1062
+
1063
+ await candidate.prepareRetiredOwnerListenerHandoff(ready.replacementId, processKey)
1064
+ await handoffRequested
1065
+ await assert.rejects(
1066
+ () => contender.prepareOwnerReplacement(authority, authority),
1067
+ /listener retirement is pending/
1068
+ )
1069
+ const committed = candidate.waitForEvent("replacement-committed")
1070
+ const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
1071
+ const connectionState = candidate.waitForEvent("owner-connection-state")
1072
+
1073
+ await fixture.client.publishOwnerConnectionState(ready.replacementId, "listener-a", "v1", {http: 1, websocket: 2}, true)
1074
+ assert.deepEqual(await connectionState, {connections: {http: 1, websocket: 2}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
1075
+ await fixture.client.completeOwnerListenerRetirement(ready.replacementId)
1076
+ await listenersRetired
1077
+ const retirementRequested = fixture.client.waitForEvent("replacement-retirement-requested")
1078
+
1079
+ await candidate.commitRetiredOwnerReplacement(ready.replacementId, processKey)
1080
+ await retirementRequested
1081
+ await candidate.finalizeOwnerReplacement(ready.replacementId)
1082
+ await committed
1083
+ await candidate.shutdown()
1084
+ await fixture.client.guardianExit()
1085
+ } finally {
1086
+ contender.disconnect()
1087
+ candidate.disconnect()
1088
+ await cleanupGuardian(fixture)
1089
+ }
1090
+ })
1091
+
1092
+ test("completed listener retirement survives owner recovery and clears a crashed local source", async () => {
1093
+ const fixture = await createGuardian()
1094
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1095
+ const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1096
+ const controlPath = path.join(fixture.root, "rollbridge.sock")
1097
+ const processKey = "release:v1:worker"
1098
+ const authority = {configDigest: "owner", runtime: null}
1099
+ const snapshot = {
1100
+ activeReleaseId: "v1",
1101
+ control: {path: controlPath},
1102
+ releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
1103
+ services: [],
1104
+ singletons: []
1105
+ }
1106
+
1107
+ try {
1108
+ await fixture.client.process(processKey, definition("worker")).recover()
1109
+ await fixture.client.publishOwnerState({authority, snapshot})
1110
+ await candidate.connect()
1111
+ const prepared = await candidate.prepareOwnerReplacement(authority, authority)
1112
+
1113
+ await candidate.stageOwnerReplacement(prepared.replacementId, {
1114
+ authority,
1115
+ listenerConnectionSources: {"candidate-local": {v1: {http: 0, websocket: 1}}},
1116
+ listenerSourceId: "candidate-local",
1117
+ snapshot
1118
+ })
1119
+ const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
1120
+
1121
+ void handoffRequested.catch(() => undefined)
1122
+
1123
+ await candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
1124
+ await handoffRequested
1125
+ const initial = candidate.waitForEvent("owner-connection-state")
1126
+
1127
+ await fixture.client.publishOwnerConnectionState(prepared.replacementId, "listener-a", "v1", {http: 0, websocket: 1}, true)
1128
+ assert.deepEqual(await initial, {connections: {http: 0, websocket: 1}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
1129
+ const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
1130
+
1131
+ await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
1132
+ await listenersRetired
1133
+ await candidate.commitRetiredOwnerReplacement(prepared.replacementId, processKey)
1134
+ candidate.disconnect()
1135
+ await recovered.connect()
1136
+ await recovered.claimOwner(1000, authority)
1137
+ const stateAfterCandidateCrash = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
1138
+
1139
+ assert.equal(stateAfterCandidateCrash.listenerConnectionSources?.["candidate-local"], undefined)
1140
+ assert.deepEqual(stateAfterCandidateCrash.listenerConnectionSources?.["listener-a"], {v1: {http: 0, websocket: 1}})
1141
+ assert.deepEqual(await recovered.replacementStatus(), {
1142
+ committedReplacementId: prepared.replacementId,
1143
+ ownerClaimed: true,
1144
+ retirementFailed: false,
1145
+ retirementPending: true,
1146
+ retirementReady: true
1147
+ })
1148
+ await recovered.finalizeOwnerReplacement(prepared.replacementId)
1149
+ const cleared = recovered.waitForEvent("owner-connection-state")
1150
+
1151
+ fixture.client.disconnect()
1152
+ assert.deepEqual(await Promise.race([
1153
+ cleared,
1154
+ new Promise((_, reject) => {
1155
+ const timer = setTimeout(() => reject(new Error("Recovered owner did not receive the retired source tombstone")), 500)
1156
+
1157
+ timer.unref()
1158
+ })
1159
+ ]), {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "listener-a"})
1160
+ const recoveredOwnerState = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
1161
+
1162
+ assert.deepEqual(recoveredOwnerState.listenerConnectionSources, {})
1163
+ await recovered.publishOwnerState({
1164
+ authority,
1165
+ listenerConnectionSources: {"listener-a": {v1: {http: 0, websocket: 1}}},
1166
+ listenerSourceId: "recovered-local",
1167
+ snapshot
1168
+ })
1169
+ const afterStalePublication = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
1170
+
1171
+ assert.deepEqual(afterStalePublication.listenerConnectionSources, {})
1172
+ const recoveredProcess = recovered.process(processKey, definition("worker"))
1173
+
1174
+ await recoveredProcess.recover()
1175
+ await recoveredProcess.updateDefinition({...definition("worker"), env: {REVISION: "stale"}}, {
1176
+ authority,
1177
+ listenerConnectionSources: {"listener-a": {v1: {http: 0, websocket: 1}}},
1178
+ listenerSourceId: "recovered-local",
1179
+ snapshot
1180
+ })
1181
+ const afterStaleProcessUpdate = /** @type {{listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>}} */ (await recovered.ownerState())
1182
+
1183
+ assert.deepEqual(afterStaleProcessUpdate.listenerConnectionSources, {})
1184
+ await recovered.shutdown()
1185
+ await fixture.client.guardianExit()
1186
+ } finally {
1187
+ recovered.disconnect()
1188
+ candidate.disconnect()
1189
+ await cleanupGuardian(fixture)
1190
+ }
1191
+ })
1192
+
1193
+ test("direct retired-listener source relay survives committed owner recovery", async () => {
302
1194
  const fixture = await createGuardian()
303
1195
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
304
- const processKey = "release:v1:worker"
1196
+ const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1197
+ const authority = {configDigest: "owner", runtime: null}
1198
+ const snapshot = {activeReleaseId: "v1", control: {path: path.join(fixture.root, "rollbridge.sock")}, releases: [], services: [], singletons: []}
305
1199
 
306
1200
  try {
307
- await fixture.client.process(processKey, definition("worker")).recover()
308
- const [registration] = await fixture.client.inventory()
309
-
310
- assert.ok(registration)
1201
+ await fixture.client.publishOwnerState({authority, snapshot})
311
1202
  await candidate.connect()
312
- candidate.reserveProcessRecovery(processKey, registration.provenance)
313
- await assert.rejects(
314
- () => candidate.process(processKey, definition("different-worker")).recover(),
315
- /provenance mismatch for reserved process/
316
- )
317
- assert.deepEqual((await fixture.client.inventory()).map(({key}) => key), [processKey])
1203
+ const prepared = await candidate.prepareOwnerReplacement(authority, authority)
1204
+
1205
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority, listenerSourceId: "candidate-local", snapshot})
1206
+ await fixture.client.commitOwnerReplacement(prepared.replacementId)
1207
+ const initial = candidate.waitForEvent("owner-connection-state")
1208
+
1209
+ await fixture.client.publishOwnerConnectionState(prepared.replacementId, "direct-listener", "v1", {http: 0, websocket: 1}, true)
1210
+ assert.deepEqual(await initial, {connections: {http: 0, websocket: 1}, event: "owner-connection-state", releaseId: "v1", sourceId: "direct-listener"})
1211
+ const listenersRetired = candidate.waitForEvent("replacement-listeners-retired")
1212
+
1213
+ await fixture.client.completeOwnerListenerRetirement(prepared.replacementId)
1214
+ await listenersRetired
1215
+ await fixture.client.finalizeOwnerReplacement(prepared.replacementId)
1216
+ candidate.disconnect()
1217
+ await recovered.connect()
1218
+ await recovered.claimOwner(1000, authority)
1219
+ const cleared = recovered.waitForEvent("owner-connection-state")
1220
+
1221
+ await fixture.client.publishOwnerConnectionState(prepared.replacementId, "direct-listener", "v1", {http: 0, websocket: 0}, true)
1222
+ assert.deepEqual(await cleared, {connections: {http: 0, websocket: 0}, event: "owner-connection-state", releaseId: "v1", sourceId: "direct-listener"})
1223
+ await recovered.shutdown()
1224
+ await fixture.client.guardianExit()
318
1225
  } finally {
1226
+ recovered.disconnect()
319
1227
  candidate.disconnect()
320
1228
  await cleanupGuardian(fixture)
321
1229
  }
322
1230
  })
323
1231
 
324
- test("retired owner replacement rejects a registered process absent from committed owner state", async () => {
1232
+ test("incumbent listener disconnect before state completion aborts without committing the candidate", async () => {
325
1233
  const fixture = await createGuardian()
326
1234
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
327
- const committedProcessKey = "release:v1:worker"
328
- const candidateProcessKey = "release:candidate:worker"
1235
+ const recovered = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1236
+ const controlPath = path.join(fixture.root, "rollbridge.sock")
1237
+ const processKey = "release:v1:worker"
329
1238
  const authority = {configDigest: "owner", runtime: null}
330
1239
  const snapshot = {
331
1240
  activeReleaseId: "v1",
332
- control: {path: path.join(fixture.root, "rollbridge.sock")},
1241
+ control: {path: controlPath},
333
1242
  releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
334
1243
  services: [],
335
1244
  singletons: []
336
1245
  }
337
- const candidateSnapshot = {
338
- ...snapshot,
339
- releases: [...snapshot.releases, {processes: [{id: "worker"}], releaseId: "candidate"}]
340
- }
341
1246
 
342
1247
  try {
343
- await fixture.client.process(committedProcessKey, definition("worker")).recover()
344
- await fixture.client.process(candidateProcessKey, definition("candidate-worker")).recover()
1248
+ await fixture.client.process(processKey, definition("worker")).recover()
345
1249
  await fixture.client.publishOwnerState({authority, snapshot})
346
1250
  await candidate.connect()
347
1251
  const prepared = await candidate.prepareOwnerReplacement(authority, authority)
348
1252
 
349
- await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot: candidateSnapshot})
350
- await assert.rejects(
351
- () => candidate.commitRetiredOwnerReplacement(prepared.replacementId, candidateProcessKey),
352
- /process .* does not belong to the committed owner/
353
- )
1253
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot})
1254
+ const handoffRequested = fixture.client.waitForEvent("replacement-listener-handoff-requested")
1255
+ const failed = candidate.waitForEvent("replacement-retirement-failed")
1256
+ const aborted = candidate.waitForEvent("replacement-aborted")
1257
+ let tombstones = 0
1258
+
1259
+ candidate.onEvent("owner-connection-state", (event) => {
1260
+ if (event.sourceId === "incumbent-local" && event.releaseId === "v1" && event.connections && typeof event.connections === "object" && !Array.isArray(event.connections) && event.connections.http === 0 && event.connections.websocket === 0) {
1261
+ tombstones += 1
1262
+ }
1263
+ })
1264
+
1265
+ await candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
1266
+ await handoffRequested
1267
+ const sourcePublished = candidate.waitForEvent("owner-connection-state")
1268
+
1269
+ await fixture.client.publishOwnerConnectionState(prepared.replacementId, "incumbent-local", "v1", {http: 0, websocket: 1}, true)
1270
+ await sourcePublished
1271
+ fixture.client.disconnect()
1272
+ assert.deepEqual(await failed, {
1273
+ event: "replacement-retirement-failed",
1274
+ reason: "Incumbent listener disconnected during the prepared handoff",
1275
+ replacementId: prepared.replacementId
1276
+ })
1277
+ assert.deepEqual(await aborted, {
1278
+ event: "replacement-aborted",
1279
+ reason: "Incumbent listener disconnected during the prepared handoff"
1280
+ })
1281
+ assert.equal(tombstones, 1)
1282
+ assert.deepEqual(await candidate.replacementStatus(), {
1283
+ committedReplacementId: null,
1284
+ ownerClaimed: false,
1285
+ retirementFailed: false,
1286
+ retirementPending: false,
1287
+ retirementReady: false
1288
+ })
1289
+ await recovered.connect()
1290
+ await recovered.claimOwner(1000, authority)
1291
+ await recovered.shutdown()
1292
+ await fixture.client.guardianExit()
354
1293
  } finally {
1294
+ recovered.disconnect()
355
1295
  candidate.disconnect()
356
1296
  await cleanupGuardian(fixture)
357
1297
  }
358
1298
  })
359
1299
 
360
- test("retired owner replacement requires unchanged authority and the exact control path absent", async () => {
1300
+ test("replacement abort during listener handoff validation leaves the incumbent available", async () => {
361
1301
  const fixture = await createGuardian()
362
1302
  const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
363
1303
  const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
364
- const controlPath = path.join(fixture.root, "rollbridge.sock")
365
1304
  const processKey = "release:v1:worker"
366
- const authority = {configDigest: "incumbent", runtime: null}
367
- const nextAuthority = {configDigest: "candidate", runtime: null}
1305
+ const authority = {configDigest: "owner", runtime: null}
368
1306
  const snapshot = {
369
1307
  activeReleaseId: "v1",
370
- control: {path: controlPath},
1308
+ control: {path: path.join(fixture.root, "rollbridge.sock")},
371
1309
  releases: [{processes: [{id: "worker"}], releaseId: "v1"}],
372
1310
  services: [],
373
1311
  singletons: []
@@ -377,45 +1315,68 @@ test("retired owner replacement requires unchanged authority and the exact contr
377
1315
  await fixture.client.process(processKey, definition("worker")).recover()
378
1316
  await fixture.client.publishOwnerState({authority, snapshot})
379
1317
  await candidate.connect()
380
- const changed = await candidate.prepareOwnerReplacement(authority, nextAuthority)
1318
+ const prepared = await candidate.prepareOwnerReplacement(authority, authority)
381
1319
 
382
- await candidate.stageOwnerReplacement(changed.replacementId, {authority: nextAuthority, snapshot})
383
- await assert.rejects(() => candidate.commitRetiredOwnerReplacement(changed.replacementId, processKey), /unchanged owner authority/)
384
- await candidate.abortOwnerReplacement(changed.replacementId)
1320
+ await candidate.stageOwnerReplacement(prepared.replacementId, {authority, snapshot})
1321
+ const abandonedHandoff = candidate.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
385
1322
 
386
- const occupied = await candidate.prepareOwnerReplacement(authority, authority)
1323
+ await candidate.abortOwnerReplacement(prepared.replacementId)
1324
+ await assert.rejects(() => abandonedHandoff, /prepared candidate/)
1325
+ await contender.connect()
1326
+ const fresh = await contender.prepareOwnerReplacement(authority, authority)
387
1327
 
388
- await candidate.stageOwnerReplacement(occupied.replacementId, {authority, snapshot})
389
- await fs.writeFile(controlPath, "occupied\n")
390
- await assert.rejects(() => candidate.commitRetiredOwnerReplacement(occupied.replacementId, processKey), /control socket .* still exists/)
391
- await candidate.abortOwnerReplacement(occupied.replacementId)
1328
+ await contender.abortOwnerReplacement(fresh.replacementId)
1329
+ await fixture.client.shutdown()
1330
+ await fixture.client.guardianExit()
1331
+ } finally {
1332
+ contender.disconnect()
1333
+ candidate.disconnect()
1334
+ await cleanupGuardian(fixture)
1335
+ }
1336
+ })
392
1337
 
393
- await fs.rm(controlPath)
394
- const ready = await candidate.prepareOwnerReplacement(authority, authority)
1338
+ test("explicit replacement abort notifies the incumbent owner", async () => {
1339
+ const fixture = await createGuardian()
1340
+ const candidate = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1341
+ const authority = {configDigest: "incumbent", runtime: null}
1342
+ const nextAuthority = {configDigest: "candidate", runtime: null}
395
1343
 
396
- await candidate.stageOwnerReplacement(ready.replacementId, {authority, snapshot})
1344
+ try {
1345
+ await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v1"}})
1346
+ await candidate.connect()
1347
+ const aborted = fixture.client.waitForEvent("replacement-aborted")
1348
+ const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
1349
+
1350
+ await candidate.abortOwnerReplacement(prepared.replacementId)
1351
+ await aborted
1352
+ await fixture.client.shutdown()
1353
+ await fixture.client.guardianExit()
1354
+ } finally {
1355
+ candidate.disconnect()
1356
+ await cleanupGuardian(fixture)
1357
+ }
1358
+ })
1359
+
1360
+ test("queued owner claim is revalidated against the latest committed authority", async () => {
1361
+ const fixture = await createGuardian()
1362
+ const contender = new GuardianClient({socketPath: fixture.client.socketPath, token: fixture.token})
1363
+ const authority = {configDigest: "old", runtime: null}
1364
+ const nextAuthority = {configDigest: "new", runtime: null}
1365
+
1366
+ try {
1367
+ await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: null}})
397
1368
  await contender.connect()
398
- await assert.rejects(
399
- () => contender.request({command: "commit-retired-owner-replacement", key: processKey, replacementId: ready.replacementId}),
400
- /not the prepared candidate/
401
- )
402
- await assert.rejects(
403
- () => candidate.commitRetiredOwnerReplacement("stale-replacement", processKey),
404
- /not the prepared candidate/
405
- )
406
- await assert.rejects(
407
- () => candidate.commitRetiredOwnerReplacement(ready.replacementId, "release:v1:wrong"),
408
- /process .* is not registered/
409
- )
410
- const committed = candidate.waitForEvent("replacement-committed")
1369
+ const claim = contender.claimOwner(500, authority)
411
1370
 
412
- await candidate.commitRetiredOwnerReplacement(ready.replacementId, processKey)
413
- await committed
414
- await candidate.shutdown()
1371
+ await new Promise((resolve) => setTimeout(resolve, 20))
1372
+ await fixture.client.publishOwnerState({authority: nextAuthority, snapshot: {activeReleaseId: null}})
1373
+ fixture.client.disconnect()
1374
+ await assert.rejects(claim, /authority changed while the claim was queued/)
1375
+ await contender.claimOwner(500, nextAuthority)
1376
+ await contender.shutdown()
415
1377
  await fixture.client.guardianExit()
416
1378
  } finally {
417
1379
  contender.disconnect()
418
- candidate.disconnect()
419
1380
  await cleanupGuardian(fixture)
420
1381
  }
421
1382
  })
@@ -429,6 +1390,7 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
429
1390
 
430
1391
  try {
431
1392
  await legacyProcess.start()
1393
+ await assert.rejects(() => fixture.client.capabilities(), /Guardian capabilities requires a process key/)
432
1394
  legacyPid = legacyProcess.status().pid
433
1395
  const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
434
1396
  const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
@@ -469,6 +1431,190 @@ test("first upgrade migrates a real pre-split guardian without replacing its own
469
1431
  }
470
1432
  })
471
1433
 
1434
+ test("split guardian rejects an owner-state update when its nested legacy definition update fails", async () => {
1435
+ const fixture = await createLegacyGuardian()
1436
+ const processDefinition = definition("legacy-worker")
1437
+ const legacyProcess = fixture.client.process("release:v1:legacy-worker", processDefinition)
1438
+ const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
1439
+ const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
1440
+ const ownerState = {
1441
+ authority,
1442
+ snapshot: {activeReleaseId: "v1", releases: [{processes: [{id: "legacy-worker"}], releaseId: "v1"}], services: [], singletons: []}
1443
+ }
1444
+ let upgraded
1445
+
1446
+ try {
1447
+ await legacyProcess.start()
1448
+ upgraded = await fixture.client.upgradeLegacyGuardian({
1449
+ ownerState,
1450
+ socketPath: path.join(fixture.root, "guardian-v2.sock"),
1451
+ token: "candidate-guardian-capability"
1452
+ })
1453
+ const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
1454
+ const committedOwnerState = {authority: nextAuthority, snapshot: ownerState.snapshot}
1455
+
1456
+ await upgraded.stageOwnerReplacement(prepared.replacementId, committedOwnerState)
1457
+ const restored = upgraded.process("release:v1:legacy-worker", processDefinition)
1458
+
1459
+ await restored.recover()
1460
+ await legacyProcess.updateDefinition({...processDefinition, env: {REVISION: "external"}})
1461
+ await assert.rejects(
1462
+ () => restored.updateDefinition({...processDefinition, env: {REVISION: "candidate"}}, {authority: nextAuthority, snapshot: {...ownerState.snapshot, serviceReleaseIds: {service: "v2"}}}),
1463
+ /provenance mismatch/
1464
+ )
1465
+ assert.deepEqual(await upgraded.ownerState(), committedOwnerState)
1466
+ } finally {
1467
+ await legacyProcess.stop().catch(() => {})
1468
+ upgraded?.disconnect()
1469
+ fixture.client.disconnect()
1470
+ if (upgraded?.pid) killExactProcessGroup(upgraded.pid)
1471
+ if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
1472
+ await fs.rm(fixture.root, {force: true, recursive: true})
1473
+ }
1474
+ })
1475
+
1476
+ test("split guardian defers owner handoff until a nested legacy definition update commits", async () => {
1477
+ const fixture = await createLegacyGuardian()
1478
+ const processDefinition = definition("legacy-worker")
1479
+ const legacyProcess = fixture.client.process("release:v1:legacy-worker", processDefinition)
1480
+ const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
1481
+ const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime", path: "/candidate"}}
1482
+ const ownerState = {
1483
+ authority,
1484
+ snapshot: {activeReleaseId: "v1", releases: [{processes: [{id: "legacy-worker"}], releaseId: "v1"}], services: [], singletons: []}
1485
+ }
1486
+ const socketPath = path.join(fixture.root, "guardian-v2.sock")
1487
+ const token = "candidate-guardian-capability"
1488
+ const gatePath = path.join(fixture.root, "legacy-update.allow")
1489
+ const committedOwnerState = {
1490
+ authority: nextAuthority,
1491
+ listenerConnectionSources: {"updating-owner": {v1: {http: 0, websocket: 1}}},
1492
+ listenerSourceId: "updating-owner",
1493
+ snapshot: {...ownerState.snapshot, update: "committed"}
1494
+ }
1495
+ const contender = new GuardianClient({socketPath, token})
1496
+ let upgraded
1497
+
1498
+ try {
1499
+ await legacyProcess.start()
1500
+ upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, socketPath, token})
1501
+ const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
1502
+
1503
+ await upgraded.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: ownerState.snapshot})
1504
+ const restored = upgraded.process("release:v1:legacy-worker", processDefinition)
1505
+
1506
+ await restored.recover()
1507
+ await contender.connect()
1508
+ const updateResult = restored.updateDefinition({...processDefinition, env: {ROLLBRIDGE_TEST_UPDATE_GATE: gatePath}}, committedOwnerState)
1509
+ .then(() => undefined, (error) => error)
1510
+
1511
+ await waitForFileText(`${gatePath}.waiting`)
1512
+ const claim = contender.claimOwner(1000, nextAuthority)
1513
+ let claimSettled = false
1514
+
1515
+ void claim.finally(() => { claimSettled = true })
1516
+ upgraded.disconnect()
1517
+ await new Promise((resolve) => setTimeout(resolve, 50))
1518
+ assert.equal(claimSettled, false, "owner handoff must wait for the nested definition update")
1519
+
1520
+ await fs.writeFile(gatePath, "allow\n")
1521
+ await claim
1522
+ const updateError = await updateResult
1523
+
1524
+ assert.match(String(updateError), /connection closed while awaiting update/)
1525
+ assert.deepEqual(await contender.ownerState(), {...committedOwnerState, listenerConnectionSources: {}})
1526
+ } finally {
1527
+ await fs.writeFile(gatePath, "allow\n").catch(() => {})
1528
+ await contender.shutdown().catch(() => {})
1529
+ contender.disconnect()
1530
+ upgraded?.disconnect()
1531
+ await legacyProcess.stop().catch(() => {})
1532
+ fixture.client.disconnect()
1533
+ if (upgraded?.pid) killExactProcessGroup(upgraded.pid)
1534
+ if (fixture.child.exitCode === null && fixture.child.signalCode === null) fixture.child.kill("SIGKILL")
1535
+ await fs.rm(fixture.root, {force: true, recursive: true})
1536
+ }
1537
+ })
1538
+
1539
+ test("a disconnected legacy upgrade candidate does not strand its bridge guardian", async () => {
1540
+ const fixture = await createLegacyGuardian()
1541
+ const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
1542
+ const ownerState = {
1543
+ authority,
1544
+ snapshot: {activeReleaseId: null, releases: [], services: [], singletons: []}
1545
+ }
1546
+ let upgraded
1547
+ let bridgeExited = false
1548
+
1549
+ try {
1550
+ upgraded = await fixture.client.upgradeLegacyGuardian({
1551
+ ownerState,
1552
+ socketPath: path.join(fixture.root, "guardian-v2.sock"),
1553
+ token: "candidate-guardian-capability"
1554
+ })
1555
+ await upgraded.prepareOwnerReplacement(authority, {...authority, runtime: {...authority.runtime, digest: "candidate-runtime"}})
1556
+ upgraded.disconnect()
1557
+ let timeout = /** @type {ReturnType<typeof setTimeout> | undefined} */ (undefined)
1558
+
1559
+ await Promise.race([
1560
+ upgraded.guardianExit(),
1561
+ new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Legacy upgrade bridge guardian did not exit after candidate disconnect")), 1000) })
1562
+ ]).finally(() => { if (timeout) clearTimeout(timeout) })
1563
+ bridgeExited = true
1564
+ assert.equal(fixture.child.exitCode, null, "the pre-split guardian must remain available after bridge abandonment")
1565
+ } finally {
1566
+ upgraded?.disconnect()
1567
+ if (!bridgeExited && upgraded?.pid) killExactProcessGroup(upgraded.pid)
1568
+ await cleanupGuardian({client: fixture.client, root: fixture.root})
1569
+ }
1570
+ })
1571
+
1572
+ test("a legacy bridge remains discoverable when its candidate disconnects after the disruptive boundary", async () => {
1573
+ const fixture = await createLegacyGuardian()
1574
+ const statePath = path.join(fixture.root, "state.json")
1575
+ const authority = {configDigest: "legacy-config", runtime: {digest: "legacy-runtime", format: 1, path: "/legacy", version: "0.1.28"}}
1576
+ const nextAuthority = {...authority, runtime: {...authority.runtime, digest: "candidate-runtime"}}
1577
+ const ownerState = {
1578
+ authority,
1579
+ config: {statePath},
1580
+ snapshot: {activeReleaseId: null, control: {path: path.join(fixture.root, "control.sock")}, releases: [], services: [], singletons: []}
1581
+ }
1582
+ let upgraded
1583
+ let replacement
1584
+
1585
+ try {
1586
+ const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
1587
+
1588
+ upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
1589
+ const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
1590
+ const recoverySnapshot = {
1591
+ ...ownerState.snapshot,
1592
+ recovery: {configDigest: authority.configDigest, format: 1, guardian: {...identity, pid: upgraded.pid}, reconnectGraceMs: 3000}
1593
+ }
1594
+
1595
+ await fs.writeFile(statePath, `${JSON.stringify(ownerState.snapshot, null, 2)}\n`)
1596
+ await upgraded.beginLegacyOwnerClaim(prepared.replacementId, 1000, statePath, recoverySnapshot)
1597
+ fixture.client.disconnect()
1598
+ await upgraded.completeLegacyOwnerClaim(prepared.replacementId)
1599
+ assert.deepEqual(JSON.parse(await fs.readFile(statePath, "utf8")), recoverySnapshot)
1600
+ upgraded.disconnect()
1601
+
1602
+ replacement = new GuardianClient({...identity, pid: upgraded.pid})
1603
+ await replacement.connect()
1604
+ const resumed = await replacement.prepareOwnerReplacement(authority, nextAuthority)
1605
+ const committed = replacement.waitForEvent("replacement-committed")
1606
+
1607
+ assert.deepEqual(await replacement.stageOwnerReplacement(resumed.replacementId, {authority: nextAuthority, config: {statePath}, snapshot: ownerState.snapshot}), {committed: true})
1608
+ await committed
1609
+ await replacement.shutdown()
1610
+ await upgraded.guardianExit()
1611
+ } finally {
1612
+ upgraded?.disconnect()
1613
+ replacement?.disconnect()
1614
+ await cleanupGuardian({client: fixture.client, root: fixture.root})
1615
+ }
1616
+ })
1617
+
472
1618
  /** @returns {Promise<{client: GuardianClient, root: string, token: string}>} Started guardian fixture. */
473
1619
  async function createGuardian() {
474
1620
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-guardian-client-"))
@@ -524,9 +1670,139 @@ async function cleanupGuardian(fixture) {
524
1670
  await fs.rm(fixture.root, {force: true, recursive: true})
525
1671
  }
526
1672
 
1673
+ /**
1674
+ * @param {Record<string, import("../src/json.js").JsonValue>} authority - Recovery authority.
1675
+ * @param {string} markerPath - Restart marker path.
1676
+ * @param {string} marker - Expected accepted command marker.
1677
+ * @param {number} reconnectGraceMs - Initial reconnect grace.
1678
+ * @returns {Record<string, import("../src/json.js").JsonValue>} Guardian owner state.
1679
+ */
1680
+ function recoveryOwnerState(authority, markerPath, marker, reconnectGraceMs) {
1681
+ return {
1682
+ authority,
1683
+ recovery: {
1684
+ command: {
1685
+ args: ["-e", "require('node:fs').appendFileSync(process.argv[1], JSON.stringify({at: Date.now(), home: process.env.HOME ?? null, marker: process.env.RESTART_MARKER}) + '\\n')", markerPath],
1686
+ cwd: path.dirname(markerPath),
1687
+ env: {RESTART_MARKER: marker},
1688
+ executable: process.execPath
1689
+ },
1690
+ reconnectGraceMs,
1691
+ startupTimeoutMs: 1000
1692
+ },
1693
+ snapshot: {activeReleaseId: null}
1694
+ }
1695
+ }
1696
+
1697
+ /**
1698
+ * @param {{client: GuardianClient, root: string, token: string}} fixture - Guardian fixture.
1699
+ * @param {Record<string, import("../src/json.js").JsonValue>} authority - Recovery authority.
1700
+ * @param {string} markerPath - Claim marker path.
1701
+ * @param {{claimDelayMs?: number, descendantPath?: string, exitAfterClaim?: boolean, ready?: boolean, replacementCommittedPath?: string, replacementPreparedPath?: string, startedLogPath?: string, startedPath?: string, startupTimeoutMs?: number}} [options] - Optional recovery behavior.
1702
+ * @returns {Record<string, import("../src/json.js").JsonValue>} Guardian owner state.
1703
+ */
1704
+ function claimingRecoveryOwnerState(fixture, authority, markerPath, options = {}) {
1705
+ return {
1706
+ authority,
1707
+ recovery: {
1708
+ command: {
1709
+ args: [recoveryOwnerPath],
1710
+ cwd: fixture.root,
1711
+ env: {
1712
+ GUARDIAN_AUTHORITY: JSON.stringify(authority),
1713
+ GUARDIAN_CLAIM_DELAY_MS: String(options.claimDelayMs || 0),
1714
+ ...(options.descendantPath ? {GUARDIAN_DESCENDANT_PATH: options.descendantPath} : {}),
1715
+ ...(options.exitAfterClaim ? {GUARDIAN_EXIT_AFTER_CLAIM: "1"} : {}),
1716
+ GUARDIAN_MARKER_PATH: markerPath,
1717
+ ...(options.replacementCommittedPath ? {GUARDIAN_REPLACEMENT_COMMITTED_PATH: options.replacementCommittedPath} : {}),
1718
+ ...(options.replacementPreparedPath ? {GUARDIAN_REPLACEMENT_PREPARED_PATH: options.replacementPreparedPath} : {}),
1719
+ ...(options.ready === false ? {GUARDIAN_SKIP_READY: "1"} : {}),
1720
+ GUARDIAN_SOCKET_PATH: fixture.client.socketPath,
1721
+ ...(options.startedLogPath ? {GUARDIAN_STARTED_LOG_PATH: options.startedLogPath} : {}),
1722
+ ...(options.startedPath ? {GUARDIAN_STARTED_PATH: options.startedPath} : {}),
1723
+ GUARDIAN_TOKEN: fixture.token
1724
+ },
1725
+ executable: process.execPath
1726
+ },
1727
+ reconnectGraceMs: 10,
1728
+ startupTimeoutMs: options.startupTimeoutMs || 1000
1729
+ },
1730
+ snapshot: {activeReleaseId: null}
1731
+ }
1732
+ }
1733
+
1734
+ /**
1735
+ * @param {string} markerPath - Restart marker path.
1736
+ * @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Restart records.
1737
+ */
1738
+ async function restartRecords(markerPath) {
1739
+ try {
1740
+ return (await fs.readFile(markerPath, "utf8")).trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))
1741
+ } catch (error) {
1742
+ if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return []
1743
+ throw error
1744
+ }
1745
+ }
1746
+
1747
+ /**
1748
+ * @param {string} markerPath - Restart marker path.
1749
+ * @param {number} count - Required record count.
1750
+ * @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Restart records.
1751
+ */
1752
+ async function waitForRestartRecords(markerPath, count) {
1753
+ const deadline = Date.now() + 3000
1754
+
1755
+ while (Date.now() < deadline) {
1756
+ const records = await restartRecords(markerPath)
1757
+
1758
+ if (records.length >= count) return records
1759
+ await new Promise((resolve) => setTimeout(resolve, 10))
1760
+ }
1761
+ throw new Error(`Timed out waiting for ${count} guardian restart record${count === 1 ? "" : "s"}`)
1762
+ }
1763
+
1764
+ /**
1765
+ * @param {string} filePath - File to read after creation.
1766
+ * @param {RegExp} [expected] - Content that must be present before returning.
1767
+ * @returns {Promise<string>} Non-empty accepted file contents.
1768
+ */
1769
+ async function waitForFileText(filePath, expected) {
1770
+ const deadline = Date.now() + 3000
1771
+
1772
+ while (Date.now() < deadline) {
1773
+ try {
1774
+ const contents = await fs.readFile(filePath, "utf8")
1775
+
1776
+ if (contents && (!expected || expected.test(contents))) return contents
1777
+ } catch (error) {
1778
+ if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
1779
+ }
1780
+ await new Promise((resolve) => setTimeout(resolve, 10))
1781
+ }
1782
+ throw new Error(`Timed out waiting for ${filePath}`)
1783
+ }
1784
+
1785
+ /**
1786
+ * @param {{client: GuardianClient, root: string, token: string}} fixture - Guardian fixture.
1787
+ * @param {Record<string, import("../src/json.js").JsonValue>} authority - Committed authority.
1788
+ */
1789
+ async function reconnectAndShutdownGuardian(fixture, authority) {
1790
+ const cleanup = new GuardianClient({pid: fixture.client.pid, socketPath: fixture.client.socketPath, token: fixture.token})
1791
+
1792
+ try {
1793
+ await cleanup.connect()
1794
+ await cleanup.claimOwner(500, authority)
1795
+ await cleanup.shutdown()
1796
+ } finally {
1797
+ cleanup.disconnect()
1798
+ await fixture.client.guardianExit().catch(() => {})
1799
+ await fs.rm(fixture.root, {force: true, recursive: true})
1800
+ }
1801
+ }
1802
+
527
1803
  /**
528
1804
  * @param {string} id - Process id.
529
- * @returns {Parameters<GuardianClient["process"]>[1]} Managed process definition.
1805
+ * @returns {Parameters<GuardianClient["process"]>[1] & import("../src/managed-process.js").ManagedProcessDefinition} Managed process definition.
530
1806
  */
531
1807
  function definition(id) {
532
1808
  return {
@@ -536,6 +1812,7 @@ function definition(id) {
536
1812
  id,
537
1813
  lifecycle: {drainTimeoutMs: 0},
538
1814
  logger: () => {},
1815
+ memory: undefined,
539
1816
  outputLines: 10,
540
1817
  restart: {backoffFactor: 1, maxDelayMs: 0, maxRestarts: 0, windowMs: 0},
541
1818
  restartDelayMs: 0,