rollbridge 0.1.39 → 0.1.40

package/test/owner-recovery.test.js

@@ -13,6 +13,7 @@ import {normalizeConfig} from "../src/config.js"
 import {sendControlCommand} from "../src/control-client.js"
 import RollbridgeDaemon from "../src/daemon.js"
 import GuardianClient from "../src/guardian-client.js"
+import {isProcessRunning, waitForProcessExit} from "./support/process.js"
 
 const currentDir = path.dirname(fileURLToPath(import.meta.url))
 const binPath = path.join(currentDir, "..", "bin", "rollbridge")
@@ -20,7 +21,7 @@ const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
 const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
 
 /** @typedef {import("../src/daemon.js").DaemonStatus} DaemonStatus */
-/** @typedef {DaemonStatus & {recovery: {configDigest: string}, singletonReleaseIds?: Record<string, string>}} RecoveryState */
+/** @typedef {DaemonStatus & {recovery: {configDigest: string}, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} RecoveryState */
 
 test("external owner retirement releases guardian authority without losing its generation", async () => {
   const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-retirement-recovery-"))
@@ -85,7 +86,7 @@ test("external owner retirement releases guardian authority without losing its g
     await waitForFile(path.join(v1Path, "drain-started"), 1000)
     await fs.writeFile(path.join(v1Path, "drained"), "done\n")
     await waitForProcessExit(v1WorkerPid, 1000)
-    assert.equal(isAlive(v2WorkerPid), true, "the active candidate worker must remain usable while the old generation drains")
+    assert.equal(isProcessRunning(v2WorkerPid), true, "the active candidate worker must remain usable while the old generation drains")
   } finally {
     await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "drained"), "done\n").catch(() => {})))
     await replacement?.shutdown().catch(() => {})
@@ -134,6 +135,12 @@ test("exact bootstrap restores the committed generation after external owner ret
     const recoveredService = recovered.services.get("beacon")
 
     assert.ok(recoveredCandidate && recoveredService && recovered.singletons.get("singleton"))
+    recovered.serviceReleaseIds.set("beacon", "v1")
+    assert.throws(
+      () => recovered?.assertCommittedBootstrapRecoveryReady(),
+      /service beacon belongs to retained release v1/
+    )
+    recovered.serviceReleaseIds.set("beacon", "v2")
     const activateGeneration = recoveredCandidate.activateGeneration.bind(recoveredCandidate)
     const startService = recoveredService.start.bind(recoveredService)
     const replaceSingletons = recovered.replaceSingletons.bind(recovered)
@@ -158,7 +165,7 @@ test("exact bootstrap restores the committed generation after external owner ret
     assert.equal(active.generationTransition?.phase, "committed")
     assert.equal(active.releases.find(({releaseId}) => releaseId === "v1")?.state, "draining")
     assert.equal(releaseProcessPid(active, "v1", "worker"), v1WorkerPid)
-    assert.equal(isAlive(v1WorkerPid), true, "the retained previous generation must keep draining")
+    assert.equal(isProcessRunning(v1WorkerPid), true, "the retained previous generation must keep draining")
     assert.equal(active.releases.find(({releaseId}) => releaseId === "v2")?.state, "active")
     assert.ok(active.releases.find(({releaseId}) => releaseId === "v2")?.processes.every(({pid, state}) => typeof pid === "number" && state === "running"))
     assert.ok(active.services.every(({process}) => typeof process.pid === "number" && process.state === "running"))
@@ -180,6 +187,68 @@ test("exact bootstrap restores the committed generation after external owner ret
   }
 })
 
+test("exact bootstrap restores a committed generation after its previous release was pruned", async () => {
+  const fixture = await createFixture()
+
+  fixture.config.processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
+    .filter((processConfig) => processConfig.id !== "beacon" && processConfig.id !== "singleton")
+  const worker = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
+    .find((processConfig) => processConfig.id === "worker")
+
+  assert.ok(worker)
+  const workerLifecycle = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (worker.lifecycle)
+
+  workerLifecycle.drainTimeoutMs = 500
+  fixture.config.releaseRetention = {keep: 0, maxAgeMs: 0}
+  await writeConfig(fixture.configPath, fixture.config)
+  const config = normalizeConfig(fixture.config, fixture.configPath)
+  const retired = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
+  /** @type {RollbridgeDaemon | undefined} */
+  let recovered
+  const v1Path = await prepareRelease(fixture.root, "v1")
+  const v2Path = await prepareRelease(fixture.root, "v2")
+
+  try {
+    await retired.start()
+    await retired.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
+    await retired.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
+    await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
+    await waitForState(fixture.statePath, (state) => !state.releases.some(({releaseId}) => releaseId === "v1"), AbortSignal.timeout(5000))
+    const committed = retired.status()
+    const retiredPids = committed.releases.find(({releaseId}) => releaseId === "v2")?.processes
+      .map(({pid}) => pid)
+      .filter((pid) => typeof pid === "number") || []
+
+    assert.equal(committed.releases.some(({releaseId}) => releaseId === "v1"), false)
+    await retired.retireOwner({attestation: `sha256:${"d".repeat(64)}`})
+    await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
+    await Promise.all(retiredPids.map((pid) => waitForProcessExit(pid, 3000)))
+
+    recovered = new RollbridgeDaemon({
+      bootstrap: {releaseId: "v2", releasePath: v2Path, revision: "v2"},
+      config,
+      configPath: fixture.configPath,
+      logger: () => {}
+    })
+    await recovered.start({exposeControl: false})
+    await recovered.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
+
+    assert.equal(recovered.status().activeReleaseId, "v2")
+    assert.equal(recovered.status().generationTransition?.phase, "committed")
+  } finally {
+    if (recovered) {
+      const activeRecovery = recovered.status().activeReleaseId === "v2"
+      const shutdown = recovered.shutdown()
+
+      if (activeRecovery) await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n").catch(() => undefined)
+      await shutdown.catch(() => undefined)
+    }
+    retired.guardian?.disconnect()
+    await stopFixtureGuardian(fixture.statePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
 test("journaled committed bootstrap recovery resumes after a restart begins", async () => {
   const fixture = await createFixture()
   const config = normalizeConfig(fixture.config, fixture.configPath)
@@ -237,7 +306,7 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
     assert.deepEqual(active.services.map(({process}) => process.pid), restartedServicePids)
     assert.ok(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running"))
     assert.equal(releaseProcessPid(active, "v1", "worker"), v1WorkerPid)
-    assert.equal(isAlive(v1WorkerPid), true)
+    assert.equal(isProcessRunning(v1WorkerPid), true)
     assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
   } finally {
     if (recovered) {
@@ -320,7 +389,7 @@ test("committed bootstrap tuple mismatches fail closed without singletons", asyn
     assert.equal(preserved.generationTransition?.phase, "committed")
     assert.equal(preserved.releases.find(({releaseId}) => releaseId === "v2")?.state, "draining")
     assert.equal(releaseProcessPid(preserved, "v1", "worker"), v1WorkerPid)
-    assert.equal(isAlive(v1WorkerPid), true)
+    assert.equal(isProcessRunning(v1WorkerPid), true)
     assert.equal(preserved.releases.some(({releaseId}) => releaseId === "wrong"), false)
 
     await owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
@@ -425,6 +494,260 @@ test("replacement owner reconstructs one active and two draining generations aft
   }
 })
 
+test("guardian restarts an abruptly exited daemon without replacing managed processes", async () => {
+  const fixture = await createFixture()
+  const owner = spawnDaemon(fixture.configPath)
+  let recoveredDaemonPid
+  let workerPid
+
+  try {
+    await waitForLog(owner, "control socket listening")
+    const releasePath = await prepareRelease(fixture.root, "v1")
+
+    await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
+    const before = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
+
+    workerPid = releaseProcessPid(before, "v1", "worker")
+    const recoveredListenerLog = waitForLog(owner, "control socket listening", {allowChildExit: true})
+
+    owner.kill("SIGKILL")
+    await once(owner, "exit")
+    await recoveredListenerLog
+    const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
+
+    recoveredDaemonPid = recovered.daemonPid
+    assert.notEqual(recovered.daemonPid, before.daemonPid)
+    assert.equal(recovered.activeReleaseId, "v1")
+    assert.equal(releaseProcessPid(recovered, "v1", "worker"), workerPid)
+
+    const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
+
+    await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
+    await shutdown
+  } finally {
+    await killChild(owner)
+    if (recoveredDaemonPid && isProcessRunning(recoveredDaemonPid)) process.kill(recoveredDaemonPid, "SIGKILL")
+    if (workerPid) {
+      try { process.kill(-workerPid, "SIGKILL") } catch (_error) { /* The exact managed group already exited. */ }
+    }
+    await stopFixtureGuardian(fixture.statePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("guardian recovers a persistent service after its final active release stops", async () => {
+  const fixture = await createFixture()
+  const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
+  const jobs = processes.find((processConfig) => processConfig.id === "jobs")
+  const worker = processes.find((processConfig) => processConfig.id === "worker")
+
+  assert.ok(jobs && worker)
+  jobs.port = {from: 17000, to: 17001}
+  worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
+  fixture.config.processes = processes.filter((processConfig) => processConfig.id !== "singleton")
+  await writeConfig(fixture.configPath, fixture.config)
+  const owner = spawnDaemon(fixture.configPath)
+  let recoveredDaemonPid
+
+  try {
+    await waitForLog(owner, "control socket listening")
+    const releasePath = await prepareRelease(fixture.root, "v1")
+
+    await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
+    await sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: fixture.socketPath})
+    const before = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
+    const servicePid = before.services[0]?.process.pid
+
+    assert.equal(before.activeReleaseId, null)
+    assert.equal(typeof servicePid, "number")
+    const persisted = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
+
+    assert.equal(persisted.serviceReleaseIds?.beacon, "v1")
+    const recoveredListenerLog = waitForLog(owner, "control socket listening", {allowChildExit: true})
+
+    owner.kill("SIGKILL")
+    await once(owner, "exit")
+    await recoveredListenerLog
+    const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
+
+    recoveredDaemonPid = recovered.daemonPid
+    assert.equal(recovered.activeReleaseId, null)
+    assert.equal(recovered.services[0]?.process.pid, servicePid)
+    assert.deepEqual(recovered.releaseReferences, [{releaseId: "v1", releasePath}])
+    const nextReleasePath = await prepareRelease(fixture.root, "v2")
+
+    await sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: nextReleasePath, revision: "v2"}, path: fixture.socketPath})
+    const finalReleasePath = await prepareRelease(fixture.root, "v3")
+
+    await sendControlCommand({command: {command: "deploy", releaseId: "v3", releasePath: finalReleasePath, revision: "v3"}, path: fixture.socketPath})
+    assert.equal((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId, "v3")
+    await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
+  } finally {
+    await killChild(owner)
+    if (recoveredDaemonPid && isProcessRunning(recoveredDaemonPid)) process.kill(recoveredDaemonPid, "SIGKILL")
+    await stopFixtureGuardian(fixture.statePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("owner state omits a new persistent service until its defining release is retained", async () => {
+  const fixture = await createFixture()
+  const config = normalizeConfig(fixture.config, fixture.configPath)
+  const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
+  const releasePath = await prepareRelease(fixture.root, "v1", {holdJobsBind: true})
+  /** @type {Promise<Record<string, import("../src/json.js").JsonValue>> | undefined} */
+  let deploy
+
+  try {
+    await owner.start()
+    deploy = owner.deploy({releaseId: "v1", releasePath, revision: "v1"})
+    await waitForFile(path.join(releasePath, "jobs.bind-waiting"), 3000)
+    await owner.persistState({throwOnError: true})
+    const persisted = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
+    const guardianState = /** @type {Record<string, import("../src/json.js").JsonValue> | undefined} */ (await owner.guardian?.ownerState())
+    const guardianSnapshot = /** @type {RecoveryState | undefined} */ (guardianState?.snapshot)
+
+    assert.deepEqual(persisted.releases, [])
+    assert.deepEqual(persisted.services, [])
+    assert.deepEqual(persisted.serviceReleaseIds, {})
+    assert.deepEqual(guardianSnapshot?.releases, [])
+    assert.deepEqual(guardianSnapshot?.services, [])
+    assert.deepEqual(guardianState?.serviceReleaseIds, {})
+
+    await fs.writeFile(path.join(releasePath, "jobs.bind"), "ready\n")
+    await deploy
+    assert.equal(owner.status().services.find(({id}) => id === "beacon")?.process.state, "running")
+    assert.equal(owner.serviceReleaseIds.get("beacon"), "v1")
+  } finally {
+    await fs.writeFile(path.join(releasePath, "jobs.bind"), "ready\n").catch(() => {})
+    await deploy?.catch(() => {})
+    await Promise.all([
+      fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n").catch(() => {}),
+      owner.shutdown().catch(() => {})
+    ])
+    owner.guardian?.disconnect()
+    await stopFixtureGuardian(fixture.statePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("owner recovery accepts released format-2 state without journal revision or service owner metadata", async () => {
+  const fixture = await createFixture()
+  const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
+  const worker = processes.find((processConfig) => processConfig.id === "worker")
+
+  assert.ok(worker)
+  worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
+  await writeConfig(fixture.configPath, fixture.config)
+  const config = normalizeConfig(fixture.config, fixture.configPath)
+  const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
+  const releasePath = await prepareRelease(fixture.root, "v1")
+  /** @type {RollbridgeDaemon | undefined} */
+  let recovered
+
+  try {
+    await owner.start()
+    await owner.deploy({releaseId: "v1", releasePath, revision: "v1"})
+    if (owner.pendingWrite) await owner.pendingWrite
+    const publicState = JSON.parse(await fs.readFile(fixture.statePath, "utf8"))
+    const privateOwnerState = owner.transferableOwnerState()
+    const privateSnapshot = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (privateOwnerState.snapshot)
+    const privateTransition = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (privateSnapshot.generationTransition)
+
+    delete publicState.generationTransition.journalRevision
+    delete publicState.serviceReleaseIds
+    delete privateTransition.journalRevision
+    delete privateOwnerState.serviceReleaseIds
+    await owner.guardian?.publishOwnerState(privateOwnerState)
+    await fs.writeFile(fixture.statePath, `${JSON.stringify(publicState, null, 2)}\n`)
+    await owner.retireCommittedOwner(undefined)
+    owner.guardian?.disconnect()
+
+    recovered = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
+    await recovered.start()
+
+    assert.equal(recovered.status().activeReleaseId, "v1")
+    assert.equal(recovered.status().generationTransition?.journalRevision, undefined)
+    assert.equal(recovered.serviceReleaseIds.get("beacon"), "v1")
+  } finally {
+    if (recovered) await recovered.shutdown().catch(() => {})
+    else await owner.shutdown().catch(() => {})
+    owner.guardian?.disconnect()
+    await stopFixtureGuardian(fixture.statePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("guardian recovery becomes ready before replaying a gated generation hook", async () => {
+  const fixture = await createFixture()
+  const retirementGatePath = path.join(fixture.root, "retirement.allow")
+  const retirementWaitingPath = path.join(fixture.root, "retirement.waiting")
+  const daemonPidPath = path.join(fixture.root, "daemon.pid")
+  const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
+  const jobs = processes.find((processConfig) => processConfig.id === "jobs")
+  const worker = processes.find((processConfig) => processConfig.id === "worker")
+  const lifecycle = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobs?.lifecycle)
+
+  assert.ok(jobs && worker)
+  jobs.gracefulStopMs = 5000
+  lifecycle.quietCommand = `printf 'waiting\n' >> ${JSON.stringify(retirementWaitingPath)}; while [ ! -f ${JSON.stringify(retirementGatePath)} ]; do sleep 0.02; done; printf 'retire:%s\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(fixture.lifecycleLogPath)}`
+  worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
+  await writeConfig(fixture.configPath, fixture.config)
+  const owner = spawnDaemon(fixture.configPath, undefined, {daemonPidPath, startupTimeoutMs: 3000})
+  const v1Path = await prepareRelease(fixture.root, "v1")
+  const v2Path = await prepareRelease(fixture.root, "v2")
+  let recoveredDaemonPid
+
+  try {
+    await waitForLog(owner, "control socket listening")
+    await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})
+    const interruptedDeploy = sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})
+
+    void interruptedDeploy.catch(() => {})
+    await waitForFile(retirementWaitingPath)
+    await fs.rm(daemonPidPath, {force: true})
+    const recoveredListenerLog = waitForLog(owner, "control socket listening", {allowChildExit: true})
+
+    owner.kill("SIGKILL")
+    await once(owner, "exit")
+    await recoveredListenerLog
+    await waitForFile(daemonPidPath, 3000)
+    recoveredDaemonPid = Number((await fs.readFile(daemonPidPath, "utf8")).trim())
+    const recovering = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
+    const guardianPid = JSON.parse(await fs.readFile(fixture.statePath, "utf8")).recovery?.guardian?.pid
+
+    assert.equal(recovering.daemonPid, recoveredDaemonPid)
+    assert.equal(recovering.ownerRecovery?.ready, true)
+    assert.equal(recovering.generationTransition?.phase, "retiring_previous")
+    assert.equal(typeof guardianPid, "number")
+    await assert.rejects(
+      sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: fixture.socketPath}),
+      /Another owner mutation/
+    )
+    await assert.rejects(
+      sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}),
+      /Cannot shut down while generation transition recovery is in progress/
+    )
+    await assert.rejects(
+      sendControlCommand({command: {attestation: `sha256:${"a".repeat(64)}`, command: "retire-owner"}, path: fixture.socketPath}),
+      /Cannot retire owner while generation transition recovery is in progress/
+    )
+
+    process.kill(recoveredDaemonPid, "SIGTERM")
+    await fs.writeFile(retirementGatePath, "release retirement\n")
+    await waitForProcessExit(recoveredDaemonPid, 5000)
+    await waitForProcessExit(guardianPid, 5000)
+    assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "retire:v1", "activate:v2", "retire:v2"])
+  } finally {
+    await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
+    await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
+    await killChild(owner)
+    if (recoveredDaemonPid && isProcessRunning(recoveredDaemonPid)) process.kill(recoveredDaemonPid, "SIGKILL")
+    await stopFixtureGuardian(fixture.statePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
 test("owner recovery preserves a failed generation transition without firing hooks until exact resume", async () => {
   const fixture = await createFixture({activationFailureRelease: "v2"})
   let owner = spawnDaemon(fixture.configPath)
@@ -483,10 +806,15 @@ test("owner recovery replays one journaled ambiguous activation by exact generat
     const ambiguous = JSON.parse(await fs.readFile(fixture.statePath, "utf8"))
 
     delete ambiguous.generationTransition.error
+    ambiguous.generationTransition.journalRevision += 1
     await fs.writeFile(fixture.statePath, `${JSON.stringify(ambiguous, null, 2)}\n`)
     await fs.writeFile(fixture.activationGatePath, "allow\n")
     owner = spawnDaemon(fixture.configPath)
+    const recoverySettled = waitForLog(owner, "release generation transition recovery settled")
+
     await waitForLog(owner, "control socket listening")
+    await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed", AbortSignal.timeout(5000))
+    await recoverySettled
 
     const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
 
@@ -504,8 +832,14 @@ test("owner recovery replays one journaled ambiguous activation by exact generat
   }
 })
 
-test("owner recovery preserves exact release definitions across a candidate config authority change", async () => {
+test("owner recovery preserves complete private transition authority across a candidate config change", async () => {
   const fixture = await createFixture()
+  const initialProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
+  const initialWorker = initialProcesses.find((processConfig) => processConfig.id === "worker")
+
+  assert.ok(initialWorker)
+  initialWorker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
+  await writeConfig(fixture.configPath, fixture.config)
   const initialConfig = normalizeConfig(fixture.config, fixture.configPath)
   const owner = new RollbridgeDaemon({config: initialConfig, configPath: fixture.configPath, logger: () => {}})
   /** @type {RollbridgeDaemon | undefined} */
@@ -534,6 +868,12 @@ test("owner recovery preserves exact release definitions across a candidate conf
 
     await owner.retireCommittedOwner(undefined)
     owner.guardian?.disconnect()
+    const partialPublicState = JSON.parse(await fs.readFile(fixture.statePath, "utf8"))
+
+    partialPublicState.generationTransition.phase = "committed"
+    partialPublicState.releases = partialPublicState.releases.filter((/** @type {{releaseId: string}} */ release) => release.releaseId !== "v2")
+    partialPublicState.releaseReferences = partialPublicState.releaseReferences.filter((/** @type {{releaseId: string}} */ release) => release.releaseId !== "v2")
+    await fs.writeFile(fixture.statePath, `${JSON.stringify(partialPublicState, null, 2)}\n`)
 
     recovered = new RollbridgeDaemon({config: normalizeConfig(changedConfig, fixture.configPath), configPath: fixture.configPath, logger: () => {}})
     await recovered.start()
@@ -542,12 +882,7 @@ test("owner recovery preserves exact release definitions across a candidate conf
     assert.equal(recovered.status().generationTransition?.phase, "committed")
     assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "activate:v2"])
   } finally {
-    if (recovered) {
-      const shutdown = recovered.shutdown()
-
-      await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n").catch(() => undefined)))
-      await shutdown.catch(() => {})
-    }
+    if (recovered) await recovered.shutdown().catch(() => {})
     await stopFixtureGuardian(fixture.statePath)
     await fs.rm(fixture.root, {force: true, recursive: true})
   }
@@ -583,6 +918,7 @@ test("owner recovery reconstructs a stopped release that still owns a committed-
 
     assert.equal(pending.activeReleaseId, "v2")
     assert.equal(pending.generationTransition?.phase, "committed_pending")
+    assert.equal(pending.serviceReleaseIds?.beacon, "v2")
     assert.equal(pending.singletonReleaseIds?.singleton, "v1")
     const stoppedRelease = pending.releases.find((release) => release.releaseId === "v1" && release.state === "stopped")
 
@@ -622,6 +958,9 @@ test("owner recovery reconstructs a stopped release that still owns a committed-
 
 test("owner recovery uses the owning release singleton definition during a committed-pending config change", async () => {
   const fixture = await createFixture()
+  const fixtureProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
+
+  fixture.config.processes = fixtureProcesses.filter((processConfig) => processConfig.id !== "beacon")
   const initialProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
   const initialSingleton = initialProcesses.find((processConfig) => processConfig.id === "singleton")
 
@@ -688,6 +1027,67 @@ test("owner recovery uses the owning release singleton definition during a commi
   }
 })
 
+test("owner recovery retains a stopped previous release until committed-pending singleton work completes", async () => {
+  const fixture = await createFixture()
+  const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
+  const worker = processes.find((processConfig) => processConfig.id === "worker")
+
+  assert.ok(worker)
+  worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
+  await writeConfig(fixture.configPath, fixture.config)
+  const config = normalizeConfig(fixture.config, fixture.configPath)
+  const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
+  const replaceSingletons = owner.replaceSingletons.bind(owner)
+  /** @type {() => void} */
+  let continueReplacement = () => {}
+  /** @type {() => void} */
+  let markReplacementComplete = () => {}
+  const replacementGate = new Promise((resolve) => { continueReplacement = () => resolve(undefined) })
+  const replacementComplete = new Promise((resolve) => { markReplacementComplete = () => resolve(undefined) })
+  const v1Path = await prepareRelease(fixture.root, "v1")
+  const v2Path = await prepareRelease(fixture.root, "v2")
+  /** @type {Promise<Record<string, import("../src/json.js").JsonValue>> | undefined} */
+  let deployPromise
+  /** @type {RollbridgeDaemon | undefined} */
+  let recovered
+
+  owner.replaceSingletons = async (release) => {
+    await replaceSingletons(release)
+    if (release.releaseId === "v2") {
+      markReplacementComplete()
+      await replacementGate
+    }
+  }
+
+  try {
+    await owner.start()
+    await owner.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
+    deployPromise = owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
+    void deployPromise.catch(() => {})
+    await replacementComplete
+    const pending = await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed_pending" && state.releases?.some((release) => release.releaseId === "v1" && release.state === "stopped"), AbortSignal.timeout(5000))
+
+    assert.deepEqual(pending.releaseReferences.map((reference) => reference.releaseId), ["v1", "v2"])
+    assert.equal(pending.singletonReleaseIds?.singleton, "v2")
+    await owner.retireCommittedOwner(undefined)
+    owner.guardian?.disconnect()
+
+    recovered = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
+    await recovered.start()
+
+    assert.equal(recovered.status().generationTransition?.phase, "committed")
+    assert.equal(recovered.status().activeReleaseId, "v2")
+  } finally {
+    continueReplacement()
+    await deployPromise?.catch(() => {})
+    if (recovered) await recovered.shutdown().catch(() => {})
+    else await owner.shutdown().catch(() => {})
+    owner.guardian?.disconnect()
+    await stopFixtureGuardian(fixture.statePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
 test("owner recovery replays ambiguous retirement with the previous release's exact definition", async () => {
   const fixture = await createFixture()
   const retirementGatePath = path.join(fixture.root, "retirement.allow")
@@ -722,8 +1122,11 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
     await fs.writeFile(retirementGatePath, "release retirement\n")
 
     owner = spawnDaemon(fixture.configPath)
+    const recoverySettled = waitForLog(owner, "release generation transition recovery settled")
 
     await waitForLog(owner, "control socket listening")
+    await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed", AbortSignal.timeout(5000))
+    await recoverySettled
     assert.equal((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length, 2, "ambiguous retirement must replay exactly once")
     assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "retire:v1", "activate:v2"])
     assert.equal((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId, "v2")
@@ -760,7 +1163,7 @@ test("owner recovery rejects config identity mismatch without changing the valid
     const rejected = await runDaemon(fixture.configPath)
 
     assert.notEqual(rejected.code, 0)
-    assert.match(rejected.output, /config identity does not match/)
+    assert.match(rejected.output, /authority does not match/)
     assert.equal(await fs.readFile(fixture.statePath, "utf8"), validState)
 
     await writeConfig(fixture.configPath, fixture.config)
@@ -819,9 +1222,10 @@ test("failed recovery bootstrap keeps the reconstructed active generation servin
   }
 })
 
-test("owner recovery fails closed on a partial snapshot and preserves it for repair", async () => {
+test("owner recovery repairs a partial public snapshot from committed guardian state", async () => {
   const fixture = await createFixture()
-  let owner = spawnDaemon(fixture.configPath)
+  const owner = spawnDaemon(fixture.configPath)
+  let recoveredDaemonPid
   let workerPid
 
   try {
@@ -838,21 +1242,24 @@ test("owner recovery fails closed on a partial snapshot and preserves it for rep
     const partialState = {...validState, releases: []}
     await fs.writeFile(fixture.statePath, `${JSON.stringify(partialState, null, 2)}\n`)
 
-    const rejected = await runDaemon(fixture.configPath)
+    const repairedState = await waitForState(
+      fixture.statePath,
+      (state) => state.daemonPid !== validState.daemonPid && state.releases.some((release) => release.releaseId === "v1"),
+      AbortSignal.timeout(5000)
+    )
+    recoveredDaemonPid = repairedState.daemonPid
+    const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
 
-    assert.notEqual(rejected.code, 0)
-    assert.match(rejected.output, /does not contain active release v1/)
-    assert.deepEqual(JSON.parse(await fs.readFile(fixture.statePath, "utf8")), partialState)
+    assert.equal(recovered.activeReleaseId, "v1")
+    assert.equal(releaseProcessPid(recovered, "v1", "worker"), workerPid)
+    assert.notDeepEqual(JSON.parse(await fs.readFile(fixture.statePath, "utf8")), partialState)
 
-    await fs.writeFile(fixture.statePath, `${JSON.stringify(validState, null, 2)}\n`)
-    owner = spawnDaemon(fixture.configPath)
-    await waitForLog(owner, "control socket listening")
     const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
     await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
     await shutdown
-    await once(owner, "exit")
   } finally {
     await killChild(owner)
+    if (recoveredDaemonPid && isProcessRunning(recoveredDaemonPid)) process.kill(recoveredDaemonPid, "SIGKILL")
     if (workerPid) {
       try { process.kill(-workerPid, "SIGKILL") } catch (_error) { /* The exact group already exited. */ }
     }
@@ -1014,6 +1421,75 @@ test("deploy rejects a live ownerRecovery mode change", async () => {
   }
 })
 
+test("deploy rejects a live activation lifecycle mode change", async () => {
+  const fixture = await createFixture()
+  const owner = spawnDaemon(fixture.configPath)
+  const v1Path = await prepareRelease(fixture.root, "v1")
+  const v2Path = await prepareRelease(fixture.root, "v2")
+
+  try {
+    await waitForLog(owner, "control socket listening")
+    await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})
+    const changedConfig = structuredClone(fixture.config)
+    const jobs = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (changedConfig.processes).find((processConfig) => processConfig.id === "jobs")
+    const lifecycle = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobs?.lifecycle)
+
+    delete lifecycle.activateCommand
+    await writeConfig(fixture.configPath, changedConfig)
+    await assert.rejects(
+      sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath}),
+      /lifecycle\.activateCommand.*cannot be applied live/
+    )
+
+    await writeConfig(fixture.configPath, fixture.config)
+    const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
+
+    await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
+    await shutdown
+  } finally {
+    await killChild(owner)
+    await stopFixtureGuardian(fixture.statePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("public state does not advance when private guardian publication fails", async () => {
+  const fixture = await createFixture()
+  const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
+  const worker = processes.find((processConfig) => processConfig.id === "worker")
+
+  assert.ok(worker)
+  worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
+  await writeConfig(fixture.configPath, fixture.config)
+  const config = normalizeConfig(fixture.config, fixture.configPath)
+  const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
+  const releasePath = await prepareRelease(fixture.root, "v1")
+
+  try {
+    await owner.start()
+    await owner.deploy({releaseId: "v1", releasePath, revision: "v1"})
+    if (owner.pendingWrite) await owner.pendingWrite
+    const before = await fs.readFile(fixture.statePath, "utf8")
+    const publishOwnerState = owner.publishOwnerState.bind(owner)
+
+    owner.ownerReady = true
+    owner.publishOwnerState = async () => { throw new Error("injected guardian publication failure") }
+    const write = owner.persistState({throwOnError: true})
+
+    assert.ok(write)
+    await assert.rejects(write, /injected guardian publication failure/)
+    assert.equal(await fs.readFile(fixture.statePath, "utf8"), before)
+    owner.publishOwnerState = publishOwnerState
+    await owner.persistState({throwOnError: true})
+    await owner.shutdown()
+  } finally {
+    owner.publishOwnerState = RollbridgeDaemon.prototype.publishOwnerState.bind(owner)
+    await owner.shutdown().catch(() => {})
+    await stopFixtureGuardian(fixture.statePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
 test("replacement removes only guardian-owned candidate inventory left before deploy commit", async () => {
   const fixture = await createFixture()
   let owner = spawnDaemon(fixture.configPath)
@@ -1056,7 +1532,7 @@ test("replacement removes only guardian-owned candidate inventory left before de
 
     assert.equal(recovered.activeReleaseId, "v1")
     assert.deepEqual(recovered.releaseReferences, [{releaseId: "v1", releasePath: v1Path}])
-    assert.equal(isAlive(candidatePid), false, "uncommitted candidate must be stopped before replacement becomes healthy")
+    assert.equal(isProcessRunning(candidatePid), false, "uncommitted candidate must be stopped before replacement becomes healthy")
 
     await sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})
     assert.equal((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId, "v2", "removed candidate keys must be reusable by a later valid deploy")
@@ -1067,7 +1543,7 @@ test("replacement removes only guardian-owned candidate inventory left before de
     await once(owner, "exit")
   } finally {
     await killChild(owner)
-    if (candidatePid && isAlive(candidatePid)) {
+    if (candidatePid && isProcessRunning(candidatePid)) {
       try { process.kill(-candidatePid, "SIGKILL") } catch (_error) { /* Exact candidate group already exited. */ }
     }
     await stopFixtureGuardian(fixture.statePath)
@@ -1075,6 +1551,47 @@ test("replacement removes only guardian-owned candidate inventory left before de
   }
 })
 
+test("ensure-daemon replaces a same-authority owner whose control socket disappeared", async () => {
+  const fixture = await createFixture()
+  const runtimePath = path.join(fixture.root, "runtime")
+  const daemonLogPath = path.join(fixture.root, "same-authority-replacement.log")
+  const daemonPidPath = path.join(fixture.root, "same-authority-replacement.pid")
+  const v1Path = await prepareRelease(fixture.root, "v1")
+  const ensureArgs = [
+    "ensure-daemon", "--config", fixture.configPath,
+    "--daemon-log-path", daemonLogPath,
+    "--daemon-pid-path", daemonPidPath,
+    "--daemon-runtime-path", runtimePath,
+    "--daemon-start-timeout-ms", "5000"
+  ]
+
+  try {
+    const first = await runCli(ensureArgs)
+
+    assert.equal(first.code, 0, first.output)
+    await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})
+    const before = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
+
+    await fs.rm(fixture.socketPath, {force: true})
+    const replacement = await runCli(ensureArgs)
+
+    assert.equal(replacement.code, 0, `${replacement.output}\n${await fs.readFile(daemonLogPath, "utf8")}`)
+    const after = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
+
+    assert.equal(after.activeReleaseId, "v1")
+    assert.notEqual(after.daemonPid, before.daemonPid)
+    await waitForProcessExit(before.daemonPid)
+
+    const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
+
+    await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
+    await shutdown
+  } finally {
+    await stopFixtureGuardian(fixture.statePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
 test("ensure-daemon atomically replaces an incompatible owner without losing retained generations", async () => {
   const fixture = await createFixture()
   const oldControlPath = fixture.socketPath
@@ -1137,6 +1654,16 @@ test("ensure-daemon atomically replaces an incompatible owner without losing ret
     assert.equal(after.services[0]?.process.pid, before.services[0]?.process.pid)
     assert.equal(after.singletons[0]?.process.pid, before.singletons[0]?.process.pid)
     assert.equal(retainedConnectionClosed, false, "listener-owned WebSocket must remain supervised across replacement")
+    assert.ok(after.daemonPid)
+    await fs.writeFile(daemonLogPath, "")
+    process.kill(after.daemonPid, "SIGKILL")
+    const restartedState = await waitForState(fixture.statePath, (state) => state.daemonPid !== after.daemonPid, AbortSignal.timeout(5000))
+    const restarted = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: newControlPath}))
+
+    assert.equal(restarted.daemonPid, restartedState.daemonPid)
+    assert.equal(Number((await fs.readFile(daemonPidPath, "utf8")).trim()), restarted.daemonPid)
+    assert.deepEqual(restarted.releases.map((release) => release.processes.map((processStatus) => processStatus.pid)), before.releases.map((release) => release.processes.map((processStatus) => processStatus.pid)))
+    assert.match(await fs.readFile(daemonLogPath, "utf8"), /owner state recovered/)
 
     const v3Path = await prepareRelease(fixture.root, "v3")
     await sendControlCommand({command: {command: "deploy", releaseId: "v3", releasePath: v3Path, revision: "v3"}, path: newControlPath})
@@ -1380,10 +1907,11 @@ async function stopFixtureGuardian(statePath) {
 /**
  * @param {string} statePath - State path.
  * @param {(state: RecoveryState) => boolean} predicate - Completion predicate.
+ * @param {AbortSignal} [signal] - Optional deadline signal.
  * @returns {Promise<RecoveryState>} Matching state.
  */
-async function waitForState(statePath, predicate) {
-  const watcher = fs.watch(path.dirname(statePath))
+async function waitForState(statePath, predicate, signal) {
+  const watcher = fs.watch(path.dirname(statePath), {signal})
 
   try {
     const initial = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(statePath, "utf8")))
@@ -1437,17 +1965,6 @@ async function waitForFile(filePath, timeoutMs) {
   }
 }
 
-/**
- * @param {number} pid - Exact fixture process.
- * @param {number} timeoutMs - Bounded exit wait.
- */
-async function waitForProcessExit(pid, timeoutMs) {
-  const deadline = Date.now() + timeoutMs
-
-  while (isAlive(pid) && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 10))
-  assert.equal(isAlive(pid), false, `process ${pid} did not exit within ${timeoutMs}ms`)
-}
-
 /**
  * Opens a live WebSocket through the fixture proxy.
  * @param {number} port - Proxy port.
@@ -1485,29 +2002,18 @@ function releaseProcessPid(status, releaseId, processId) {
   return pid
 }
 
-/**
- * @param {number} pid - Exact fixture pid.
- * @returns {boolean} Whether the exact fixture process is alive.
- */
-function isAlive(pid) {
-  try {
-    process.kill(pid, 0)
-    return true
-  } catch (error) {
-    if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") return false
-    throw error
-  }
-}
-
 /**
  * @param {string} configPath - Config path.
  * @param {{releaseId: string, releasePath: string, revision: string}} [bootstrap] - Optional bootstrap tuple.
+ * @param {{daemonPidPath?: string, startupTimeoutMs?: number}} [options] - Recovery command options.
  * @returns {import("node:child_process").ChildProcess} Daemon process.
  */
-function spawnDaemon(configPath, bootstrap) {
+function spawnDaemon(configPath, bootstrap, options = {}) {
   const args = [binPath, "daemon", "--config", configPath]
 
   if (bootstrap) args.push("--release-id", bootstrap.releaseId, "--release-path", bootstrap.releasePath, "--revision", bootstrap.revision)
+  if (options.daemonPidPath) args.push("--guardian-daemon-pid-path", options.daemonPidPath)
+  if (options.startupTimeoutMs) args.push("--guardian-daemon-start-timeout-ms", String(options.startupTimeoutMs))
   return spawn(process.execPath, args, {stdio: ["ignore", "pipe", "pipe"]})
 }
 
@@ -1554,14 +2060,16 @@ async function runCli(args) {
 /**
  * @param {import("node:child_process").ChildProcess} child - Daemon child.
  * @param {string} message - Structured log message.
+ * @param {{allowChildExit?: boolean}} [options] - Whether inherited descriptors may outlive the original child.
  */
-async function waitForLog(child, message) {
+async function waitForLog(child, message, {allowChildExit = false} = {}) {
   assert.ok(child.stdout)
   child.stdout.setEncoding("utf8")
 
   await new Promise((resolve, reject) => {
     let buffer = ""
     let stderr = ""
+    const timer = setTimeout(() => finish(new Error(`Timed out waiting for daemon log ${message}: ${stderr.trim()}`)), 5000)
     const onErrorData = (/** @type {string} */ chunk) => { stderr += chunk }
     const onExit = () => finish(new Error(`Daemon exited before logging ${message}: ${stderr.trim()}`))
     /** @param {string} chunk - Output chunk. */
@@ -1579,6 +2087,7 @@ async function waitForLog(child, message) {
     }
     /** @param {Error} [error] - Failure. */
     const finish = (error) => {
+      clearTimeout(timer)
       child.off("exit", onExit)
       child.stdout?.off("data", onData)
       child.stderr?.off("data", onErrorData)
@@ -1586,7 +2095,7 @@ async function waitForLog(child, message) {
       else resolve(undefined)
     }
 
-    child.once("exit", onExit)
+    if (!allowChildExit) child.once("exit", onExit)
     child.stdout?.on("data", onData)
     child.stderr?.setEncoding("utf8").on("data", onErrorData)
   })