rollbridge 0.1.39 → 0.1.41

package/test/release-group.test.js

@@ -7,9 +7,10 @@ import {normalizeConfig} from "../src/config.js"
 
 /**
  * @param {import("../src/json.js").JsonValue} webProcess - The single proxied process definition.
+ * @param {() => boolean} [shouldStart] - Whether process starts remain allowed.
  * @returns {ReleaseGroup} A release group ready for buildProcess.
  */
-function buildRelease(webProcess) {
+function buildRelease(webProcess, shouldStart = () => true) {
   const config = normalizeConfig({
     application: "demo",
     control: {path: "/tmp/rollbridge-release-group.sock"},
@@ -17,7 +18,7 @@ function buildRelease(webProcess) {
     proxy: {host: "127.0.0.1", port: 0}
   })
 
-  return new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1"})
+  return new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1", shouldStart})
 }
 
 test("templates interpolate values from the daemon environment", () => {
@@ -78,3 +79,19 @@ test("a referenced daemon environment variable that is unset fails fast", () =>
     /Missing template value for \{\{env.ROLLBRIDGE_ENV_MISSING\}\}/
   )
 })
+
+test("committed generation restoration does not start after shutdown begins", async () => {
+  const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => false)
+  const process = release.buildProcess(release.config.processes[0])
+  let starts = 0
+
+  release.state = "draining"
+  process.start = async () => {
+    starts += 1
+    throw new Error("process started after shutdown")
+  }
+  release.processes.set("web", process)
+
+  await assert.rejects(() => release.restartCommittedGeneration(), /shutting down/)
+  assert.equal(starts, 0)
+})

package/test/release-runtime-retention.test.js

@@ -10,6 +10,7 @@ import path from "node:path"
 import test from "node:test"
 import {fileURLToPath} from "node:url"
 import {sendControlCommand} from "../src/control-client.js"
+import GuardianClient from "../src/guardian-client.js"
 
 const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
 const dummyAppPath = path.join(repoRoot, "test", "fixtures", "dummy-app.js")
@@ -23,12 +24,14 @@ test("detached daemon survives deletion of the release-local Rollbridge installa
   const logPath = path.join(root, "daemon.log")
   const pidPath = path.join(root, "daemon.pid")
   const runtimePath = path.join(root, "runtime")
+  const statePath = path.join(root, "state.json")
 
   try {
     await Promise.all([prepareRelease(releaseA, true), prepareRelease(releaseB, true)])
     await fs.writeFile(configPath, `export default ${JSON.stringify({
       application: "runtime-retention-test",
       control: {path: socketPath},
+      ownerRecovery: {reconnectGraceMs: 50},
       processes: [{
         command: `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`,
         health: {intervalMs: 25, path: "/ping", timeoutMs: 3000},
@@ -36,7 +39,8 @@ test("detached daemon survives deletion of the release-local Rollbridge installa
         policy: "proxied",
         port: {from: 0, to: 0}
       }],
-      proxy: {drainTimeoutMs: 100, forceStopTimeoutMs: 100, host: "127.0.0.1", port: 0}
+      proxy: {drainTimeoutMs: 100, forceStopTimeoutMs: 100, host: "127.0.0.1", port: 0},
+      statePath
     }, null, 2)}\n`)
 
     await runReleaseCli(releaseA, [
@@ -52,7 +56,7 @@ test("detached daemon survives deletion of the release-local Rollbridge installa
       "--daemon-runtime-path", runtimePath
     ])
 
-    await fs.rm(path.join(releaseA, "node_modules"), {recursive: true})
+    await fs.rm(releaseA, {recursive: true})
 
     const status = await sendControlCommand({command: {command: "status"}, path: socketPath})
     const proxyPort = /** @type {{port: number}} */ (status.proxy).port
@@ -66,14 +70,67 @@ test("detached daemon survives deletion of the release-local Rollbridge installa
     assert.ok(!runtime.path.startsWith(releaseA), `runtime must be outside release A: ${runtime.path}`)
     assert.equal(response.status, 200)
     assert.equal(await response.text(), "deferred runtime loaded\n")
+    assert.equal(typeof status.daemonPid, "number")
+    const daemonPid = /** @type {number} */ (status.daemonPid)
+
+    process.kill(daemonPid, "SIGKILL")
+    const recoveredPid = await waitForChangedPid(pidPath, daemonPid)
+    const recovered = await sendControlCommand({command: {command: "status"}, path: socketPath})
+
+    assert.equal(recovered.daemonPid, recoveredPid)
+    assert.equal(recovered.activeReleaseId, "B")
   } finally {
     try {
       await sendControlCommand({command: {command: "shutdown"}, path: socketPath})
     } catch {
       const pid = Number.parseInt(await fs.readFile(pidPath, "utf8").catch(() => ""), 10)
-      if (Number.isInteger(pid)) process.kill(pid, "SIGKILL")
+      if (Number.isInteger(pid)) killProcessIfAlive(pid)
     }
+    await stopGuardian(statePath)
+
+    await fs.rm(root, {force: true, recursive: true})
+  }
+})
+
+test("ensure-daemon resolves an explicit relative config before changing to its durable directory", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-runtime-relative-config-"))
+  const release = path.join(root, "release")
+  const configDirectory = path.join(release, "configs")
+  const configPath = path.join(configDirectory, "rollbridge.js")
+  const pidPath = path.join(root, "daemon.pid")
+  const socketPath = path.join(root, "control.sock")
+  const statePath = path.join(root, "state.json")
+
+  try {
+    await prepareRelease(release, false)
+    await fs.mkdir(configDirectory)
+    await fs.writeFile(configPath, `export default ${JSON.stringify({
+      ...basicConfig(path.relative(configDirectory, socketPath)),
+      ownerRecovery: {reconnectGraceMs: 50},
+      processes: [{
+        command: `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`,
+        health: {intervalMs: 25, path: "/ping", timeoutMs: 3000},
+        id: "web",
+        policy: "proxied",
+        port: {from: 0, to: 0}
+      }],
+      statePath: path.relative(configDirectory, statePath)
+    }, null, 2)}\n`)
+    await runReleaseCli(release, [
+      "deploy", "--ensure-daemon", "--config", path.relative(release, configPath),
+      "--release-path", release, "--release-id", "relative-config",
+      "--daemon-log-path", path.join(root, "daemon.log"), "--daemon-pid-path", pidPath,
+      "--daemon-runtime-path", path.join(root, "runtime"), "--daemon-start-timeout-ms", "1000"
+    ])
+    const status = await sendControlCommand({command: {command: "status"}, path: socketPath})
+
+    assert.equal(status.activeReleaseId, "relative-config")
+  } finally {
+    await sendControlCommand({command: {command: "shutdown"}, path: socketPath}).catch(() => undefined)
+    await stopGuardian(statePath)
+    const pid = Number.parseInt(await fs.readFile(pidPath, "utf8").catch(() => ""), 10)
 
+    if (Number.isInteger(pid)) killProcessIfAlive(pid)
     await fs.rm(root, {force: true, recursive: true})
   }
 })
@@ -349,7 +406,7 @@ function basicConfig(socketPath) {
  */
 async function runReleaseCli(releasePath, args) {
   const binPath = path.join(releasePath, "node_modules", "rollbridge", "bin", "rollbridge")
-  const child = spawn(process.execPath, [binPath, ...args], {stdio: ["ignore", "pipe", "pipe"]})
+  const child = spawn(process.execPath, [binPath, ...args], {cwd: releasePath, stdio: ["ignore", "pipe", "pipe"]})
   let output = ""
 
   child.stdout.setEncoding("utf8").on("data", (chunk) => { output += chunk })
@@ -358,3 +415,63 @@ async function runReleaseCli(releasePath, args) {
 
   if (code !== 0) throw new Error(output)
 }
+
+/**
+ * @param {string} pidPath - Daemon PID file.
+ * @param {number} previousPid - Exited daemon PID.
+ * @returns {Promise<number>} Recovered daemon PID.
+ */
+async function waitForChangedPid(pidPath, previousPid) {
+  const deadline = Date.now() + 5000
+
+  while (Date.now() < deadline) {
+    const pid = Number.parseInt(await fs.readFile(pidPath, "utf8").catch(() => ""), 10)
+
+    if (Number.isInteger(pid) && pid !== previousPid) return pid
+    await new Promise((resolve) => setTimeout(resolve, 25))
+  }
+  throw new Error(`Timed out waiting for ${pidPath} to publish a recovered daemon PID`)
+}
+
+/** @param {number} pid - Exact fixture process PID. */
+function killProcessIfAlive(pid) {
+  try {
+    process.kill(pid, "SIGKILL")
+  } catch (error) {
+    if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ESRCH") throw error
+  }
+}
+
+/** @param {string} statePath - Owner-recovery state path. */
+async function stopGuardian(statePath) {
+  try {
+    const state = JSON.parse(await fs.readFile(statePath, "utf8"))
+    const identity = state.recovery?.guardian
+    const pid = identity?.pid
+
+    if (identity?.socketPath && identity.token) {
+      const client = new GuardianClient(identity)
+
+      try {
+        await client.connect()
+        for (const entry of await client.inventory()) {
+          if (entry.status.pid) {
+            try { process.kill(-entry.status.pid, "SIGKILL") } catch (error) {
+              if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ESRCH") throw error
+            }
+          }
+        }
+      } finally {
+        client.disconnect()
+      }
+    }
+    if (typeof pid === "number") {
+      const command = (await fs.readFile(`/proc/${pid}/cmdline`, "utf8")).replaceAll("\0", " ")
+
+      if (!command.includes("process-guardian.js") || !command.includes(statePath)) throw new Error(`Refusing to stop unverified fixture guardian pid ${pid}`)
+      process.kill(pid, "SIGKILL")
+    }
+  } catch (error) {
+    if (!error || typeof error !== "object" || !("code" in error) || !["ENOENT", "ESRCH"].includes(String(error.code))) throw error
+  }
+}

package/test/rollbridge.test.js

@@ -525,6 +525,27 @@ test("opt-in generation lifecycle retires the old generation before activating a
   }
 })
 
+test("manual restart reaches the active handoff coordinator and restores its lifecycle role", async () => {
+  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, webDependsOnService: true})
+  const daemon = await startDaemon(fixture.config)
+
+  try {
+    await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
+    const before = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "beacon")?.pid
+    const result = await daemon.restartProcesses({processId: "beacon"})
+    const after = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "beacon")?.pid
+
+    assert.deepEqual(result, {restarted: ["beacon"]})
+    assert.ok(before)
+    assert.ok(after)
+    assert.notEqual(after, before)
+    assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "activate:v1"])
+  } finally {
+    await daemon.shutdown()
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
 test("generation commit is durable before awaited post-transition work", async () => {
   const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, includeSingleton: true, webDependsOnService: true})
   const daemon = await startDaemon(fixture.config)
@@ -639,25 +660,248 @@ test("retirement failure retains the exact transition, blocks other deploys, and
   }
 })
 
-test("candidate activation failure never reactivates old and exact resume activates only the candidate", async () => {
+test("candidate activation failure reports restoration failure and exact recovery clears the fence", async () => {
   const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
   const daemon = await startDaemon(fixture.config)
 
   try {
     await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
-    await assert.rejects(() => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}), /activate command exited non-zero/)
+    const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
+    const reactivate = incumbentCoordinator?.reactivateStrict.bind(incumbentCoordinator)
+
+    assert.ok(incumbentCoordinator && reactivate)
+    incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
+    await assert.rejects(
+      () => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
+      error => {
+        assert.ok(error instanceof AggregateError)
+        assert.match(error.message, /activate command exited non-zero/)
+        assert.match(error.message, /incumbent v1 restoration failed: incumbent restoration rejected/i)
+        return true
+      }
+    )
 
     const failed = daemon.status()
 
     assert.equal(failed.activeReleaseId, "v1")
-    assert.equal(failed.generationTransition?.phase, "activating_candidate")
-    assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1"])
+    assert.equal(failed.generationTransition?.phase, "restoring_previous")
+    assert.match(String(failed.generationTransition?.activationError), /activate command exited non-zero/)
+    assert.match(String(failed.generationTransition?.compensationError), /incumbent restoration rejected/)
+    const failedEvents = daemon.eventLog.recent()
+    const activationEvent = failedEvents.find((event) => event.message === "release generation activation failed")
+    const restorationEvent = failedEvents.find((event) => event.message === "release generation compensation restoration failed")
+
+    assert.match(String(activationEvent?.data.error), /activate command exited non-zero/)
+    assert.match(String(restorationEvent?.data.activationError), /activate command exited non-zero/)
+    assert.match(String(restorationEvent?.data.error), /incumbent restoration rejected/)
+    assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "retire:v2"])
+    await assert.rejects(() => daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"}), /transition.*v2.*unresolved/i)
 
-    await fs.writeFile(fixture.activationGatePath, "allow\n")
-    await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
+    incumbentCoordinator.reactivateStrict = reactivate
+    const recovery = await sendControlCommand({
+      command: {
+        command: "recover-generation-transition",
+        previousReleaseId: "v1",
+        releaseId: "v2",
+        releasePath: fixture.root,
+        revision: "v2"
+      },
+      path: fixture.config.control.path
+    })
 
-    assert.equal(daemon.status().activeReleaseId, "v2")
-    assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "activate:v2"])
+    assert.equal(recovery.recoveryStatus, "recovered")
+    assert.equal(daemon.status().activeReleaseId, "v1")
+    assert.equal(daemon.status().generationTransition, undefined)
+    const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
+
+    assert.equal(persisted?.generationTransition, undefined)
+    assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
+
+    const idempotent = await sendControlCommand({
+      command: {
+        command: "recover-generation-transition",
+        previousReleaseId: "v1",
+        releaseId: "v2",
+        releasePath: fixture.root,
+        revision: "v2"
+      },
+      path: fixture.config.control.path
+    })
+
+    assert.equal(idempotent.recoveryStatus, "already_recovered")
+    await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
+    await assert.rejects(
+      () => sendControlCommand({
+        command: {
+          command: "recover-generation-transition",
+          previousReleaseId: "v1",
+          releaseId: "v3",
+          releasePath: fixture.root,
+          revision: "v3"
+        },
+        path: fixture.config.control.path
+      }),
+      /not a safe failed pre-commit transition/i
+    )
+  } finally {
+    await daemon.shutdown()
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("candidate activation failure compensates to the incumbent and admits a different later release", async () => {
+  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
+  const daemon = await startDaemon(fixture.config)
+
+  try {
+    await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
+    await assert.rejects(
+      () => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
+      /activate command exited non-zero.*compensation restored incumbent v1 as authoritative and retired failed candidate v2/i
+    )
+
+    const compensated = daemon.status()
+
+    assert.equal(compensated.activeReleaseId, "v1")
+    assert.equal(compensated.generationTransition, undefined)
+    assert.equal(await fetchText(daemon, "/release"), "v1")
+    assert.equal(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state, "running")
+    assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
+
+    await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
+
+    assert.equal(daemon.status().activeReleaseId, "v3")
+    assert.equal(await fetchText(daemon, "/release"), "v3")
+  } finally {
+    await daemon.shutdown()
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("ambiguous candidate activation retires the candidate before reactivating the incumbent", async () => {
+  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateAmbiguousFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
+  const daemon = await startDaemon(fixture.config)
+
+  try {
+    await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
+    await assert.rejects(
+      () => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
+      /activate command exited non-zero.*compensation restored incumbent v1 as authoritative and retired failed candidate v2/i
+    )
+
+    assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), [
+      "activate:v1",
+      "retire:v1",
+      "activate:v2",
+      "retire:v2",
+      "activate:v1"
+    ])
+    assert.equal(daemon.status().activeReleaseId, "v1")
+    assert.equal(daemon.status().generationTransition, undefined)
+  } finally {
+    await daemon.shutdown()
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("candidate activation recovery reverses a worker-specific quiet hook before reporting active", async () => {
+  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true, workerReactivationLifecycle: true})
+  const daemon = await startDaemon(fixture.config)
+
+  try {
+    await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
+    await assert.rejects(
+      () => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
+      /compensation restored incumbent v1 as authoritative/i
+    )
+
+    const events = await lifecycleEvents(fixture.lifecycleLogPath)
+    const candidateRetired = events.indexOf("worker-retire:v2")
+    const workerReactivated = events.indexOf("worker-reactivate:v1")
+
+    assert.ok(candidateRetired >= 0, JSON.stringify(events))
+    assert.ok(workerReactivated > candidateRetired, JSON.stringify(events))
+    assert.equal(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state, "running")
+    assert.equal(daemon.status().activeReleaseId, "v1")
+    assert.equal(daemon.status().generationTransition, undefined)
+  } finally {
+    await daemon.shutdown()
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("candidate activation recovery keeps the fence when a worker-specific resume hook fails", async () => {
+  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true, workerReactivationFailure: true, workerReactivationLifecycle: true})
+  const daemon = await startDaemon(fixture.config)
+
+  try {
+    await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
+    await assert.rejects(
+      () => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
+      error => {
+        const failure = /** @type {Error} */ (error)
+
+        assert.match(failure.message, /activate command exited non-zero/)
+        assert.match(failure.message, /reactivate command exited non-zero/)
+        return true
+      }
+    )
+
+    const status = daemon.status()
+    const restorationEvent = daemon.eventLog.recent().find((event) => event.message === "release generation compensation restoration failed")
+
+    assert.equal(status.activeReleaseId, "v1")
+    assert.equal(status.generationTransition?.phase, "restoring_previous")
+    assert.match(String(status.generationTransition?.activationError), /activate command exited non-zero/)
+    assert.match(String(status.generationTransition?.compensationError), /reactivate command exited non-zero/)
+    assert.equal(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state, "quiesced")
+    assert.match(String(restorationEvent?.data.activationError), /activate command exited non-zero/)
+    assert.match(String(restorationEvent?.data.error), /reactivate command exited non-zero/)
+    await assert.rejects(() => daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"}), /transition.*v2.*unresolved/i)
+  } finally {
+    await daemon.shutdown()
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("compensation keeps the fence when the cleared checkpoint cannot be persisted", async () => {
+  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
+  const daemon = await startDaemon(fixture.config)
+
+  try {
+    await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
+    const checkpoint = daemon.checkpointGenerationTransition.bind(daemon)
+
+    daemon.checkpointGenerationTransition = async () => {
+      if (!daemon.generationTransition) throw new Error("cleared checkpoint unavailable")
+      await checkpoint()
+    }
+    await assert.rejects(
+      () => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
+      /activate command exited non-zero.*compensation checkpoint clear failed: cleared checkpoint unavailable/i
+    )
+
+    assert.equal(daemon.status().activeReleaseId, "v1")
+    assert.equal(daemon.status().generationTransition?.phase, "restoring_previous")
+    const persisted = /** @type {{generationTransition?: {phase?: string}} | undefined} */ (await readState(fixture.statePath))
+
+    assert.equal(persisted?.generationTransition?.phase, "restoring_previous")
+    await assert.rejects(() => daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"}), /transition.*v2.*unresolved/i)
+
+    daemon.checkpointGenerationTransition = checkpoint
+    const recovery = await sendControlCommand({
+      command: {
+        command: "recover-generation-transition",
+        previousReleaseId: "v1",
+        releaseId: "v2",
+        releasePath: fixture.root,
+        revision: "v2"
+      },
+      path: fixture.config.control.path
+    })
+
+    assert.equal(recovery.recoveryStatus, "recovered")
+    assert.equal(daemon.status().generationTransition, undefined)
   } finally {
     await daemon.shutdown()
     await fs.rm(fixture.root, {force: true, recursive: true})
@@ -670,6 +914,10 @@ test("unresolved generation transition fences stop, restart, and rollback mutati
 
   try {
     await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
+    const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
+
+    assert.ok(incumbentCoordinator)
+    incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
     await assert.rejects(() => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}), /activate command exited non-zero/)
 
     await assert.rejects(() => daemon.stopRelease("v2"), /cannot stop.*generation transition.*unresolved/i)
@@ -677,9 +925,7 @@ test("unresolved generation transition fences stop, restart, and rollback mutati
     await assert.rejects(() => daemon.rollback({releaseId: "v2"}), /cannot rollback.*generation transition.*unresolved/i)
 
     assert.notEqual(statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state, "stopped")
-    await fs.writeFile(fixture.activationGatePath, "allow\n")
-    await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
-    assert.equal(await fetchText(daemon, "/release"), "v2")
+    assert.equal(await fetchText(daemon, "/release"), "v1")
   } finally {
     await daemon.shutdown()
     await fs.rm(fixture.root, {force: true, recursive: true})
@@ -1633,7 +1879,7 @@ test("deploy can ensure the daemon before sending the release command", async ()
 })
 
 /**
- * @param {{companionReplicas?: number, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerStopDelayMs?: number}} [options] - Fixture options.
+ * @param {{companionReplicas?: number, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
  * @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
  */
 async function createFixture(options = {}) {
@@ -1651,7 +1897,7 @@ async function createFixture(options = {}) {
   if (options.includeService || options.handoffService) {
     const activationFailureRelease = typeof options.handoffServiceActivateFailure === "string" ? options.handoffServiceActivateFailure : "v2"
     const lifecycle = options.handoffServiceActivate ? {
-      activateCommand: `${options.handoffServiceActivateFailure ? `[ "$ROLLBRIDGE_RELEASE_ID" != ${JSON.stringify(activationFailureRelease)} ] || [ -f ${JSON.stringify(activationGatePath)} ] || exit 24; ` : ""}printf 'activate:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`,
+      activateCommand: `${options.handoffServiceActivateFailure && !options.handoffServiceActivateAmbiguousFailure ? `[ "$ROLLBRIDGE_RELEASE_ID" != ${JSON.stringify(activationFailureRelease)} ] || [ -f ${JSON.stringify(activationGatePath)} ] || exit 24; ` : ""}printf 'activate:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}${options.handoffServiceActivateAmbiguousFailure ? `; [ "$ROLLBRIDGE_RELEASE_ID" != ${JSON.stringify(activationFailureRelease)} ] || [ -f ${JSON.stringify(activationGatePath)} ] || exit 24` : ""}`,
       quietCommand: `${options.handoffServiceQuietFailure ? `[ -f ${JSON.stringify(retirementGatePath)} ] || exit 23; ` : ""}printf 'retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`
     } : options.handoffServiceQuiet || options.handoffServiceQuietFailure ? {
       quietCommand: options.handoffServiceQuietFailure ? "exit 23" : `printf '%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(serviceQuietPath)}`
@@ -1692,6 +1938,10 @@ async function createFixture(options = {}) {
     processes.push({
       command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`process.on('SIGTERM', () => setTimeout(() => process.exit(0), ${options.workerStopDelayMs || 0})); setInterval(() => {}, 1000)`)}`,
       id: "worker",
+      ...(options.workerReactivationLifecycle ? {lifecycle: {
+        quietCommand: `printf 'worker-retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`,
+        reactivateCommand: `${options.workerReactivationFailure ? "exit 25; " : ""}printf 'worker-reactivate:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`
+      }} : {}),
       nonBlockingDrain: true,
       policy: "companion"
     })

package/test/support/process.js

@@ -0,0 +1,41 @@
+// @ts-check
+
+import fs from "node:fs"
+
+/**
+ * Reports whether an exact fixture process can still run. Linux keeps exited children in
+ * procfs until they are reaped, so kill(2) alone cannot distinguish a zombie from a live process.
+ * @param {number} pid - Exact fixture process PID.
+ * @returns {boolean} Whether the process can still run.
+ */
+export function isProcessRunning(pid) {
+  try {
+    process.kill(pid, 0)
+  } catch (error) {
+    if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") return false
+    throw error
+  }
+
+  if (process.platform !== "linux") return true
+
+  try {
+    const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8")
+    const state = stat.slice(stat.lastIndexOf(")") + 2).split(" ")[0]
+
+    return state !== "Z" && state !== "X"
+  } catch (error) {
+    if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return false
+    throw error
+  }
+}
+
+/**
+ * @param {number} pid - Exact fixture process PID.
+ * @param {number} [timeoutMs] - Bounded exit wait.
+ */
+export async function waitForProcessExit(pid, timeoutMs = 5000) {
+  const deadline = Date.now() + timeoutMs
+
+  while (isProcessRunning(pid) && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 10))
+  if (isProcessRunning(pid)) throw new Error(`Timed out waiting for process ${pid} to exit`)
+}