rollbridge 0.1.45 → 0.1.46

package/changelog.d/20260901-accept-retired-incumbent-recovery.md

@@ -0,0 +1,8 @@
+### Fixed
+
+- Add explicit `recover-generation-transition --accept-retired-incumbent`
+  recovery that stops the exact failed candidate and durably fences degraded
+  incumbent web authority across owner recovery without reactivating either jobs
+  generation. A fresh deployment replaces the fence through normal cutover.
+  Incompatible owner replacement now accepts the unresolved transition's exact
+  retained candidate config without admitting unrelated config authority.

package/docs/cli.md

@@ -207,6 +207,20 @@ instruction; see [Config reloads](config.md#config-reloads).
 - `--ensure-daemon` — start the daemon first if it isn't running (honors the
   same `--daemon-*` options as `ensure-daemon`).
 
+## `recover-generation-transition`
+
+`rollbridge recover-generation-transition --release-path <path> --release-id <id>
+--revision <sha> --previous-release-id <id> [--config <path>]
+[--accept-retired-incumbent]`
+
+By default, restores the incumbent before clearing an exact failed transition.
+`--accept-retired-incumbent` instead requires an exact `restoring_previous`
+journal, terminal restoration failure, retired candidate, retired incumbent
+coordinator, and live incumbent proxy processes. It safely stops the failed
+candidate and persists a `degraded_active` fence without reactivating either jobs
+generation, reporting `jobsStatus: "degraded"`. Incumbent web survives owner
+recovery, and a fresh normal deployment replaces the fence through normal cutover.
+
 ## `rollback`
 
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rollbridge",
-  "version": "0.1.45",
+  "version": "0.1.46",
   "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
   "keywords": [
     "deploy",

package/src/cli.js

@@ -215,17 +215,19 @@ export async function runCli(argv) {
 
   program
     .command("recover-generation-transition")
-    .description("Restore the exact incumbent and retire a failed pre-commit generation candidate.")
+    .description("Recover an exact failed pre-commit generation transition.")
     .option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
     .requiredOption("--release-path <path>", "Failed candidate release path")
     .requiredOption("--release-id <id>", "Failed candidate release id")
     .requiredOption("--revision <sha>", "Failed candidate revision")
     .requiredOption("--previous-release-id <id>", "Expected authoritative incumbent release id")
+    .option("--accept-retired-incumbent", "Persist terminal restoration as degraded incumbent web authority")
     .action(async (options) => {
       const configPath = await resolveConfigPath(options.config)
       const config = await loadConfig(configPath)
       const response = await sendControlCommand({
         command: {
+          acceptRetiredIncumbent: options.acceptRetiredIncumbent === true,
           command: "recover-generation-transition",
           previousReleaseId: options.previousReleaseId,
           releaseId: options.releaseId,

package/src/daemon.js

@@ -24,7 +24,7 @@ const STATE_PERSIST_INTERVAL_MS = 5000
  * @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
  * @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
  * @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
- * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
+ * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
  * @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
  * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
  * @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
@@ -127,7 +127,7 @@ export default class RollbridgeDaemon {
     const transition = this.generationTransition
     const recoveredReplacementProxy = this.guardian ? await this.finalizeRecoveredOwnerReplacement() : false
 
-    if (this.guardian && transition && transition.phase !== "committed" && !transition.error) {
+    if (this.guardian && transition && transition.phase !== "committed" && transition.phase !== "degraded_active" && !transition.error) {
       this.generationTransitionRecovery = true
       try {
         await this.executeOwnerMutation("recover generation transition", async () => {
@@ -505,10 +505,8 @@ export default class RollbridgeDaemon {
       if (!transfer?.config || !transfer.snapshot) throw new Error("Committed owner published incomplete replacement state")
       const unresolvedTransition = transfer.snapshot.generationTransition
 
-      const transferredTransitionConfigDigest = transfer.config ? ownerConfigDigest(transfer.config) : undefined
       if (unresolvedTransition && unresolvedTransition.phase !== "committed" &&
-        unresolvedTransition.configDigest !== this.ownerRecoveryConfigDigest() &&
-        unresolvedTransition.configDigest !== transferredTransitionConfigDigest) {
+        !ownerReplacementTransitionAuthorityMatches(transfer, unresolvedTransition, this.ownerRecoveryConfigDigest())) {
         throw new Error(`Owner replacement cannot change config authority while unresolved generation transition ${unresolvedTransition.candidateReleaseId} remains at ${unresolvedTransition.phase}`)
       }
       const registeredProcesses = new Map((await this.guardian.inventory()).map(({key, provenance}) => [key, provenance]))
@@ -1342,6 +1340,7 @@ export default class RollbridgeDaemon {
 
     if (commandName === "recover-generation-transition") {
       return await this.executeOwnerMutation("recover generation transition", async () => await this.recoverGenerationTransition({
+        acceptRetiredIncumbent: data.acceptRetiredIncumbent === true,
         previousReleaseId: requiredString(data.previousReleaseId, "previousReleaseId"),
         releaseId: requiredString(data.releaseId, "releaseId"),
         releasePath: requiredString(data.releasePath, "releasePath"),
@@ -1561,7 +1560,7 @@ export default class RollbridgeDaemon {
     const newReleaseId = releaseId || revision || new Date().toISOString().replace(/[^0-9]/g, "")
     const transition = this.generationTransition
 
-    if (transition && transition.phase !== "committed") {
+    if (transition && transition.phase !== "committed" && transition.phase !== "degraded_active") {
       this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
       return await this.resumeGenerationTransition()
     }
@@ -1639,6 +1638,7 @@ export default class RollbridgeDaemon {
     const transition = this.generationTransition
 
     if (!transition) throw new Error("No release generation transition to resume")
+    if (transition.phase === "degraded_active") throw new Error("Degraded incumbent authority requires a fresh deployment, not transition resume")
     if (transition.phase === "restoring_previous" || transition.phase === "retiring_failed_candidate") {
       return await this.compensatePreCommitActivationFailure()
     }
@@ -1874,7 +1874,7 @@ export default class RollbridgeDaemon {
 
   /**
    * Authenticated control recovery for one exact failed pre-commit transition.
-   * @param {{previousReleaseId: string, releaseId: string, releasePath: string, revision: string}} identity - Exact transition fence.
+   * @param {{acceptRetiredIncumbent?: boolean, previousReleaseId: string, releaseId: string, releasePath: string, revision: string}} identity - Exact transition fence.
    * @returns {Promise<Record<string, JsonValue>>} Structured recovery result.
    */
   async recoverGenerationTransition(identity) {
@@ -1894,9 +1894,12 @@ export default class RollbridgeDaemon {
     if (transition.previousReleaseId !== identity.previousReleaseId) {
       throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
     }
-    if (transition.phase !== "activating_candidate" && transition.phase !== "restoring_previous" && transition.phase !== "retiring_failed_candidate") {
+    if (transition.phase !== "activating_candidate" && transition.phase !== "restoring_previous" && transition.phase !== "retiring_failed_candidate" && transition.phase !== "degraded_active") {
       throw new Error(`Generation transition ${transition.candidateReleaseId} at ${transition.phase} is not a safe failed pre-commit transition`)
     }
+    if (transition.phase === "degraded_active" && !identity.acceptRetiredIncumbent) {
+      throw new Error("Degraded incumbent authority may only be acknowledged with --accept-retired-incumbent or replaced by a fresh deployment")
+    }
     const nextConfig = this.configPath ? await loadConfig(this.configPath) : this.config
 
     this.assertReloadCompatible(nextConfig)
@@ -1909,9 +1912,76 @@ export default class RollbridgeDaemon {
     if (this.activeRelease?.releaseId !== transition.previousReleaseId) {
       throw new Error(`Previous release ${transition.previousReleaseId} is not the authoritative proxy target`)
     }
+    if (identity.acceptRetiredIncumbent) return await this.acceptRetiredIncumbentTransition(transition)
     return await this.compensatePreCommitActivationFailure()
   }
 
+  /**
+   * Stops an exact failed candidate and persists degraded incumbent web authority until cutover.
+   * @param {GenerationTransition} transition - Exact failed transition proved by the caller.
+   * @returns {Promise<Record<string, JsonValue>>} Explicit degraded recovery result.
+   */
+  async acceptRetiredIncumbentTransition(transition) {
+    if (transition.phase !== "restoring_previous" && transition.phase !== "degraded_active") throw new Error(`Retired-incumbent recovery requires exactly restoring_previous; transition ${transition.candidateReleaseId} is at ${transition.phase}`)
+    if (!terminalRetirementFailure(transition.compensationError)) throw new Error("Retired-incumbent recovery requires a recorded restoration failure compatible with terminal retirement")
+    if (!transition.previousReleaseId) throw new Error("Retired-incumbent recovery requires a retained previous release")
+    const candidate = this.releases.get(transition.candidateReleaseId)
+    const previous = this.releases.get(transition.previousReleaseId)
+
+    if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
+    if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
+    if (candidate.releasePath !== transition.candidateReleasePath || candidate.revision !== transition.candidateRevision || ownerConfigDigest(candidate.config) !== transition.configDigest) {
+      throw new Error(`Generation transition candidate ${candidate.releaseId} does not retain its exact path, revision, and config authority`)
+    }
+    if (candidate.state !== "draining" && candidate.state !== "stopped") throw new Error(`Failed candidate ${candidate.releaseId} must be retired before accepting a retired incumbent`)
+    if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
+    const activationConfig = previous.config.processes.find((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
+    const coordinator = activationConfig ? previous.getProcesses(activationConfig.id)[0]?.process : undefined
+    if (!coordinator || coordinator.status().lifecycleRole !== "retired") throw new Error(`Previous release ${previous.releaseId} does not retain a retired generation coordinator`)
+    if (!releaseOwnsLiveProxyTraffic(previous)) throw new Error(`Previous release ${previous.releaseId} no longer owns live proxy/web traffic`)
+    await candidate.drainAndStop(candidate.config.proxy.drainTimeoutMs, candidate.config)
+    const candidateProcesses = candidate.status().processes
+
+    if (candidate.state !== "stopped" || candidateProcesses.some(({pid, state}) => pid !== undefined || (state !== "stopped" && state !== "failed"))) {
+      throw new Error(`Failed candidate ${candidate.releaseId} did not fully stop during retired-incumbent recovery`)
+    }
+    if (this.activeRelease !== previous || !releaseOwnsLiveProxyTraffic(previous)) throw new Error(`Previous release ${previous.releaseId} lost live proxy/web authority during retired-incumbent recovery`)
+    const result = /** @type {Record<string, JsonValue>} */ ({
+      activeReleaseId: previous.releaseId,
+      candidateReleaseId: candidate.releaseId,
+      failedCandidateStatus: candidate.state,
+      jobsStatus: "degraded",
+      previousReleaseId: previous.releaseId,
+      recoveryStatus: "retired_incumbent_accepted"
+    })
+
+    if (transition.phase === "degraded_active") return result
+    const previousError = transition.error
+    const previousJournalRevision = transition.journalRevision
+    const previousUpdatedAt = transition.updatedAt
+
+    transition.phase = "degraded_active"
+    transition.error = undefined
+    transition.journalRevision = (transition.journalRevision ?? 0) + 1
+    transition.updatedAt = new Date().toISOString()
+    try {
+      await this.checkpointGenerationTransition()
+    } catch (error) {
+      transition.phase = "restoring_previous"
+      transition.error = previousError
+      transition.journalRevision = previousJournalRevision
+      transition.updatedAt = previousUpdatedAt
+      try {
+        await this.publishOwnerState()
+      } catch (publishError) {
+        throw new AggregateError([error, publishError], "retired-incumbent recovery checkpoint and fence restoration failed", {cause: publishError})
+      }
+      throw new Error(`retired-incumbent recovery checkpoint failed: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
+    }
+    this.logger("retired incumbent accepted as jobs-degraded", result)
+    return result
+  }
+
   /**
    * @param {GenerationTransition} transition - Pending or committed exact transition.
    * @param {{config: import("./config.js").RollbridgeConfig, releaseId: string, releasePath: string, revision: string}} candidate - Requested identity.
@@ -2865,6 +2935,54 @@ export function ownerConfigDigest(config) {
   return crypto.createHash("sha256").update(JSON.stringify(config)).digest("hex")
 }
 
+/**
+ * @param {PrivateOwnerState} transfer - Authenticated private incumbent state.
+ * @param {GenerationTransition} transition - Unresolved generation transition.
+ * @param {string} replacementConfigDigest - Proposed replacement authority.
+ * @returns {boolean} Whether an existing exact config authority admits replacement.
+ */
+export function ownerReplacementTransitionAuthorityMatches(transfer, transition, replacementConfigDigest) {
+  const retainedRelease = transfer.snapshot.releases.find((release) =>
+    release.releaseId === transition.candidateReleaseId &&
+    release.releasePath === transition.candidateReleasePath &&
+    release.revision === transition.candidateRevision)
+  const retainedConfig = transfer.releaseConfigs?.[transition.candidateReleaseId]
+  const retainedDigest = retainedRelease && retainedConfig ? ownerConfigDigest(retainedConfig) : undefined
+
+  return transition.configDigest === replacementConfigDigest ||
+    transition.configDigest === ownerConfigDigest(transfer.config) ||
+    transition.configDigest === retainedDigest
+}
+
+/**
+ * @param {string | undefined} failure - Recorded incumbent restoration failure.
+ * @returns {boolean} Whether the diagnostic can represent terminal external retirement.
+ */
+function terminalRetirementFailure(failure) {
+  return typeof failure === "string" && (
+    /^Cannot activate .+ generation from retired$/iu.test(failure) ||
+    /^activate command exited non-zero with status \d+$/u.test(failure)
+  )
+}
+
+/**
+ * @param {ReleaseGroup} release - Expected authoritative release.
+ * @returns {boolean} Whether every configured proxy process is live.
+ */
+function releaseOwnsLiveProxyTraffic(release) {
+  const proxiedConfigs = release.config.processes.filter((processConfig) => processConfig.policy === "proxied")
+
+  return proxiedConfigs.length > 0 && proxiedConfigs.every((processConfig) => {
+    const instances = release.getProcesses(processConfig.id)
+
+    return instances.length === processConfig.replicas && instances.every(({process}) => {
+      const {pid, state} = process.status()
+
+      return pid !== undefined && state === "running"
+    })
+  })
+}
+
 /**
  * Accepts only the two exact authenticated diagnostics emitted by pre-split guardians.
  * @param {string} diagnostic - Guardian response diagnostic.

package/test/completion.test.js

@@ -44,7 +44,7 @@ test("completion bash prints a sourceable script with commands and option flags"
   assert.match(output, /compgen -W "daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion"/)
   // A command's own options are completed after the command.
   assert.match(output, /deploy\)\n\s+opts="[^"]*--release-path[^"]*"/)
-  assert.match(output, /recover-generation-transition\)\n\s+opts="--config --release-path --release-id --revision --previous-release-id"/)
+  assert.match(output, /recover-generation-transition\)\n\s+opts="--config --release-path --release-id --revision --previous-release-id --accept-retired-incumbent"/)
   assert.match(output, /ensure-daemon\)\n\s+opts="[^"]*--daemon-runtime-path[^"]*"/)
   assert.match(output, /restart\)\n\s+opts="[^"]*--policy[^"]*"/)
 })
@@ -55,7 +55,7 @@ test("completion zsh prints a #compdef script with per-command options", async (
   assert.match(output, /^#compdef rollbridge/)
   assert.match(output, /compdef _rollbridge rollbridge/)
   assert.match(output, /commands=\(daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion\)/)
-  assert.match(output, /recover-generation-transition\) compadd -- --config --release-path --release-id --revision --previous-release-id/)
+  assert.match(output, /recover-generation-transition\) compadd -- --config --release-path --release-id --revision --previous-release-id --accept-retired-incumbent/)
   assert.match(output, /events\) compadd -- [^\n]*--limit/)
 })
 

package/test/owner-replacement.test.js

@@ -11,7 +11,7 @@ import test from "node:test"
 import {fileURLToPath} from "node:url"
 import {normalizeConfig} from "../src/config.js"
 import {openControlSession, sendControlCommand} from "../src/control-client.js"
-import RollbridgeDaemon, {isLegacyGuardianPrepareDiagnostic} from "../src/daemon.js"
+import RollbridgeDaemon, {isLegacyGuardianPrepareDiagnostic, ownerConfigDigest, ownerReplacementTransitionAuthorityMatches} from "../src/daemon.js"
 import GuardianClient from "../src/guardian-client.js"
 import {findAvailablePort} from "../src/port-allocator.js"
 import {waitForProcessExit} from "./support/process.js"
@@ -1748,6 +1748,108 @@ test("config-changing owner replacement proceeds after activation compensation c
   }
 })
 
+test("owner replacement admits only the unresolved transition's exact retained candidate config", () => {
+  const exactConfig = normalizeConfig(config({controlPath: "/tmp/exact.sock", extraCompanion: false, statePath: "/tmp/exact-state.json"}))
+  const unrelatedConfig = normalizeConfig(config({controlPath: "/tmp/unrelated.sock", extraCompanion: true, statePath: "/tmp/exact-state.json"}))
+  const replacementConfig = structuredClone(unrelatedConfig)
+
+  replacementConfig.releaseRetention.keep += 1
+  const transition = /** @type {Parameters<typeof ownerReplacementTransitionAuthorityMatches>[1]} */ (/** @type {import("../src/json.js").JsonValue} */ ({
+    candidateReleaseId: "v2",
+    candidateReleasePath: "/srv/releases/v2",
+    candidateRevision: "revision-v2",
+    configDigest: ownerConfigDigest(exactConfig)
+  }))
+  /**
+   * @param {import("../src/config.js").RollbridgeConfig | undefined} candidateConfig - Candidate authority.
+   * @param {string} [releasePath] - Retained candidate path.
+   * @returns {boolean} Whether replacement is admitted.
+   */
+  const admitted = (candidateConfig, releasePath = "/srv/releases/v2") => ownerReplacementTransitionAuthorityMatches(
+    /** @type {Parameters<typeof ownerReplacementTransitionAuthorityMatches>[0]} */ (/** @type {import("../src/json.js").JsonValue} */ ({
+      config: unrelatedConfig,
+      releaseConfigs: candidateConfig ? {v2: candidateConfig} : {},
+      snapshot: {releases: [{releaseId: "v2", releasePath, revision: "revision-v2"}]}
+    })),
+    transition,
+    ownerConfigDigest(replacementConfig)
+  )
+
+  assert.equal(admitted(exactConfig), true)
+  assert.equal(admitted(unrelatedConfig), false)
+  assert.equal(admitted(exactConfig, "/srv/releases/wrong"), false)
+})
+
+test("owner replacement preserves accepted degraded incumbent web authority", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-degraded-active-"))
+  const socketPath = path.join(root, "rollbridge.sock")
+  const statePath = path.join(root, "state.json")
+  const configPath = path.join(root, "rollbridge.cjs")
+  const lifecycleLogPath = path.join(root, "generation.lifecycle")
+  const retiredMarkerPath = path.join(root, "incumbent.retired")
+  const v1Path = path.join(root, "v1")
+  const v2Path = path.join(root, "v2")
+  const proxyPort = await findAvailablePort({host: "127.0.0.1", range: {from: 24000, to: 24999}, usedPorts: new Set()})
+  let owner
+  let replacement
+  const failedConfig = () => {
+    const raw = config({activationLogPath: lifecycleLogPath, controlPath: socketPath, extraCompanion: false, proxyPort, statePath})
+    const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (raw.processes)
+    const worker = processes.find((processConfig) => processConfig.id === "worker")
+    const workerLifecycle = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (worker?.lifecycle)
+    const generation = processes.find((processConfig) => processConfig.id === "generation-main")
+    const lifecycle = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (generation?.lifecycle)
+
+    delete workerLifecycle.drainCommand
+    workerLifecycle.drainTimeoutMs = 0
+    lifecycle.activateCommand = `[ "$ROLLBRIDGE_RELEASE_ID" != v2 ] || exit 24; [ "$ROLLBRIDGE_RELEASE_ID" != v1 ] || [ ! -f ${JSON.stringify(retiredMarkerPath)} ] || exit 26; printf 'activate:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`
+    lifecycle.quietCommand = `touch ${JSON.stringify(retiredMarkerPath)}; printf 'retire:%s\\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(lifecycleLogPath)}`
+    return raw
+  }
+
+  try {
+    await Promise.all([fs.mkdir(v1Path), fs.mkdir(v2Path)])
+    await writeConfig(configPath, failedConfig())
+    owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
+    await waitForLog(owner, "control socket listening")
+    await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: socketPath})
+    await assert.rejects(sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: socketPath}), /pre-commit compensation failed.*status 26/i)
+    const before = await sendControlCommand({command: {command: "status"}, path: socketPath})
+    const webPid = releaseProcessPid(before, "v1", "web")
+
+    assert.equal(/** @type {{releaseId: string, state: string}[]} */ (before.releases).find(({releaseId}) => releaseId === "v2")?.state, "draining")
+    const recovery = await sendControlCommand({command: {
+      acceptRetiredIncumbent: true,
+      command: "recover-generation-transition",
+      previousReleaseId: "v1",
+      releaseId: "v2",
+      releasePath: v2Path,
+      revision: "v2"
+    }, path: socketPath})
+    const accepted = await sendControlCommand({command: {command: "status"}, path: socketPath})
+
+    assert.equal(recovery.jobsStatus, "degraded")
+    assert.equal(/** @type {{releaseId: string, state: string}[]} */ (accepted.releases).find(({releaseId}) => releaseId === "v2")?.state, "stopped")
+    assert.equal(/** @type {{phase?: string}} */ (accepted.generationTransition).phase, "degraded_active")
+    replacement = spawn(process.execPath, [binPath, "daemon", "--config", configPath, "--replace-owner"], {stdio: ["ignore", "pipe", "pipe"]})
+    await waitForLog(replacement, "owner replacement committed")
+    await new Promise((resolve) => setTimeout(resolve, 250))
+    const recovered = await sendControlCommand({command: {command: "status"}, path: socketPath})
+    const response = await fetch(`http://127.0.0.1:${proxyPort}/release`)
+
+    assert.equal(recovered.activeReleaseId, "v1")
+    assert.equal(/** @type {{phase?: string}} */ (recovered.generationTransition).phase, "degraded_active")
+    assert.equal(releaseProcessPid(recovered, "v1", "web"), webPid)
+    assert.equal(response.status, 200)
+    assert.equal((await response.text()).trim(), "v1")
+    assert.equal(await fs.readFile(lifecycleLogPath, "utf8"), "activate:v1\nretire:v1\nretire:v2\n")
+  } finally {
+    for (const child of [owner, replacement]) if (child && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL")
+    await stopGuardian(statePath)
+    await fs.rm(root, {force: true, recursive: true})
+  }
+})
+
 test("replacement publishes an unchanged control path only after incumbent retirement", async () => {
   const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-same-control-"))
   const socketPath = path.join(root, "rollbridge.sock")

package/test/rollbridge.test.js

@@ -757,6 +757,90 @@ test("candidate activation failure reports restoration failure and exact recover
   }
 })
 
+test("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
+  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true})
+  const daemon = await startDaemon(fixture.config)
+
+  try {
+    await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
+    const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
+
+    assert.ok(incumbentCoordinator)
+    incumbentCoordinator.reactivateStrict = async () => { throw new Error("Cannot activate background jobs generation from retired") }
+    await assert.rejects(
+      () => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
+      /Cannot activate background jobs generation from retired/i
+    )
+
+    const candidate = daemon.releases.get("v2")
+    const transition = daemon.generationTransition
+    const incumbentWebPid = statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.pid
+    const exactRecovery = (overrides = {}) => sendControlCommand({
+      command: {
+        acceptRetiredIncumbent: true,
+        command: "recover-generation-transition",
+        previousReleaseId: "v1",
+        releaseId: "v2",
+        releasePath: fixture.root,
+        revision: "v2",
+        ...overrides
+      },
+      path: fixture.config.control.path
+    })
+
+    assert.ok(candidate && transition && incumbentWebPid)
+    assert.equal(candidate.state, "draining", "ordinary failed compensation leaves the candidate draining")
+
+    const retainedCandidateConfig = candidate.config
+
+    candidate.config = {...candidate.config, releaseRetention: {...candidate.config.releaseRetention, keep: candidate.config.releaseRetention.keep + 1}}
+    await assert.rejects(() => exactRecovery(), /does not retain its exact path, revision, and config authority/i)
+    candidate.config = retainedCandidateConfig
+    await assert.rejects(() => exactRecovery({previousReleaseId: "wrong-v1"}), /refusing stale recovery/i)
+    await assert.rejects(() => exactRecovery({revision: "wrong-v2"}), /exact same release, path, revision, and config authority/i)
+    transition.phase = "retiring_failed_candidate"
+    await assert.rejects(() => exactRecovery(), /exactly restoring_previous/i)
+    transition.phase = "restoring_previous"
+    const terminalFailure = transition.compensationError
+
+    transition.compensationError = "incumbent activation was temporarily unavailable"
+    await assert.rejects(() => exactRecovery(), /terminal retirement/i)
+    transition.compensationError = terminalFailure
+    const checkpoint = daemon.checkpointGenerationTransition.bind(daemon)
+
+    daemon.checkpointGenerationTransition = async () => { throw new Error("injected checkpoint failure") }
+    await assert.rejects(() => exactRecovery(), /checkpoint failed: injected checkpoint failure/i)
+    assert.equal(daemon.generationTransition, transition)
+    daemon.checkpointGenerationTransition = checkpoint
+
+    const eventsBeforeRecovery = await lifecycleEvents(fixture.lifecycleLogPath)
+    const recovery = await exactRecovery()
+
+    assert.equal(recovery.recoveryStatus, "retired_incumbent_accepted")
+    assert.equal(recovery.jobsStatus, "degraded")
+    assert.equal(daemon.status().generationTransition?.phase, "degraded_active")
+    assert.equal(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.pid, incumbentWebPid)
+    assert.equal(await fetchText(daemon, "/release"), "v1")
+    assert.equal(candidate.state, "stopped")
+    assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), eventsBeforeRecovery, "recovery must not activate either retained generation")
+    const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
+
+    assert.equal(/** @type {{phase?: string} | undefined} */ (persisted?.generationTransition)?.phase, "degraded_active")
+    await assert.rejects(() => daemon.deploy({releaseId: "bad-v3", releasePath: fixture.root, revision: "bad-v3"}), /health check failed/i)
+    assert.equal(daemon.status().generationTransition?.phase, "degraded_active")
+    assert.equal(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.pid, incumbentWebPid)
+    assert.equal(await fetchText(daemon, "/release"), "v1")
+    await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
+    assert.equal(daemon.status().activeReleaseId, "v3")
+    assert.equal(daemon.status().generationTransition?.phase, "committed")
+    assert.equal(await fetchText(daemon, "/release"), "v3")
+    assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "retire:v2", "retire:bad-v3", "retire:v1", "activate:v3"])
+  } finally {
+    await daemon.shutdown()
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
 test("candidate activation failure compensates to the incumbent and admits a different later release", async () => {
   const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
   const daemon = await startDaemon(fixture.config)