rollbridge 0.1.73 → 0.1.75

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-bounded-control-client-close.md +4 -0
  24. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  25. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  26. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  27. package/docs/cli.md +128 -3
  28. package/docs/compose-services.md +222 -0
  29. package/docs/config.md +177 -6
  30. package/docs/troubleshooting.md +97 -6
  31. package/docs/velocious.md +1 -1
  32. package/docs/workers.md +11 -0
  33. package/examples/tensorbuzz.com.js +3 -2
  34. package/package.json +1 -1
  35. package/src/cli.js +69 -3
  36. package/src/compose-unit.js +725 -0
  37. package/src/config.js +346 -8
  38. package/src/control-client.js +32 -5
  39. package/src/daemon.js +448 -76
  40. package/src/doctor.js +74 -0
  41. package/src/drain-policy.js +252 -0
  42. package/src/guardian-client.js +92 -9
  43. package/src/health.js +99 -8
  44. package/src/managed-process.js +197 -16
  45. package/src/process-guardian.js +212 -22
  46. package/src/process-memory.js +62 -2
  47. package/src/process-order.js +26 -0
  48. package/src/release-group.js +452 -39
  49. package/tensorbuzz.yml +20 -1
  50. package/test/completion.test.js +5 -2
  51. package/test/compose-config.test.js +160 -0
  52. package/test/compose-docker.test.js +297 -0
  53. package/test/compose-unit.test.js +415 -0
  54. package/test/config-validation.test.js +135 -5
  55. package/test/control-protocol.test.js +22 -0
  56. package/test/drain-policy.test.js +328 -0
  57. package/test/fixtures/deferred-boot-app.js +106 -0
  58. package/test/fixtures/gated-app.js +50 -0
  59. package/test/guardian-client.test.js +149 -5
  60. package/test/health.test.js +74 -1
  61. package/test/managed-process.test.js +260 -1
  62. package/test/owner-recovery.test.js +151 -20
  63. package/test/owner-replacement.test.js +176 -15
  64. package/test/process-memory.test.js +47 -1
  65. package/test/release-group.test.js +447 -2
  66. package/test/rollbridge.test.js +774 -72
  67. package/test/shutdown-completion.test.js +119 -0
  68. package/test/shutdown-retirement-bound.test.js +162 -0
  69. package/test/support/process.js +3 -1
  70. package/test/tensorbuzz-config.test.js +228 -0
package/src/daemon.js CHANGED
@@ -9,8 +9,10 @@ import {isDeepStrictEqual} from "node:util"
9
9
  import httpProxy from "http-proxy"
10
10
  import {loadConfig} from "./config.js"
11
11
  import {openControlSession} from "./control-client.js"
12
+ import {admitGenerationRelease} from "./drain-policy.js"
12
13
  import EventLog from "./event-log.js"
13
- import GuardianClient from "./guardian-client.js"
14
+ import GuardianClient, {guardianVersionFromProcess, usableSocketPath} from "./guardian-client.js"
15
+ import {childRecordIsGone} from "./managed-process.js"
14
16
  import ReleaseGroup from "./release-group.js"
15
17
  import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
16
18
  import {resolveGroupId, resolveUserId} from "./system-ids.js"
@@ -24,17 +26,19 @@ const STATE_PERSIST_INTERVAL_MS = 5000
24
26
  * @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
25
27
  * @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
26
28
  * @typedef {{disruptive: boolean, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
29
+ * @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
27
30
  * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
28
31
  * @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
29
- * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
32
+ * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, guardian?: {guardianPid: number | null, ownerMutation: Record<string, JsonValue> | null, version: string | null}, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
30
33
  * @typedef {Omit<import("./managed-process.js").ManagedProcessStatus, "logs">} ManagedProcessStatusWithoutLogs
31
34
  * @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
32
35
  * @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
33
36
  * @typedef {Omit<DaemonStatus, "releases" | "services" | "singletons"> & {releases: ReleaseStatusWithoutLogs[], services: ProcessStatusWithoutLogs[], singletons: ProcessStatusWithoutLogs[]}} DaemonStatusWithoutLogs
37
+ * @typedef {{count: number, hold: string[], processes: {heldProcessGroupPgid: number | null, pid: number | null, processId: string, state: import("./managed-process.js").ManagedProcessState}[], state: import("./release-group.js").ReleaseState}} HeldGenerationHold
34
38
  * @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
35
39
  * @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
36
40
  * @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
37
- * @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
41
+ * @typedef {{boundaryCrossed: boolean, guardianVersion?: string, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, protocol: GuardianHandoffProtocol, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
38
42
  */
39
43
 
40
44
  export default class RollbridgeDaemon {
@@ -146,7 +150,7 @@ export default class RollbridgeDaemon {
146
150
  this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
147
151
  }
148
152
  return {}
149
- })
153
+ }, {waitingFor: "the retained release generations to settle"})
150
154
  } finally {
151
155
  this.generationTransitionRecovery = false
152
156
  }
@@ -203,7 +207,7 @@ export default class RollbridgeDaemon {
203
207
  if (!snapshot || snapshot.recovery?.guardian || !exactBootstrap) throw new Error("Retired owner reset requires an exact committed bootstrap with missing guardian identity")
204
208
  const control = await inspectControlSocket(this.config.control.path)
205
209
  if (control.alive) throw new Error(`Retired owner reset refuses responsive control owner at ${this.config.control.path}`)
206
- this.guardianIdentity = {socketPath: `${this.statePath}.guardian.sock`, token: crypto.randomBytes(32).toString("hex")}
210
+ this.guardianIdentity = {socketPath: usableSocketPath(`${this.statePath}.guardian.sock`), token: crypto.randomBytes(32).toString("hex")}
207
211
  this.guardian = new GuardianClient(this.guardianIdentity)
208
212
  await this.guardian.launch()
209
213
  this.guardianIdentity.pid = this.guardian.pid
@@ -221,10 +225,12 @@ export default class RollbridgeDaemon {
221
225
 
222
226
  if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
223
227
 
224
- const guardianIdentity = recovery?.guardian || {
225
- socketPath: `${this.statePath}.guardian.sock`,
226
- token: crypto.randomBytes(32).toString("hex")
227
- }
228
+ const guardianIdentity = recovery?.guardian
229
+ ? {...recovery.guardian, socketPath: usableSocketPath(recovery.guardian.socketPath)}
230
+ : {
231
+ socketPath: usableSocketPath(`${this.statePath}.guardian.sock`),
232
+ token: crypto.randomBytes(32).toString("hex")
233
+ }
228
234
  this.guardianIdentity = guardianIdentity
229
235
  this.guardian = new GuardianClient(guardianIdentity)
230
236
  if (recovery) {
@@ -504,15 +510,36 @@ export default class RollbridgeDaemon {
504
510
  /**
505
511
  * Prepares from private guardian state, starts listeners, then asks the committed owner
506
512
  * to atomically fence itself and transfer authority.
513
+ * @returns {Promise<void>} Resolves after the replacement is committed.
507
514
  */
508
515
  async replaceIncompatibleOwner() {
516
+ const maxAttempts = 3
517
+ let lastError
518
+
519
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
520
+ try {
521
+ return await this.replaceIncompatibleOwnerAttempt()
522
+ } catch (error) {
523
+ lastError = error
524
+ const diagnostic = error instanceof Error ? error.message : String(error)
525
+
526
+ if (!/^Guardian owner state changed after prepare \(prepared revision \d+, current revision \d+\); abort the stale candidate before preparing again$/.test(diagnostic) || attempt === maxAttempts) {
527
+ throw error
528
+ }
529
+ this.logger("owner replacement candidate became stale; re-preparing", {attempt, maxAttempts, error: diagnostic})
530
+ }
531
+ }
532
+ throw lastError
533
+ }
534
+
535
+ async replaceIncompatibleOwnerAttempt() {
509
536
  if (!this.statePath || !this.config.ownerRecovery) throw new Error("Atomic owner replacement requires ownerRecovery and the same statePath transaction anchor")
510
537
  this.controlCommandsReady = false
511
538
  const state = await readState(this.statePath)
512
539
  const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
513
540
 
514
541
  if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
515
- this.guardianIdentity = persisted.recovery.guardian
542
+ this.guardianIdentity = {...persisted.recovery.guardian, socketPath: usableSocketPath(persisted.recovery.guardian.socketPath)}
516
543
  this.guardian = new GuardianClient(this.guardianIdentity)
517
544
  await this.guardian.connect()
518
545
  this.watchOwnerReplacementEvents()
@@ -520,9 +547,16 @@ export default class RollbridgeDaemon {
520
547
  configDigest: persisted.recovery.configDigest,
521
548
  runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
522
549
  }
523
- const replacementProtocol = await this.guardian.ownerReplacementProtocol()
550
+ const guardianVersion = await this.retainedGuardianVersion()
551
+ const versionDrift = Boolean(this.runtime?.version && guardianVersion && guardianVersion !== this.runtime.version)
552
+ const replacementProtocol = versionDrift ? "legacy" : await this.guardian.ownerReplacementProtocol()
524
553
  const legacyBridge = replacementProtocol === "legacy"
525
- ? await this.prepareLegacyOwnerReplacement({persisted, persistedAuthority})
554
+ ? await this.prepareLegacyOwnerReplacement({
555
+ guardianVersion,
556
+ persisted,
557
+ persistedAuthority,
558
+ protocol: versionDrift ? "guardian-version-drift" : "legacy-protocol"
559
+ })
526
560
  : undefined
527
561
  const prepared = legacyBridge?.prepared ?? await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
528
562
  let preparedStatus
@@ -768,6 +802,33 @@ export default class RollbridgeDaemon {
768
802
  }
769
803
  }
770
804
 
805
+ /**
806
+ * Reads the retained guardian's package version without deciding the replacement protocol.
807
+ *
808
+ * The version is read from the guardian's own executable rather than its capabilities
809
+ * command: a pre-replacement guardian only answers that command once a replacement
810
+ * transaction is prepared, so asking here would move the probe ahead of the transaction
811
+ * every capability-gated guardian depends on. A guardian whose version cannot be read
812
+ * stays on the protocol classification path.
813
+ * @returns {Promise<string | undefined>} Retained guardian package version.
814
+ */
815
+ async retainedGuardianVersion() {
816
+ if (!this.guardian) throw new Error("Guardian version reconciliation requires the durable process guardian")
817
+ return await guardianVersionFromProcess(this.guardian.pid)
818
+ }
819
+
820
+ /**
821
+ * Names the last-resort refusal for a guardian this daemon cannot hand off automatically.
822
+ * @param {string | undefined} guardianVersion - Retained guardian package version.
823
+ * @param {string} cause - Measured reason the automatic handoff cannot run.
824
+ * @returns {Error} Refusal naming both versions and the exact repair command.
825
+ */
826
+ guardianVersionHandoffRefusal(guardianVersion, cause) {
827
+ const configArgument = this.configPath ? ` --config ${this.configPath}` : ""
828
+
829
+ return new Error(`The persistent Rollbridge guardian is version ${guardianVersion ?? "unknown"} while this daemon runs rollbridge ${this.runtime?.version ?? "unknown"}; ${cause}. Run "rollbridge shutdown${configArgument}" and then "rollbridge ensure-daemon${configArgument}" to replace the supervisor with this Rollbridge installation.`)
830
+ }
831
+
771
832
  /** Requires the persistent guardian protocol which can restart an accepted daemon. */
772
833
  async assertGuardianDaemonRecoveryCapability() {
773
834
  if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
@@ -776,10 +837,18 @@ export default class RollbridgeDaemon {
776
837
  try {
777
838
  capabilities = await this.guardian.capabilities()
778
839
  } catch (error) {
779
- if (!(error instanceof Error) || !["Guardian capabilities requires a process key", "Unknown guardian command: capabilities"].includes(error.message)) throw error
840
+ if (!(error instanceof Error) || !isGuardianDaemonRecoveryDiagnostic(error.message)) throw error
780
841
  throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
781
842
  }
782
843
  if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
844
+ const clientVersion = this.runtime?.version
845
+ const guardianVersion = capabilities.guardianVersion
846
+
847
+ if (!clientVersion) return
848
+ if (!guardianVersion) throw this.guardianVersionHandoffRefusal(guardianVersion, "its version cannot be attested, so no handoff can be authorised")
849
+ if (guardianVersion !== clientVersion) {
850
+ throw this.guardianVersionHandoffRefusal(guardianVersion, "this startup path has no incumbent daemon to hand guardian ownership from")
851
+ }
783
852
  }
784
853
 
785
854
  /**
@@ -900,13 +969,15 @@ export default class RollbridgeDaemon {
900
969
  }
901
970
 
902
971
  /**
903
- * Authenticates and prepares the one-time disruptive bridge for an exact legacy guardian protocol.
904
- * @param {{persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
972
+ * Authenticates and prepares the disruptive guardian-upgrade bridge, for a guardian that predates
973
+ * the replacement protocol or whose package version drifted from this daemon's runtime.
974
+ * @param {{guardianVersion?: string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}, protocol: GuardianHandoffProtocol}} options - Bridge evidence and why the bridge is required.
905
975
  * @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
906
976
  */
907
- async prepareLegacyOwnerReplacement({persisted, persistedAuthority}) {
977
+ async prepareLegacyOwnerReplacement({guardianVersion, persisted, persistedAuthority, protocol}) {
908
978
  const legacyProcessKey = ownerSnapshotProcessKeys(persisted, persisted.singletonReleaseIds)[0]
909
979
  const legacyGuardian = this.guardian
980
+ const versionHandoff = protocol === "guardian-version-drift"
910
981
  let legacyPrepared
911
982
 
912
983
  if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state")
@@ -939,12 +1010,23 @@ export default class RollbridgeDaemon {
939
1010
 
940
1011
  try {
941
1012
  if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
1013
+ if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "the handoff keeps the incumbent config identity unchanged and this deploy changes config authority")
942
1014
  throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
943
1015
  }
944
- if (!this.legacyIncumbentPid) throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
1016
+ if (!this.legacyIncumbentPid) {
1017
+ if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "ensure-daemon reported no incumbent daemon to hand guardian ownership from")
1018
+ throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
1019
+ }
945
1020
  if (!this.guardianIdentity?.pid) throw new Error("The authenticated legacy guardian state is missing its exact guardian PID")
946
1021
  await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
947
- const incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
1022
+ let incumbentStartTime
1023
+
1024
+ try {
1025
+ incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
1026
+ } catch (error) {
1027
+ if (!versionHandoff) throw error
1028
+ throw this.guardianVersionHandoffRefusal(guardianVersion, `the incumbent daemon PID ${this.legacyIncumbentPid} is not the attested owner: ${error instanceof Error ? error.message : String(error)}`)
1029
+ }
948
1030
  if (legacyPrepared) assertLegacyPrivateOwnerState(legacyPrepared.ownerState, persisted, persistedAuthority)
949
1031
  try {
950
1032
  incumbentControl = await openControlSession(persisted.control.path)
@@ -971,7 +1053,9 @@ export default class RollbridgeDaemon {
971
1053
 
972
1054
  assertLegacyGuardianInventoryMembership(legacyInventory, /** @type {PrivateOwnerState} */ (ownerState))
973
1055
  const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
974
- socketPath: `${this.statePath}.split3-guardian.sock`,
1056
+ socketPath: usableSocketPath(versionHandoff
1057
+ ? `${this.statePath}.guardian-${this.runtime?.version}.sock`
1058
+ : `${this.statePath}.split3-guardian.sock`),
975
1059
  token: crypto.randomBytes(32).toString("hex")
976
1060
  })
977
1061
 
@@ -986,12 +1070,15 @@ export default class RollbridgeDaemon {
986
1070
  recovery: {...persisted.recovery, guardian: upgradedIdentity}
987
1071
  })
988
1072
 
989
- this.logger("legacy owner replacement bridge prepared", {
1073
+ this.logger("guardian upgrade bridge prepared", {
990
1074
  guardianPid: upgradedIdentity.pid ?? null,
1075
+ guardianVersion: guardianVersion ?? "unknown",
991
1076
  incumbentPid: this.legacyIncumbentPid,
992
- replacementId: prepared.replacementId
1077
+ protocol,
1078
+ replacementId: prepared.replacementId,
1079
+ runtimeVersion: this.runtime?.version ?? "unknown"
993
1080
  })
994
- return {boundaryCrossed: false, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, recoverySnapshot}
1081
+ return {boundaryCrossed: false, guardianVersion, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, protocol, recoverySnapshot}
995
1082
  } catch (upgradeError) {
996
1083
  incumbentControl?.close()
997
1084
  let cleanupError
@@ -1071,10 +1158,20 @@ export default class RollbridgeDaemon {
1071
1158
 
1072
1159
  if (staged.committed) throw new Error("Legacy guardian committed replacement before the authenticated disruptive boundary")
1073
1160
  }
1074
- this.logger("legacy owner replacement disruptive boundary", {
1161
+ const boundaryReason = bridge.protocol === "guardian-version-drift"
1162
+ ? `retained guardian ${bridge.guardianVersion ?? "unknown"} predates this daemon runtime ${this.runtime?.version ?? "unknown"}`
1163
+ : "retained guardian and daemon lacked atomic replacement protocol"
1164
+
1165
+ this.logger("guardian upgrade bridge crossed", {
1075
1166
  disruptive: true,
1167
+ guardianVersion: bridge.guardianVersion ?? "unknown",
1076
1168
  incumbentPid: this.legacyIncumbentPid,
1077
- reason: "retained guardian and daemon lacked atomic replacement protocol"
1169
+ protocol: bridge.protocol,
1170
+ reason: boundaryReason,
1171
+ // The retained guardian keeps its own children; the boundary retires only the incumbent daemon.
1172
+ retainedGuardianPid: bridge.legacyGuardian.pid ?? null,
1173
+ retainedGuardianProcesses: bridge.legacyInventory.length,
1174
+ runtimeVersion: this.runtime?.version ?? "unknown"
1078
1175
  })
1079
1176
  await this.guardian.beginLegacyOwnerClaim(
1080
1177
  bridge.prepared.replacementId,
@@ -1090,8 +1187,8 @@ export default class RollbridgeDaemon {
1090
1187
  await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
1091
1188
  this.ownerTransition = /** @type {OwnerTransition} */ ({
1092
1189
  disruptive: true,
1093
- mode: "legacy-first-upgrade",
1094
- reason: "retained guardian and daemon lacked atomic replacement protocol"
1190
+ mode: bridge.protocol === "guardian-version-drift" ? "guardian-version-handoff" : "legacy-first-upgrade",
1191
+ reason: boundaryReason
1095
1192
  })
1096
1193
  }
1097
1194
 
@@ -1393,7 +1490,7 @@ export default class RollbridgeDaemon {
1393
1490
  releaseId: stringOrUndefined(data.releaseId),
1394
1491
  releasePath: requiredString(data.releasePath, "releasePath"),
1395
1492
  revision: stringOrUndefined(data.revision)
1396
- }))
1493
+ }), {waitingFor: "candidate activation and the retired generation handoff"})
1397
1494
  }
1398
1495
 
1399
1496
  if (commandName === "recover-generation-transition") {
@@ -1403,11 +1500,22 @@ export default class RollbridgeDaemon {
1403
1500
  releaseId: requiredString(data.releaseId, "releaseId"),
1404
1501
  releasePath: requiredString(data.releasePath, "releasePath"),
1405
1502
  revision: requiredString(data.revision, "revision")
1406
- }))
1503
+ }), {waitingFor: "the retained release generations to settle"})
1407
1504
  }
1408
1505
 
1409
1506
  if (commandName === "status") {
1410
- return this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
1507
+ const status = this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
1508
+ let guardian
1509
+
1510
+ try {
1511
+ const diagnostics = await this.guardian?.guardianStatus()
1512
+
1513
+ guardian = diagnostics ? {guardianPid: diagnostics.guardianPid, ownerMutation: diagnostics.ownerMutation, version: diagnostics.version} : {guardianPid: null, ownerMutation: null, version: null}
1514
+ } catch (error) {
1515
+ guardian = {error: error instanceof Error ? error.message : String(error), guardianPid: this.guardianIdentity?.pid ?? null, ownerMutation: null, version: null}
1516
+ }
1517
+
1518
+ return {...status, guardian}
1411
1519
  }
1412
1520
 
1413
1521
  if (commandName === "events") {
@@ -1418,18 +1526,18 @@ export default class RollbridgeDaemon {
1418
1526
  return await this.executeOwnerMutation("stop", async () => {
1419
1527
  await this.stopRelease(stringOrUndefined(data.releaseId))
1420
1528
  return this.status()
1421
- })
1529
+ }, {waitingFor: "the release's managed processes to stop"})
1422
1530
  }
1423
1531
 
1424
1532
  if (commandName === "restart") {
1425
1533
  return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
1426
1534
  policy: stringOrUndefined(data.policy),
1427
1535
  processId: stringOrUndefined(data.processId)
1428
- }))
1536
+ }), {waitingFor: "the managed process to stop and start again"})
1429
1537
  }
1430
1538
 
1431
1539
  if (commandName === "rollback") {
1432
- return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
1540
+ return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}), {waitingFor: "the previous release to activate"})
1433
1541
  }
1434
1542
 
1435
1543
  if (commandName === "shutdown") {
@@ -1493,11 +1601,13 @@ export default class RollbridgeDaemon {
1493
1601
  * Runs one owner mutation under the guardian's prepare/commit exclusion fence.
1494
1602
  * @param {string} operation - Mutation diagnostic name.
1495
1603
  * @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
1604
+ * @param {{waitingFor?: string}} [options] - What this mutation blocks on, so a refused
1605
+ * later mutation names the real blocker instead of just "in progress".
1496
1606
  * @returns {Promise<Record<string, JsonValue>>} Operation result.
1497
1607
  */
1498
- async executeOwnerMutation(operation, callback) {
1608
+ async executeOwnerMutation(operation, callback, {waitingFor} = {}) {
1499
1609
  if (!this.guardian) return await callback()
1500
- const mutationId = await this.guardian.beginOwnerMutation(operation)
1610
+ const mutationId = await this.guardian.beginOwnerMutation(operation, {ownerPid: process.pid, waitingFor})
1501
1611
  let result
1502
1612
  let operationError
1503
1613
 
@@ -1633,6 +1743,46 @@ export default class RollbridgeDaemon {
1633
1743
  await this.updateGenerationTransition("restoring_committed")
1634
1744
  return await this.resumeGenerationTransition()
1635
1745
  }
1746
+ await this.escalateWedgedGenerations(nextConfig, newReleaseId)
1747
+ // A release id that is still retained by a generation holding live work cannot be started
1748
+ // a second time: the candidate's processes re-register the guardian keys that the retained
1749
+ // generation's own processes still hold, so the deploy aborts partway through `start()`
1750
+ // with a bare `Guardian provenance mismatch` — after the caller has committed to this
1751
+ // revision and, for a restore, after the deployment pointer has already moved. Refusing
1752
+ // from the retained record before any process starts keeps the incumbent serving, names the
1753
+ // release the guardian reports as active, and leaves the retained generation intact for
1754
+ // `rollbridge status` to reconcile against. Journaled transition resume above is exempt:
1755
+ // it re-drives the transaction that already owns this release id.
1756
+ //
1757
+ // The escalation above runs first on purpose: it is what clears the residue that made this
1758
+ // guard refuse the measured 2026-11-15 and 2026-11-16 deploys. A wedge that escalation can
1759
+ // prove is kill-safe stops holding its keys and the handoff proceeds; a generation that
1760
+ // still holds live work is not killed and is named here instead of aborting mid-`start()`.
1761
+ const heldGeneration = this.retainedGenerationHoldingLiveWork(newReleaseId)
1762
+
1763
+ if (heldGeneration) {
1764
+ const activeReleaseId = this.activeRelease?.releaseId ?? null
1765
+
1766
+ this.logger("deploy refused: retained generation still holds live processes", {
1767
+ activeReleaseId,
1768
+ heldProcesses: heldGeneration.processes,
1769
+ releaseId: newReleaseId,
1770
+ retainedState: heldGeneration.state
1771
+ })
1772
+
1773
+ throw new Error(`Deploy refused before activation: release "${newReleaseId}" is retained in state ${heldGeneration.state} while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; deploy a new release id or reconcile the deployment pointer with \`rollbridge status\`.`)
1774
+ }
1775
+ const admission = this.evaluateGenerationAdmission(newReleaseId, nextConfig)
1776
+
1777
+ if (!admission.allowed) {
1778
+ this.logger("deploy refused by draining-generation bound", {
1779
+ candidateReleaseId: newReleaseId,
1780
+ draining: admission.draining.map((group) => ({releaseId: group.releaseId, status: group.status})),
1781
+ limit: nextConfig.releaseRetention.maxConcurrentDrainingReleaseGroups,
1782
+ wouldBeDrainingCount: admission.wouldBeDrainingCount
1783
+ })
1784
+ throw new Error(/** @type {string} */ (admission.reason))
1785
+ }
1636
1786
  const release = new ReleaseGroup({
1637
1787
  config: nextConfig,
1638
1788
  logger: this.logger,
@@ -1689,6 +1839,79 @@ export default class RollbridgeDaemon {
1689
1839
  return await this.resumeGenerationTransition()
1690
1840
  }
1691
1841
 
1842
+ /**
1843
+ * Applies the draining-generation bound before a handoff.
1844
+ *
1845
+ * Without this, every deploy retires one more generation and the backlog grows
1846
+ * monotonically whenever a drain outlasts the deploy cadence; a consumer that checks the
1847
+ * count after activation reports the growth it cannot undo. Refusing here names the
1848
+ * draining groups and whether each is still progressing or wedged, before any process of
1849
+ * the candidate is started.
1850
+ * @param {string} candidateReleaseId - Release the handoff would activate.
1851
+ * @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
1852
+ * @returns {import("./drain-policy.js").GenerationAdmission} Admission decision and the groups behind it.
1853
+ */
1854
+ evaluateGenerationAdmission(candidateReleaseId, config) {
1855
+ const draining = [...this.releases.values()]
1856
+ .filter((release) => release.state === "draining")
1857
+ .map((release) => {
1858
+ const drain = release.drainStatus()
1859
+
1860
+ return {reason: drain?.reason ?? "no liveness observation recorded", releaseId: release.releaseId, status: drain?.status ?? "unproven"}
1861
+ })
1862
+
1863
+ return admitGenerationRelease({
1864
+ candidateReleaseId,
1865
+ draining,
1866
+ incumbentWillDrain: this.activeRelease !== undefined,
1867
+ maxConcurrentDrainingReleaseGroups: config.releaseRetention.maxConcurrentDrainingReleaseGroups
1868
+ })
1869
+ }
1870
+
1871
+ /**
1872
+ * Reports whether a retained release group still holds live work that a re-activation would collide with.
1873
+ *
1874
+ * `stopped` is not the same claim as `drained`: the guardian's provenance fence only lets a
1875
+ * process key be re-registered once the record that holds it is gone. A generation recorded
1876
+ * as drained while its process group is still alive — the shape an indefinite worker drain
1877
+ * produces — still holds its keys, so its release id is not available for a restore.
1878
+ * @param {string} releaseId - Release id a handoff would activate.
1879
+ * @returns {HeldGenerationHold | undefined} The live hold, or undefined when the id is free.
1880
+ */
1881
+ retainedGenerationHoldingLiveWork(releaseId) {
1882
+ const retained = this.releases.get(releaseId)
1883
+
1884
+ if (!retained || retained === this.activeRelease) return undefined
1885
+
1886
+ return heldGenerationHold(retained)
1887
+ }
1888
+
1889
+ /**
1890
+ * Gives every retiring generation the escalation decision its own retirement left open,
1891
+ * before the handoff is judged.
1892
+ *
1893
+ * A deploy is the event that makes a wedged generation expensive: until something tries to
1894
+ * activate a successor, a retired release holding a live process group costs nothing
1895
+ * observable. Acting here means the observation that proves the wedge and the action on it
1896
+ * come from the same read the admission bound already performs, so no sampler has to run
1897
+ * alongside the daemon. Generations that already recorded `release drained` are included:
1898
+ * their retirement finished, and the process group they left behind is the residue that
1899
+ * blocked the measured 2026-11-15 deploys.
1900
+ * @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
1901
+ * @param {string} candidateReleaseId - Release the handoff would activate.
1902
+ * @returns {Promise<void>} Resolves once every retiring generation has been decided.
1903
+ */
1904
+ async escalateWedgedGenerations(config, candidateReleaseId) {
1905
+ if (!config.releaseRetention.escalateWedgedGenerations) return
1906
+
1907
+ for (const release of this.releases.values()) {
1908
+ if (release.releaseId === candidateReleaseId) continue
1909
+ if (release.state !== "draining" && release.state !== "stopped" && release.state !== "failed") continue
1910
+
1911
+ await release.escalateWedgedGeneration({reason: `deploy of ${candidateReleaseId}`})
1912
+ }
1913
+ }
1914
+
1692
1915
  /**
1693
1916
  * Continues one exact durable generation transition without an internal retry loop.
1694
1917
  * @returns {Promise<Record<string, JsonValue>>} Deploy result after commit.
@@ -1710,10 +1933,9 @@ export default class RollbridgeDaemon {
1710
1933
  let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
1711
1934
 
1712
1935
  if (transition.phase === "candidate_ready") {
1713
- // Candidate processes are already started and health-checked. Keep the incumbent
1714
- // serving until the candidate has acknowledged activation and traffic has switched;
1715
- // retiring it here can stop the only live web/jobs generation before its successor
1716
- // carries traffic (STOE-2269).
1936
+ // The candidate has already started and passed its health gates. Keep the
1937
+ // incumbent authoritative until the candidate is activated and traffic is
1938
+ // switched; retirement is deliberately launched from committed_pending.
1717
1939
  await this.updateGenerationTransition("activating_candidate")
1718
1940
  }
1719
1941
 
@@ -1788,6 +2010,22 @@ export default class RollbridgeDaemon {
1788
2010
 
1789
2011
  if (transition.phase === "committed_pending") {
1790
2012
  await this.refreshServiceDefinitions(release)
2013
+ if (previousRelease?.state === "active") {
2014
+ if (this.activeRelease !== release) {
2015
+ throw new Error(`Cannot retire incumbent release ${previousRelease.releaseId} before candidate release ${release.releaseId} is authoritative.`)
2016
+ }
2017
+ const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
2018
+
2019
+ // Traffic is already on the healthy candidate. Retirement must not hold
2020
+ // the deploy mutation or its owner fence; its own bounded drain reports
2021
+ // failures and escalates any process that misses the retirement budget.
2022
+ void previousRelease.beginRetirement(retirementConfig)
2023
+ .catch((error) => {
2024
+ retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
2025
+ this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
2026
+ })
2027
+ .then(() => this.drainAndPrune(previousRelease, retirementConfig))
2028
+ }
1791
2029
  await this.checkpointGenerationTransition()
1792
2030
  try {
1793
2031
  await this.replaceSingletons(release)
@@ -1795,29 +2033,12 @@ export default class RollbridgeDaemon {
1795
2033
  await this.failGenerationTransition(error instanceof Error ? error.message : String(error))
1796
2034
  throw error
1797
2035
  }
1798
- if (previousRelease) {
1799
- const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
1800
-
1801
- // Traffic is already on the candidate and singleton ownership has been
1802
- // transferred. Retirement is deliberately detached so a slow old generation
1803
- // cannot hold the committed deploy open or race guardian-owned replacement.
1804
- void this.drainAndPrune(previousRelease, retirementConfig)
1805
- }
1806
2036
  transition.phase = "committed"
1807
2037
  transition.error = undefined
1808
2038
  transition.journalRevision = (transition.journalRevision ?? 0) + 1
1809
2039
  transition.updatedAt = new Date().toISOString()
1810
2040
  this.pruneStoppedReleases()
1811
2041
  await this.checkpointGenerationTransition()
1812
- if (previousRelease) {
1813
- const retirementConfig = !activationLifecycle && previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
1814
-
1815
- // Traffic is already on the healthy candidate. Retirement is deliberately detached
1816
- // from the owner mutation so a slow quiet/drain hook can never hold the deploy fence
1817
- // or make the active successor unavailable. drainAndStop applies the bounded force
1818
- // deadline and records any retirement failure on the old generation.
1819
- void this.drainAndPrune(previousRelease, retirementConfig)
1820
- }
1821
2042
  }
1822
2043
 
1823
2044
  return {
@@ -1847,6 +2068,8 @@ export default class RollbridgeDaemon {
1847
2068
  if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
1848
2069
  if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
1849
2070
  if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
2071
+ /** @type {Promise<void> | undefined} */
2072
+ let candidateDrain
1850
2073
 
1851
2074
  if (transition.phase === "activating_candidate") {
1852
2075
  if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
@@ -1878,6 +2101,15 @@ export default class RollbridgeDaemon {
1878
2101
  transition.compensationError = undefined
1879
2102
  await this.updateGenerationTransition("restoring_previous")
1880
2103
  this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
2104
+
2105
+ // The candidate's bounded drain starts here, before the restoration attempt. The only other
2106
+ // compensation drain runs after the success path, so when incumbent restoration throws the
2107
+ // candidate is left `draining` with a live process: measured 2026-11-17, candidate
2108
+ // `20261117063703` still held `web` pid 3201928 at 07:30Z after its compensation threw at
2109
+ // 06:42:31Z, and reached `release drained` only when a later compensation reached the
2110
+ // success path at 08:47:00Z. A retained generation holding live work is what refuses every
2111
+ // later deploy, and a failed candidate is never a restoration target.
2112
+ candidateDrain = this.drainAndPrune(candidate, candidate.config)
1881
2113
  }
1882
2114
 
1883
2115
  if (transition.phase === "restoring_previous") {
@@ -1889,10 +2121,12 @@ export default class RollbridgeDaemon {
1889
2121
 
1890
2122
  transition.compensationError = failure
1891
2123
  await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
2124
+ await previous.stopProxyTraffic()
1892
2125
  this.logger("release generation compensation restoration failed", {
1893
2126
  activationError: transition.activationError,
1894
2127
  error: failure,
1895
- releaseId: previous.releaseId
2128
+ releaseId: previous.releaseId,
2129
+ traffic: "unavailable"
1896
2130
  })
1897
2131
  throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
1898
2132
  }
@@ -1931,7 +2165,7 @@ export default class RollbridgeDaemon {
1931
2165
  this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
1932
2166
  throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
1933
2167
  }
1934
- void this.drainAndPrune(candidate, candidate.config)
2168
+ if (candidateDrain === undefined) void this.drainAndPrune(candidate, candidate.config)
1935
2169
  this.logger("release generation compensation completed", result)
1936
2170
  return result
1937
2171
  }
@@ -1955,6 +2189,19 @@ export default class RollbridgeDaemon {
1955
2189
  recoveryStatus: "already_recovered"
1956
2190
  }
1957
2191
  }
2192
+ // A committed journal describes the transition that already published the current
2193
+ // incumbent, not an in-flight transition to recover. When the recovery fence names
2194
+ // that live incumbent, the failed deploy never journaled its own transition, so the
2195
+ // only fence that describes the authoritative release is the completed one: recovery
2196
+ // is already done. In-flight phases keep the stale-fence guard failing closed.
2197
+ if (transition.phase === "committed" && transition.candidateReleaseId === identity.previousReleaseId && this.activeRelease?.releaseId === identity.previousReleaseId && this.releases.get(transition.candidateReleaseId) === this.activeRelease) {
2198
+ return {
2199
+ activeReleaseId: this.activeRelease.releaseId,
2200
+ candidateReleaseId: identity.releaseId,
2201
+ previousReleaseId: identity.previousReleaseId,
2202
+ recoveryStatus: "already_recovered"
2203
+ }
2204
+ }
1958
2205
  if (transition.previousReleaseId !== identity.previousReleaseId) {
1959
2206
  throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
1960
2207
  }
@@ -2401,11 +2648,38 @@ export default class RollbridgeDaemon {
2401
2648
  throw new Error(`Release "${target.releaseId}" is already active.`)
2402
2649
  }
2403
2650
 
2651
+ // A generation recorded as `stopped` whose processes are still alive is the shape a
2652
+ // restore cannot honour: re-deploying that release id re-registers its guardian process
2653
+ // keys under a new provenance while the old processes still hold them, and the guardian
2654
+ // answers with a bare provenance mismatch. Use the guardian's own fence here so the
2655
+ // refusal is decided before anything is touched, and name the divergence — including a
2656
+ // process group that outlived the tracked process and is invisible as a pid — together
2657
+ // with the release the guardian actually reports as active, so the deployment tool can
2658
+ // reconcile its own current pointer instead of restoring a generation that already drained.
2659
+ // A target that is still draining is not refused here: stopping it is what makes its id
2660
+ // reusable, and the deploy below applies the same fence after that stop.
2661
+ const heldGeneration = target.state === "stopped" ? heldGenerationHold(target) : undefined
2662
+
2663
+ if (heldGeneration) {
2664
+ const activeReleaseId = this.activeRelease?.releaseId ?? null
2665
+
2666
+ this.logger("rollback refused: drained generation still holds live processes", {
2667
+ activeReleaseId,
2668
+ heldProcesses: heldGeneration.processes,
2669
+ releaseId: target.releaseId,
2670
+ revision: target.revision
2671
+ })
2672
+
2673
+ throw new Error(`Release "${target.releaseId}" is recorded as drained while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; reconcile the deployment pointer with \`rollbridge status\` before restoring.`)
2674
+ }
2675
+
2404
2676
  // The target may still be draining a prior deploy (live processes). Stop it before the
2405
2677
  // deploy below re-uses its id in this.releases, otherwise the still-running instance
2406
2678
  // would be dropped from status/pruning/shutdown and could be orphaned.
2679
+ // Rollback is an owner mutation too, so this stop carries the same retirement bound as
2680
+ // `stop`: a process the supervisor must not kill must not be able to hold the fence.
2407
2681
  if (target.state !== "stopped" && target.state !== "failed") {
2408
- await target.stop()
2682
+ await target.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})
2409
2683
  }
2410
2684
 
2411
2685
  this.logger("rollback starting", {releaseId: target.releaseId, releasePath: target.releasePath})
@@ -2626,7 +2900,7 @@ export default class RollbridgeDaemon {
2626
2900
 
2627
2901
  /**
2628
2902
  * @param {string | undefined} releaseId - Release id, or active release when omitted.
2629
- * @returns {Promise<void>} Resolves when stopped.
2903
+ * @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
2630
2904
  */
2631
2905
  async stopRelease(releaseId) {
2632
2906
  this.assertNoUnresolvedGenerationTransition("stop")
@@ -2635,7 +2909,10 @@ export default class RollbridgeDaemon {
2635
2909
  if (!release) throw new Error(`Release not found: ${releaseId || "active"}`)
2636
2910
  if (release === this.activeRelease) this.activeRelease = undefined
2637
2911
 
2638
- await release.stop()
2912
+ // This command runs inside the owner mutation, so an unbudgeted wait for a process the
2913
+ // supervisor must not kill holds the fence for the whole drain and every later deploy is
2914
+ // refused. The retirement bound is the same one the deploy path already uses.
2915
+ await release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})
2639
2916
  this.logger("release stopped", {releaseId: release.releaseId})
2640
2917
  this.pruneStoppedReleases()
2641
2918
  this.persistState()
@@ -2778,17 +3055,52 @@ export default class RollbridgeDaemon {
2778
3055
 
2779
3056
  /**
2780
3057
  * Stops proxy, control socket, and child processes.
2781
- * @param {{completionSocket?: net.Socket, waitForControlConnections?: boolean}} [options] - Shutdown connection behavior.
3058
+ * @param {{completionSocket?: net.Socket, preserveActiveRelease?: boolean, waitForControlConnections?: boolean}} [options] - Shutdown connection behavior.
2782
3059
  * @returns {Promise<void>} Resolves when owned resources are stopped (and, by default, control connections close).
2783
3060
  */
2784
- async shutdown({completionSocket, waitForControlConnections = true} = {}) {
2785
- this.assertNoGenerationTransitionRecovery("shut down")
2786
- if (!this.shutdownPromise) this.shutdownPromise = this.performShutdown(completionSocket)
3061
+ async shutdown({completionSocket, preserveActiveRelease = false, waitForControlConnections = true} = {}) {
3062
+ // A supervisor signal during committed_pending is after candidate activation: the candidate
3063
+ // is already serving, so the recovery shutdown may finish the bounded retirement. Earlier
3064
+ // transition phases still refuse shutdown rather than risking the active incumbent.
3065
+ const finishCommittedPendingRetirement = preserveActiveRelease && this.generationTransition?.phase === "committed_pending"
3066
+
3067
+ if (!finishCommittedPendingRetirement) this.assertNoGenerationTransitionRecovery("shut down")
3068
+ if (!this.shutdownPromise) {
3069
+ this.shutdownPromise = finishCommittedPendingRetirement
3070
+ ? this.performShutdown(completionSocket)
3071
+ : preserveActiveRelease && this.guardian
3072
+ ? this.performRecoveryShutdown(completionSocket)
3073
+ : this.performShutdown(completionSocket)
3074
+ }
2787
3075
 
2788
3076
  await this.shutdownPromise
2789
3077
  if (waitForControlConnections && this.controlClosePromise) await this.controlClosePromise
2790
3078
  }
2791
3079
 
3080
+ /**
3081
+ * Relinquishes this daemon during supervisor recovery without stopping guardian-owned
3082
+ * generations. The guardian will start or reconnect the replacement owner, which is the only
3083
+ * point at which the active release may be retired.
3084
+ * @param {net.Socket | undefined} completionSocket - Requesting connection, if any.
3085
+ * @returns {Promise<void>} Resolves after this daemon's listeners are released.
3086
+ */
3087
+ async performRecoveryShutdown(completionSocket) {
3088
+ this.ownerRetired = true
3089
+ this.controlCommandsReady = false
3090
+ this.stopping = true
3091
+ if (this.persistTimer) clearInterval(this.persistTimer)
3092
+ this.persistTimer = undefined
3093
+ if (this.pendingWrite) await this.pendingWrite
3094
+ this.persistenceEnabled = false
3095
+ this.stateCleanupEnabled = false
3096
+ this.controlClosePromise = this.closeServer(this.controlServer)
3097
+ this.proxyClosePromise = this.closeServer(this.proxyServer)
3098
+ for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
3099
+ await this.removeControlSocket()
3100
+ await Promise.all([this.controlClosePromise, this.proxyClosePromise])
3101
+ this.guardian?.disconnect()
3102
+ }
3103
+
2792
3104
  /**
2793
3105
  * Relinquishes stable listeners promptly while retaining draining children under
2794
3106
  * this daemon until their normal stop contract completes.
@@ -2840,11 +3152,16 @@ export default class RollbridgeDaemon {
2840
3152
  await this.guardian.retireOwner()
2841
3153
  this.guardian.disconnect()
2842
3154
  } else {
3155
+ // Without a guardian nobody else can retire these processes, so an unbounded stop would
3156
+ // leave the generation draining forever — the state that made the measured 2026-11-17
3157
+ // rollback refuse. The same retirement budget terminalizes it and names the residue.
3158
+ const retirementTimeoutMs = this.config.proxy.nonBlockingDrainTimeoutMs
3159
+
2843
3160
  void Promise.allSettled([
2844
- ...[...this.services.values()].map((processInstance) => processInstance.stop()),
2845
- ...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
2846
- ...[...this.startingReleases].map((release) => release.stop()),
2847
- ...[...this.releases.values()].map((release) => release.stop())
3161
+ ...[...this.services.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
3162
+ ...[...this.singletons.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
3163
+ ...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs})),
3164
+ ...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs}))
2848
3165
  ])
2849
3166
  }
2850
3167
  this.logger("external owner retired", {attestation, status: "draining"})
@@ -2878,12 +3195,19 @@ export default class RollbridgeDaemon {
2878
3195
  }
2879
3196
 
2880
3197
  await captureShutdownError(cleanupErrors, "proxy close", async () => this.proxy.close())
3198
+ // Shutdown must return even when a process refuses to exit. The bound is the
3199
+ // `proxy.nonBlockingDrainTimeoutMs` an explicit `rollbridge stop` already uses, and it means
3200
+ // "stop waiting", never "SIGKILL": a process whose graceful window is `"indefinite"` is left
3201
+ // draining rather than killed. Left unbudgeted, that stop keeps the daemon alive after its
3202
+ // control socket is already unlinked — alive, unreachable, and still holding the generation it
3203
+ // stopped: the wedged daemon STOE-2275 measured behind 4,800 `connect ECONNREFUSED` proxy errors.
3204
+ const retirementTimeoutMs = this.config.proxy.nonBlockingDrainTimeoutMs
2881
3205
  const dependentStopResults = await Promise.allSettled([
2882
- ...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
2883
- ...[...this.startingReleases].map((release) => release.stop()),
2884
- ...[...this.releases.values()].map((release) => release.stop())
3206
+ ...[...this.singletons.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
3207
+ ...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs})),
3208
+ ...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs}))
2885
3209
  ])
2886
- const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
3210
+ const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})))
2887
3211
  const stopResults = [...dependentStopResults, ...serviceStopResults]
2888
3212
  const guardian = this.guardian
2889
3213
  if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
@@ -3067,6 +3391,42 @@ function statusIncludeLogs(value) {
3067
3391
  return value
3068
3392
  }
3069
3393
 
3394
+ /**
3395
+ * Names the live work a retained release group still holds, using the guardian's own fence.
3396
+ *
3397
+ * A process counts as holding work when the guardian would refuse to hand its key to a new
3398
+ * provenance: a running or stopping process, or a record already `stopped` whose process group
3399
+ * outlived it and is invisible as a pid.
3400
+ * @param {ReleaseGroup} release - Retained generation to inspect.
3401
+ * @returns {HeldGenerationHold | undefined} The hold and its log payload, or undefined when every process is gone.
3402
+ */
3403
+ function heldGenerationHold(release) {
3404
+ const heldProcesses = release.status().processes.filter((processStatus) => !childRecordIsGone(processStatus))
3405
+
3406
+ if (heldProcesses.length === 0) return undefined
3407
+
3408
+ return {
3409
+ count: heldProcesses.length,
3410
+ hold: heldProcesses.map((processStatus) => {
3411
+ if (processStatus.heldProcessGroup) {
3412
+ const {liveMemberCount, pgid} = processStatus.heldProcessGroup
3413
+ const members = liveMemberCount === undefined ? "" : ` with ${liveMemberCount} live member(s)`
3414
+
3415
+ return `${processStatus.id}: process group ${pgid}${members} still alive after the tracked process exited`
3416
+ }
3417
+
3418
+ return `${processStatus.id}: state ${processStatus.state}, pid ${processStatus.pid ?? "unknown"}`
3419
+ }),
3420
+ processes: heldProcesses.map((processStatus) => ({
3421
+ heldProcessGroupPgid: processStatus.heldProcessGroup?.pgid ?? null,
3422
+ pid: processStatus.pid ?? null,
3423
+ processId: processStatus.id,
3424
+ state: processStatus.state
3425
+ })),
3426
+ state: release.state
3427
+ }
3428
+ }
3429
+
3070
3430
  /**
3071
3431
  * @param {PrivateOwnerState} transfer - Authenticated private incumbent state.
3072
3432
  * @param {GenerationTransition} transition - Unresolved generation transition.
@@ -3127,6 +3487,15 @@ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
3127
3487
  diagnostic === "Unknown guardian command: prepare-owner-replacement"
3128
3488
  }
3129
3489
 
3490
+ /**
3491
+ * Reports the capability diagnostics that identify a guardian predating daemon recovery.
3492
+ * @param {string} diagnostic - Guardian error message.
3493
+ * @returns {boolean} Whether the guardian predates the daemon recovery capability.
3494
+ */
3495
+ function isGuardianDaemonRecoveryDiagnostic(diagnostic) {
3496
+ return diagnostic === "Guardian capabilities requires a process key" || diagnostic === "Unknown guardian command: capabilities"
3497
+ }
3498
+
3130
3499
  /**
3131
3500
  * @param {DaemonStatus} snapshot - Serialized owner process snapshot.
3132
3501
  * @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
@@ -3449,7 +3818,7 @@ function requiredNonNegativeInteger(value, key) {
3449
3818
  /**
3450
3819
  * Selects stopped releases to prune by the retention policy, keeping the most recent.
3451
3820
  * @param {PrunableRelease[]} releases - Status of all tracked releases, in deploy order (oldest first).
3452
- * @param {import("./config.js").ReleaseRetentionConfig} policy - Retention policy.
3821
+ * @param {{keep: number, maxAgeMs: number}} policy - Retention policy; draining generations are never pruned here.
3453
3822
  * @param {number} now - Current epoch milliseconds.
3454
3823
  * @returns {string[]} Release ids to remove.
3455
3824
  */
@@ -3474,7 +3843,7 @@ export function releasesToPrune(releases, policy, now) {
3474
3843
  }
3475
3844
 
3476
3845
  /**
3477
- * @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, proxy?: {host: string, port: number}}} ControlSocketInspection
3846
+ * @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, guardian?: {ownerMutation: Record<string, JsonValue> | null, version: string | null}, proxy?: {host: string, port: number}}} ControlSocketInspection
3478
3847
  */
3479
3848
 
3480
3849
  /**
@@ -3549,7 +3918,7 @@ export async function inspectControlSocket(socketPath, timeoutMs = 1000) {
3549
3918
  /**
3550
3919
  * Parses a control status response line into a Rollbridge identity, if it is one.
3551
3920
  * @param {string} line - JSON response line.
3552
- * @returns {{application: string, activeReleaseId: string | null, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
3921
+ * @returns {{application: string, activeReleaseId: string | null, guardian: {ownerMutation: Record<string, JsonValue> | null, version: string | null} | undefined, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
3553
3922
  */
3554
3923
  function parseControlStatus(line) {
3555
3924
  /** @type {JsonValue} */
@@ -3566,5 +3935,8 @@ function parseControlStatus(line) {
3566
3935
 
3567
3936
  const proxy = "proxy" in parsed && parsed.proxy && typeof parsed.proxy === "object" && !Array.isArray(parsed.proxy) && typeof parsed.proxy.host === "string" && typeof parsed.proxy.port === "number" ? {host: parsed.proxy.host, port: parsed.proxy.port} : undefined
3568
3937
 
3569
- return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, proxy}
3938
+ const guardianValue = "guardian" in parsed && parsed.guardian && typeof parsed.guardian === "object" && !Array.isArray(parsed.guardian) ? parsed.guardian : undefined
3939
+ const guardian = guardianValue && typeof guardianValue.version === "string" || guardianValue && guardianValue.version === null ? {ownerMutation: guardianValue.ownerMutation && typeof guardianValue.ownerMutation === "object" && !Array.isArray(guardianValue.ownerMutation) ? guardianValue.ownerMutation : null, version: guardianValue.version} : undefined
3940
+
3941
+ return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, guardian, proxy}
3570
3942
  }