rollbridge 0.1.73 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  24. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  25. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  26. package/docs/cli.md +128 -3
  27. package/docs/compose-services.md +222 -0
  28. package/docs/config.md +177 -6
  29. package/docs/troubleshooting.md +97 -6
  30. package/docs/velocious.md +1 -1
  31. package/docs/workers.md +11 -0
  32. package/examples/tensorbuzz.com.js +3 -2
  33. package/package.json +1 -1
  34. package/src/cli.js +68 -2
  35. package/src/compose-unit.js +725 -0
  36. package/src/config.js +346 -8
  37. package/src/daemon.js +411 -78
  38. package/src/doctor.js +74 -0
  39. package/src/drain-policy.js +252 -0
  40. package/src/guardian-client.js +68 -23
  41. package/src/health.js +99 -8
  42. package/src/managed-process.js +198 -45
  43. package/src/process-guardian.js +212 -37
  44. package/src/process-memory.js +58 -23
  45. package/src/process-order.js +26 -0
  46. package/src/release-group.js +438 -45
  47. package/tensorbuzz.yml +20 -1
  48. package/test/completion.test.js +5 -2
  49. package/test/compose-config.test.js +160 -0
  50. package/test/compose-docker.test.js +297 -0
  51. package/test/compose-unit.test.js +415 -0
  52. package/test/config-validation.test.js +135 -5
  53. package/test/drain-policy.test.js +328 -0
  54. package/test/fixtures/deferred-boot-app.js +106 -0
  55. package/test/fixtures/gated-app.js +50 -0
  56. package/test/guardian-client.test.js +119 -5
  57. package/test/health.test.js +74 -1
  58. package/test/managed-process.test.js +260 -24
  59. package/test/owner-recovery.test.js +125 -21
  60. package/test/owner-replacement.test.js +175 -14
  61. package/test/process-memory.test.js +47 -1
  62. package/test/release-group.test.js +408 -2
  63. package/test/rollbridge.test.js +774 -72
  64. package/test/shutdown-completion.test.js +119 -0
  65. package/test/shutdown-retirement-bound.test.js +162 -0
  66. package/test/support/process.js +3 -1
  67. package/test/tensorbuzz-config.test.js +228 -0
package/src/daemon.js CHANGED
@@ -9,8 +9,10 @@ import {isDeepStrictEqual} from "node:util"
9
9
  import httpProxy from "http-proxy"
10
10
  import {loadConfig} from "./config.js"
11
11
  import {openControlSession} from "./control-client.js"
12
+ import {admitGenerationRelease} from "./drain-policy.js"
12
13
  import EventLog from "./event-log.js"
13
- import GuardianClient from "./guardian-client.js"
14
+ import GuardianClient, {guardianVersionFromProcess, usableSocketPath} from "./guardian-client.js"
15
+ import {childRecordIsGone} from "./managed-process.js"
14
16
  import ReleaseGroup from "./release-group.js"
15
17
  import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
16
18
  import {resolveGroupId, resolveUserId} from "./system-ids.js"
@@ -23,18 +25,20 @@ const STATE_PERSIST_INTERVAL_MS = 5000
23
25
  * @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
24
26
  * @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
25
27
  * @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
26
- * @typedef {{disruptive: boolean, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
28
+ * @typedef {{disruptive: true, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
29
+ * @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
27
30
  * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
28
31
  * @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
29
- * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
32
+ * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, guardian?: {guardianPid: number | null, ownerMutation: Record<string, JsonValue> | null, version: string | null}, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
30
33
  * @typedef {Omit<import("./managed-process.js").ManagedProcessStatus, "logs">} ManagedProcessStatusWithoutLogs
31
34
  * @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
32
35
  * @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
33
36
  * @typedef {Omit<DaemonStatus, "releases" | "services" | "singletons"> & {releases: ReleaseStatusWithoutLogs[], services: ProcessStatusWithoutLogs[], singletons: ProcessStatusWithoutLogs[]}} DaemonStatusWithoutLogs
37
+ * @typedef {{count: number, hold: string[], processes: {heldProcessGroupPgid: number | null, pid: number | null, processId: string, state: import("./managed-process.js").ManagedProcessState}[], state: import("./release-group.js").ReleaseState}} HeldGenerationHold
34
38
  * @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
35
39
  * @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
36
40
  * @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
37
- * @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
41
+ * @typedef {{boundaryCrossed: boolean, guardianVersion?: string, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, protocol: GuardianHandoffProtocol, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
38
42
  */
39
43
 
40
44
  export default class RollbridgeDaemon {
@@ -146,7 +150,7 @@ export default class RollbridgeDaemon {
146
150
  this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
147
151
  }
148
152
  return {}
149
- })
153
+ }, {waitingFor: "the retained release generations to settle"})
150
154
  } finally {
151
155
  this.generationTransitionRecovery = false
152
156
  }
@@ -203,7 +207,7 @@ export default class RollbridgeDaemon {
203
207
  if (!snapshot || snapshot.recovery?.guardian || !exactBootstrap) throw new Error("Retired owner reset requires an exact committed bootstrap with missing guardian identity")
204
208
  const control = await inspectControlSocket(this.config.control.path)
205
209
  if (control.alive) throw new Error(`Retired owner reset refuses responsive control owner at ${this.config.control.path}`)
206
- this.guardianIdentity = {socketPath: `${this.statePath}.guardian.sock`, token: crypto.randomBytes(32).toString("hex")}
210
+ this.guardianIdentity = {socketPath: usableSocketPath(`${this.statePath}.guardian.sock`), token: crypto.randomBytes(32).toString("hex")}
207
211
  this.guardian = new GuardianClient(this.guardianIdentity)
208
212
  await this.guardian.launch()
209
213
  this.guardianIdentity.pid = this.guardian.pid
@@ -221,10 +225,12 @@ export default class RollbridgeDaemon {
221
225
 
222
226
  if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
223
227
 
224
- const guardianIdentity = recovery?.guardian || {
225
- socketPath: `${this.statePath}.guardian.sock`,
226
- token: crypto.randomBytes(32).toString("hex")
227
- }
228
+ const guardianIdentity = recovery?.guardian
229
+ ? {...recovery.guardian, socketPath: usableSocketPath(recovery.guardian.socketPath)}
230
+ : {
231
+ socketPath: usableSocketPath(`${this.statePath}.guardian.sock`),
232
+ token: crypto.randomBytes(32).toString("hex")
233
+ }
228
234
  this.guardianIdentity = guardianIdentity
229
235
  this.guardian = new GuardianClient(guardianIdentity)
230
236
  if (recovery) {
@@ -406,11 +412,7 @@ export default class RollbridgeDaemon {
406
412
  shouldStart: () => !this.stopping
407
413
  })
408
414
 
409
- await release.restore(releaseStatus, {
410
- adoptCommittedGeneration: releaseStatus.releaseId === snapshot.activeReleaseId && snapshot.generationTransition?.phase === "committed" && snapshot.ownerTransition?.mode === "guardian-version-handoff",
411
- configDigest: ownerConfigDigest(config),
412
- synchronizeLifecycleRole: synchronizeLifecycleRoles
413
- })
415
+ await release.restore(releaseStatus, {synchronizeLifecycleRole: synchronizeLifecycleRoles})
414
416
  this.releases.set(release.releaseId, release)
415
417
  if (release.releaseId === snapshot.activeReleaseId) this.activeRelease = release
416
418
  }
@@ -504,15 +506,36 @@ export default class RollbridgeDaemon {
504
506
  /**
505
507
  * Prepares from private guardian state, starts listeners, then asks the committed owner
506
508
  * to atomically fence itself and transfer authority.
509
+ * @returns {Promise<void>} Resolves after the replacement is committed.
507
510
  */
508
511
  async replaceIncompatibleOwner() {
512
+ const maxAttempts = 3
513
+ let lastError
514
+
515
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
516
+ try {
517
+ return await this.replaceIncompatibleOwnerAttempt()
518
+ } catch (error) {
519
+ lastError = error
520
+ const diagnostic = error instanceof Error ? error.message : String(error)
521
+
522
+ if (!/^Guardian owner state changed after prepare \(prepared revision \d+, current revision \d+\); abort the stale candidate before preparing again$/.test(diagnostic) || attempt === maxAttempts) {
523
+ throw error
524
+ }
525
+ this.logger("owner replacement candidate became stale; re-preparing", {attempt, maxAttempts, error: diagnostic})
526
+ }
527
+ }
528
+ throw lastError
529
+ }
530
+
531
+ async replaceIncompatibleOwnerAttempt() {
509
532
  if (!this.statePath || !this.config.ownerRecovery) throw new Error("Atomic owner replacement requires ownerRecovery and the same statePath transaction anchor")
510
533
  this.controlCommandsReady = false
511
534
  const state = await readState(this.statePath)
512
535
  const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
513
536
 
514
537
  if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
515
- this.guardianIdentity = persisted.recovery.guardian
538
+ this.guardianIdentity = {...persisted.recovery.guardian, socketPath: usableSocketPath(persisted.recovery.guardian.socketPath)}
516
539
  this.guardian = new GuardianClient(this.guardianIdentity)
517
540
  await this.guardian.connect()
518
541
  this.watchOwnerReplacementEvents()
@@ -520,9 +543,16 @@ export default class RollbridgeDaemon {
520
543
  configDigest: persisted.recovery.configDigest,
521
544
  runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
522
545
  }
523
- const replacementProtocol = await this.guardian.ownerReplacementProtocol()
546
+ const guardianVersion = await this.retainedGuardianVersion()
547
+ const versionDrift = Boolean(this.runtime?.version && guardianVersion && guardianVersion !== this.runtime.version)
548
+ const replacementProtocol = versionDrift ? "legacy" : await this.guardian.ownerReplacementProtocol()
524
549
  const legacyBridge = replacementProtocol === "legacy"
525
- ? await this.prepareLegacyOwnerReplacement({persisted, persistedAuthority})
550
+ ? await this.prepareLegacyOwnerReplacement({
551
+ guardianVersion,
552
+ persisted,
553
+ persistedAuthority,
554
+ protocol: versionDrift ? "guardian-version-drift" : "legacy-protocol"
555
+ })
526
556
  : undefined
527
557
  const prepared = legacyBridge?.prepared ?? await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
528
558
  let preparedStatus
@@ -768,6 +798,33 @@ export default class RollbridgeDaemon {
768
798
  }
769
799
  }
770
800
 
801
+ /**
802
+ * Reads the retained guardian's package version without deciding the replacement protocol.
803
+ *
804
+ * The version is read from the guardian's own executable rather than its capabilities
805
+ * command: a pre-replacement guardian only answers that command once a replacement
806
+ * transaction is prepared, so asking here would move the probe ahead of the transaction
807
+ * every capability-gated guardian depends on. A guardian whose version cannot be read
808
+ * stays on the protocol classification path.
809
+ * @returns {Promise<string | undefined>} Retained guardian package version.
810
+ */
811
+ async retainedGuardianVersion() {
812
+ if (!this.guardian) throw new Error("Guardian version reconciliation requires the durable process guardian")
813
+ return await guardianVersionFromProcess(this.guardian.pid)
814
+ }
815
+
816
+ /**
817
+ * Names the last-resort refusal for a guardian this daemon cannot hand off automatically.
818
+ * @param {string | undefined} guardianVersion - Retained guardian package version.
819
+ * @param {string} cause - Measured reason the automatic handoff cannot run.
820
+ * @returns {Error} Refusal naming both versions and the exact repair command.
821
+ */
822
+ guardianVersionHandoffRefusal(guardianVersion, cause) {
823
+ const configArgument = this.configPath ? ` --config ${this.configPath}` : ""
824
+
825
+ return new Error(`The persistent Rollbridge guardian is version ${guardianVersion ?? "unknown"} while this daemon runs rollbridge ${this.runtime?.version ?? "unknown"}; ${cause}. Run "rollbridge shutdown${configArgument}" and then "rollbridge ensure-daemon${configArgument}" to replace the supervisor with this Rollbridge installation.`)
826
+ }
827
+
771
828
  /** Requires the persistent guardian protocol which can restart an accepted daemon. */
772
829
  async assertGuardianDaemonRecoveryCapability() {
773
830
  if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
@@ -776,10 +833,18 @@ export default class RollbridgeDaemon {
776
833
  try {
777
834
  capabilities = await this.guardian.capabilities()
778
835
  } catch (error) {
779
- if (!(error instanceof Error) || !["Guardian capabilities requires a process key", "Unknown guardian command: capabilities"].includes(error.message)) throw error
836
+ if (!(error instanceof Error) || !isGuardianDaemonRecoveryDiagnostic(error.message)) throw error
780
837
  throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
781
838
  }
782
839
  if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
840
+ const clientVersion = this.runtime?.version
841
+ const guardianVersion = capabilities.guardianVersion
842
+
843
+ if (!clientVersion) return
844
+ if (!guardianVersion) throw this.guardianVersionHandoffRefusal(guardianVersion, "its version cannot be attested, so no handoff can be authorised")
845
+ if (guardianVersion !== clientVersion) {
846
+ throw this.guardianVersionHandoffRefusal(guardianVersion, "this startup path has no incumbent daemon to hand guardian ownership from")
847
+ }
783
848
  }
784
849
 
785
850
  /**
@@ -900,13 +965,15 @@ export default class RollbridgeDaemon {
900
965
  }
901
966
 
902
967
  /**
903
- * Authenticates and prepares the one-time disruptive bridge for an exact legacy guardian protocol.
904
- * @param {{persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
968
+ * Authenticates and prepares the disruptive guardian-upgrade bridge, for a guardian that predates
969
+ * the replacement protocol or whose package version drifted from this daemon's runtime.
970
+ * @param {{guardianVersion?: string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}, protocol: GuardianHandoffProtocol}} options - Bridge evidence and why the bridge is required.
905
971
  * @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
906
972
  */
907
- async prepareLegacyOwnerReplacement({persisted, persistedAuthority}) {
973
+ async prepareLegacyOwnerReplacement({guardianVersion, persisted, persistedAuthority, protocol}) {
908
974
  const legacyProcessKey = ownerSnapshotProcessKeys(persisted, persisted.singletonReleaseIds)[0]
909
975
  const legacyGuardian = this.guardian
976
+ const versionHandoff = protocol === "guardian-version-drift"
910
977
  let legacyPrepared
911
978
 
912
979
  if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state")
@@ -939,12 +1006,23 @@ export default class RollbridgeDaemon {
939
1006
 
940
1007
  try {
941
1008
  if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
1009
+ if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "the handoff keeps the incumbent config identity unchanged and this deploy changes config authority")
942
1010
  throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
943
1011
  }
944
- if (!this.legacyIncumbentPid) throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
1012
+ if (!this.legacyIncumbentPid) {
1013
+ if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "ensure-daemon reported no incumbent daemon to hand guardian ownership from")
1014
+ throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
1015
+ }
945
1016
  if (!this.guardianIdentity?.pid) throw new Error("The authenticated legacy guardian state is missing its exact guardian PID")
946
1017
  await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
947
- const incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
1018
+ let incumbentStartTime
1019
+
1020
+ try {
1021
+ incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
1022
+ } catch (error) {
1023
+ if (!versionHandoff) throw error
1024
+ throw this.guardianVersionHandoffRefusal(guardianVersion, `the incumbent daemon PID ${this.legacyIncumbentPid} is not the attested owner: ${error instanceof Error ? error.message : String(error)}`)
1025
+ }
948
1026
  if (legacyPrepared) assertLegacyPrivateOwnerState(legacyPrepared.ownerState, persisted, persistedAuthority)
949
1027
  try {
950
1028
  incumbentControl = await openControlSession(persisted.control.path)
@@ -971,7 +1049,9 @@ export default class RollbridgeDaemon {
971
1049
 
972
1050
  assertLegacyGuardianInventoryMembership(legacyInventory, /** @type {PrivateOwnerState} */ (ownerState))
973
1051
  const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
974
- socketPath: `${this.statePath}.split3-guardian.sock`,
1052
+ socketPath: usableSocketPath(versionHandoff
1053
+ ? `${this.statePath}.guardian-${this.runtime?.version}.sock`
1054
+ : `${this.statePath}.split3-guardian.sock`),
975
1055
  token: crypto.randomBytes(32).toString("hex")
976
1056
  })
977
1057
 
@@ -986,12 +1066,15 @@ export default class RollbridgeDaemon {
986
1066
  recovery: {...persisted.recovery, guardian: upgradedIdentity}
987
1067
  })
988
1068
 
989
- this.logger("legacy owner replacement bridge prepared", {
1069
+ this.logger("guardian upgrade bridge prepared", {
990
1070
  guardianPid: upgradedIdentity.pid ?? null,
1071
+ guardianVersion: guardianVersion ?? "unknown",
991
1072
  incumbentPid: this.legacyIncumbentPid,
992
- replacementId: prepared.replacementId
1073
+ protocol,
1074
+ replacementId: prepared.replacementId,
1075
+ runtimeVersion: this.runtime?.version ?? "unknown"
993
1076
  })
994
- return {boundaryCrossed: false, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, recoverySnapshot}
1077
+ return {boundaryCrossed: false, guardianVersion, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, protocol, recoverySnapshot}
995
1078
  } catch (upgradeError) {
996
1079
  incumbentControl?.close()
997
1080
  let cleanupError
@@ -1071,10 +1154,20 @@ export default class RollbridgeDaemon {
1071
1154
 
1072
1155
  if (staged.committed) throw new Error("Legacy guardian committed replacement before the authenticated disruptive boundary")
1073
1156
  }
1074
- this.logger("legacy owner replacement disruptive boundary", {
1157
+ const boundaryReason = bridge.protocol === "guardian-version-drift"
1158
+ ? `retained guardian ${bridge.guardianVersion ?? "unknown"} predates this daemon runtime ${this.runtime?.version ?? "unknown"}`
1159
+ : "retained guardian and daemon lacked atomic replacement protocol"
1160
+
1161
+ this.logger("guardian upgrade bridge crossed", {
1075
1162
  disruptive: true,
1163
+ guardianVersion: bridge.guardianVersion ?? "unknown",
1076
1164
  incumbentPid: this.legacyIncumbentPid,
1077
- reason: "retained guardian and daemon lacked atomic replacement protocol"
1165
+ protocol: bridge.protocol,
1166
+ reason: boundaryReason,
1167
+ // The retained guardian keeps its own children; the boundary retires only the incumbent daemon.
1168
+ retainedGuardianPid: bridge.legacyGuardian.pid ?? null,
1169
+ retainedGuardianProcesses: bridge.legacyInventory.length,
1170
+ runtimeVersion: this.runtime?.version ?? "unknown"
1078
1171
  })
1079
1172
  await this.guardian.beginLegacyOwnerClaim(
1080
1173
  bridge.prepared.replacementId,
@@ -1090,8 +1183,8 @@ export default class RollbridgeDaemon {
1090
1183
  await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
1091
1184
  this.ownerTransition = /** @type {OwnerTransition} */ ({
1092
1185
  disruptive: true,
1093
- mode: "legacy-first-upgrade",
1094
- reason: "retained guardian and daemon lacked atomic replacement protocol"
1186
+ mode: bridge.protocol === "guardian-version-drift" ? "guardian-version-handoff" : "legacy-first-upgrade",
1187
+ reason: boundaryReason
1095
1188
  })
1096
1189
  }
1097
1190
 
@@ -1393,7 +1486,7 @@ export default class RollbridgeDaemon {
1393
1486
  releaseId: stringOrUndefined(data.releaseId),
1394
1487
  releasePath: requiredString(data.releasePath, "releasePath"),
1395
1488
  revision: stringOrUndefined(data.revision)
1396
- }))
1489
+ }), {waitingFor: "candidate activation and the retired generation handoff"})
1397
1490
  }
1398
1491
 
1399
1492
  if (commandName === "recover-generation-transition") {
@@ -1403,11 +1496,22 @@ export default class RollbridgeDaemon {
1403
1496
  releaseId: requiredString(data.releaseId, "releaseId"),
1404
1497
  releasePath: requiredString(data.releasePath, "releasePath"),
1405
1498
  revision: requiredString(data.revision, "revision")
1406
- }))
1499
+ }), {waitingFor: "the retained release generations to settle"})
1407
1500
  }
1408
1501
 
1409
1502
  if (commandName === "status") {
1410
- return this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
1503
+ const status = this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
1504
+ let guardian
1505
+
1506
+ try {
1507
+ const diagnostics = await this.guardian?.guardianStatus()
1508
+
1509
+ guardian = diagnostics ? {guardianPid: diagnostics.guardianPid, ownerMutation: diagnostics.ownerMutation, version: diagnostics.version} : {guardianPid: null, ownerMutation: null, version: null}
1510
+ } catch (error) {
1511
+ guardian = {error: error instanceof Error ? error.message : String(error), guardianPid: this.guardianIdentity?.pid ?? null, ownerMutation: null, version: null}
1512
+ }
1513
+
1514
+ return {...status, guardian}
1411
1515
  }
1412
1516
 
1413
1517
  if (commandName === "events") {
@@ -1418,18 +1522,18 @@ export default class RollbridgeDaemon {
1418
1522
  return await this.executeOwnerMutation("stop", async () => {
1419
1523
  await this.stopRelease(stringOrUndefined(data.releaseId))
1420
1524
  return this.status()
1421
- })
1525
+ }, {waitingFor: "the release's managed processes to stop"})
1422
1526
  }
1423
1527
 
1424
1528
  if (commandName === "restart") {
1425
1529
  return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
1426
1530
  policy: stringOrUndefined(data.policy),
1427
1531
  processId: stringOrUndefined(data.processId)
1428
- }))
1532
+ }), {waitingFor: "the managed process to stop and start again"})
1429
1533
  }
1430
1534
 
1431
1535
  if (commandName === "rollback") {
1432
- return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
1536
+ return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}), {waitingFor: "the previous release to activate"})
1433
1537
  }
1434
1538
 
1435
1539
  if (commandName === "shutdown") {
@@ -1493,11 +1597,13 @@ export default class RollbridgeDaemon {
1493
1597
  * Runs one owner mutation under the guardian's prepare/commit exclusion fence.
1494
1598
  * @param {string} operation - Mutation diagnostic name.
1495
1599
  * @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
1600
+ * @param {{waitingFor?: string}} [options] - What this mutation blocks on, so a refused
1601
+ * later mutation names the real blocker instead of just "in progress".
1496
1602
  * @returns {Promise<Record<string, JsonValue>>} Operation result.
1497
1603
  */
1498
- async executeOwnerMutation(operation, callback) {
1604
+ async executeOwnerMutation(operation, callback, {waitingFor} = {}) {
1499
1605
  if (!this.guardian) return await callback()
1500
- const mutationId = await this.guardian.beginOwnerMutation(operation)
1606
+ const mutationId = await this.guardian.beginOwnerMutation(operation, {ownerPid: process.pid, waitingFor})
1501
1607
  let result
1502
1608
  let operationError
1503
1609
 
@@ -1633,6 +1739,46 @@ export default class RollbridgeDaemon {
1633
1739
  await this.updateGenerationTransition("restoring_committed")
1634
1740
  return await this.resumeGenerationTransition()
1635
1741
  }
1742
+ await this.escalateWedgedGenerations(nextConfig, newReleaseId)
1743
+ // A release id that is still retained by a generation holding live work cannot be started
1744
+ // a second time: the candidate's processes re-register the guardian keys that the retained
1745
+ // generation's own processes still hold, so the deploy aborts partway through `start()`
1746
+ // with a bare `Guardian provenance mismatch` — after the caller has committed to this
1747
+ // revision and, for a restore, after the deployment pointer has already moved. Refusing
1748
+ // from the retained record before any process starts keeps the incumbent serving, names the
1749
+ // release the guardian reports as active, and leaves the retained generation intact for
1750
+ // `rollbridge status` to reconcile against. Journaled transition resume above is exempt:
1751
+ // it re-drives the transaction that already owns this release id.
1752
+ //
1753
+ // The escalation above runs first on purpose: it is what clears the residue that made this
1754
+ // guard refuse the measured 2026-11-15 and 2026-11-16 deploys. A wedge that escalation can
1755
+ // prove is kill-safe stops holding its keys and the handoff proceeds; a generation that
1756
+ // still holds live work is not killed and is named here instead of aborting mid-`start()`.
1757
+ const heldGeneration = this.retainedGenerationHoldingLiveWork(newReleaseId)
1758
+
1759
+ if (heldGeneration) {
1760
+ const activeReleaseId = this.activeRelease?.releaseId ?? null
1761
+
1762
+ this.logger("deploy refused: retained generation still holds live processes", {
1763
+ activeReleaseId,
1764
+ heldProcesses: heldGeneration.processes,
1765
+ releaseId: newReleaseId,
1766
+ retainedState: heldGeneration.state
1767
+ })
1768
+
1769
+ throw new Error(`Deploy refused before activation: release "${newReleaseId}" is retained in state ${heldGeneration.state} while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; deploy a new release id or reconcile the deployment pointer with \`rollbridge status\`.`)
1770
+ }
1771
+ const admission = this.evaluateGenerationAdmission(newReleaseId, nextConfig)
1772
+
1773
+ if (!admission.allowed) {
1774
+ this.logger("deploy refused by draining-generation bound", {
1775
+ candidateReleaseId: newReleaseId,
1776
+ draining: admission.draining.map((group) => ({releaseId: group.releaseId, status: group.status})),
1777
+ limit: nextConfig.releaseRetention.maxConcurrentDrainingReleaseGroups,
1778
+ wouldBeDrainingCount: admission.wouldBeDrainingCount
1779
+ })
1780
+ throw new Error(/** @type {string} */ (admission.reason))
1781
+ }
1636
1782
  const release = new ReleaseGroup({
1637
1783
  config: nextConfig,
1638
1784
  logger: this.logger,
@@ -1689,6 +1835,79 @@ export default class RollbridgeDaemon {
1689
1835
  return await this.resumeGenerationTransition()
1690
1836
  }
1691
1837
 
1838
+ /**
1839
+ * Applies the draining-generation bound before a handoff.
1840
+ *
1841
+ * Without this, every deploy retires one more generation and the backlog grows
1842
+ * monotonically whenever a drain outlasts the deploy cadence; a consumer that checks the
1843
+ * count after activation reports the growth it cannot undo. Refusing here names the
1844
+ * draining groups and whether each is still progressing or wedged, before any process of
1845
+ * the candidate is started.
1846
+ * @param {string} candidateReleaseId - Release the handoff would activate.
1847
+ * @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
1848
+ * @returns {import("./drain-policy.js").GenerationAdmission} Admission decision and the groups behind it.
1849
+ */
1850
+ evaluateGenerationAdmission(candidateReleaseId, config) {
1851
+ const draining = [...this.releases.values()]
1852
+ .filter((release) => release.state === "draining")
1853
+ .map((release) => {
1854
+ const drain = release.drainStatus()
1855
+
1856
+ return {reason: drain?.reason ?? "no liveness observation recorded", releaseId: release.releaseId, status: drain?.status ?? "unproven"}
1857
+ })
1858
+
1859
+ return admitGenerationRelease({
1860
+ candidateReleaseId,
1861
+ draining,
1862
+ incumbentWillDrain: this.activeRelease !== undefined,
1863
+ maxConcurrentDrainingReleaseGroups: config.releaseRetention.maxConcurrentDrainingReleaseGroups
1864
+ })
1865
+ }
1866
+
1867
+ /**
1868
+ * Reports whether a retained release group still holds live work that a re-activation would collide with.
1869
+ *
1870
+ * `stopped` is not the same claim as `drained`: the guardian's provenance fence only lets a
1871
+ * process key be re-registered once the record that holds it is gone. A generation recorded
1872
+ * as drained while its process group is still alive — the shape an indefinite worker drain
1873
+ * produces — still holds its keys, so its release id is not available for a restore.
1874
+ * @param {string} releaseId - Release id a handoff would activate.
1875
+ * @returns {HeldGenerationHold | undefined} The live hold, or undefined when the id is free.
1876
+ */
1877
+ retainedGenerationHoldingLiveWork(releaseId) {
1878
+ const retained = this.releases.get(releaseId)
1879
+
1880
+ if (!retained || retained === this.activeRelease) return undefined
1881
+
1882
+ return heldGenerationHold(retained)
1883
+ }
1884
+
1885
+ /**
1886
+ * Gives every retiring generation the escalation decision its own retirement left open,
1887
+ * before the handoff is judged.
1888
+ *
1889
+ * A deploy is the event that makes a wedged generation expensive: until something tries to
1890
+ * activate a successor, a retired release holding a live process group costs nothing
1891
+ * observable. Acting here means the observation that proves the wedge and the action on it
1892
+ * come from the same read the admission bound already performs, so no sampler has to run
1893
+ * alongside the daemon. Generations that already recorded `release drained` are included:
1894
+ * their retirement finished, and the process group they left behind is the residue that
1895
+ * blocked the measured 2026-11-15 deploys.
1896
+ * @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
1897
+ * @param {string} candidateReleaseId - Release the handoff would activate.
1898
+ * @returns {Promise<void>} Resolves once every retiring generation has been decided.
1899
+ */
1900
+ async escalateWedgedGenerations(config, candidateReleaseId) {
1901
+ if (!config.releaseRetention.escalateWedgedGenerations) return
1902
+
1903
+ for (const release of this.releases.values()) {
1904
+ if (release.releaseId === candidateReleaseId) continue
1905
+ if (release.state !== "draining" && release.state !== "stopped" && release.state !== "failed") continue
1906
+
1907
+ await release.escalateWedgedGeneration({reason: `deploy of ${candidateReleaseId}`})
1908
+ }
1909
+ }
1910
+
1692
1911
  /**
1693
1912
  * Continues one exact durable generation transition without an internal retry loop.
1694
1913
  * @returns {Promise<Record<string, JsonValue>>} Deploy result after commit.
@@ -1710,10 +1929,9 @@ export default class RollbridgeDaemon {
1710
1929
  let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
1711
1930
 
1712
1931
  if (transition.phase === "candidate_ready") {
1713
- // Candidate processes are already started and health-checked. Keep the incumbent
1714
- // serving until the candidate has acknowledged activation and traffic has switched;
1715
- // retiring it here can stop the only live web/jobs generation before its successor
1716
- // carries traffic (STOE-2269).
1932
+ // The candidate has already started and passed its health gates. Keep the
1933
+ // incumbent authoritative until the candidate is activated and traffic is
1934
+ // switched; retirement is deliberately launched from committed_pending.
1717
1935
  await this.updateGenerationTransition("activating_candidate")
1718
1936
  }
1719
1937
 
@@ -1788,6 +2006,22 @@ export default class RollbridgeDaemon {
1788
2006
 
1789
2007
  if (transition.phase === "committed_pending") {
1790
2008
  await this.refreshServiceDefinitions(release)
2009
+ if (previousRelease?.state === "active") {
2010
+ if (this.activeRelease !== release) {
2011
+ throw new Error(`Cannot retire incumbent release ${previousRelease.releaseId} before candidate release ${release.releaseId} is authoritative.`)
2012
+ }
2013
+ const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
2014
+
2015
+ // Traffic is already on the healthy candidate. Retirement must not hold
2016
+ // the deploy mutation or its owner fence; its own bounded drain reports
2017
+ // failures and escalates any process that misses the retirement budget.
2018
+ void previousRelease.beginRetirement(retirementConfig)
2019
+ .catch((error) => {
2020
+ retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
2021
+ this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
2022
+ })
2023
+ .then(() => this.drainAndPrune(previousRelease, retirementConfig))
2024
+ }
1791
2025
  await this.checkpointGenerationTransition()
1792
2026
  try {
1793
2027
  await this.replaceSingletons(release)
@@ -1795,29 +2029,12 @@ export default class RollbridgeDaemon {
1795
2029
  await this.failGenerationTransition(error instanceof Error ? error.message : String(error))
1796
2030
  throw error
1797
2031
  }
1798
- if (previousRelease) {
1799
- const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
1800
-
1801
- // Traffic is already on the candidate and singleton ownership has been
1802
- // transferred. Retirement is deliberately detached so a slow old generation
1803
- // cannot hold the committed deploy open or race guardian-owned replacement.
1804
- void this.drainAndPrune(previousRelease, retirementConfig)
1805
- }
1806
2032
  transition.phase = "committed"
1807
2033
  transition.error = undefined
1808
2034
  transition.journalRevision = (transition.journalRevision ?? 0) + 1
1809
2035
  transition.updatedAt = new Date().toISOString()
1810
2036
  this.pruneStoppedReleases()
1811
2037
  await this.checkpointGenerationTransition()
1812
- if (previousRelease) {
1813
- const retirementConfig = !activationLifecycle && previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
1814
-
1815
- // Traffic is already on the healthy candidate. Retirement is deliberately detached
1816
- // from the owner mutation so a slow quiet/drain hook can never hold the deploy fence
1817
- // or make the active successor unavailable. drainAndStop applies the bounded force
1818
- // deadline and records any retirement failure on the old generation.
1819
- void this.drainAndPrune(previousRelease, retirementConfig)
1820
- }
1821
2038
  }
1822
2039
 
1823
2040
  return {
@@ -1847,6 +2064,8 @@ export default class RollbridgeDaemon {
1847
2064
  if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
1848
2065
  if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
1849
2066
  if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
2067
+ /** @type {Promise<void> | undefined} */
2068
+ let candidateDrain
1850
2069
 
1851
2070
  if (transition.phase === "activating_candidate") {
1852
2071
  if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
@@ -1878,6 +2097,15 @@ export default class RollbridgeDaemon {
1878
2097
  transition.compensationError = undefined
1879
2098
  await this.updateGenerationTransition("restoring_previous")
1880
2099
  this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
2100
+
2101
+ // The candidate's bounded drain starts here, before the restoration attempt. The only other
2102
+ // compensation drain runs after the success path, so when incumbent restoration throws the
2103
+ // candidate is left `draining` with a live process: measured 2026-11-17, candidate
2104
+ // `20261117063703` still held `web` pid 3201928 at 07:30Z after its compensation threw at
2105
+ // 06:42:31Z, and reached `release drained` only when a later compensation reached the
2106
+ // success path at 08:47:00Z. A retained generation holding live work is what refuses every
2107
+ // later deploy, and a failed candidate is never a restoration target.
2108
+ candidateDrain = this.drainAndPrune(candidate, candidate.config)
1881
2109
  }
1882
2110
 
1883
2111
  if (transition.phase === "restoring_previous") {
@@ -1889,10 +2117,12 @@ export default class RollbridgeDaemon {
1889
2117
 
1890
2118
  transition.compensationError = failure
1891
2119
  await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
2120
+ await previous.stopProxyTraffic()
1892
2121
  this.logger("release generation compensation restoration failed", {
1893
2122
  activationError: transition.activationError,
1894
2123
  error: failure,
1895
- releaseId: previous.releaseId
2124
+ releaseId: previous.releaseId,
2125
+ traffic: "unavailable"
1896
2126
  })
1897
2127
  throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
1898
2128
  }
@@ -1931,7 +2161,7 @@ export default class RollbridgeDaemon {
1931
2161
  this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
1932
2162
  throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
1933
2163
  }
1934
- void this.drainAndPrune(candidate, candidate.config)
2164
+ if (candidateDrain === undefined) void this.drainAndPrune(candidate, candidate.config)
1935
2165
  this.logger("release generation compensation completed", result)
1936
2166
  return result
1937
2167
  }
@@ -1955,6 +2185,19 @@ export default class RollbridgeDaemon {
1955
2185
  recoveryStatus: "already_recovered"
1956
2186
  }
1957
2187
  }
2188
+ // A committed journal describes the transition that already published the current
2189
+ // incumbent, not an in-flight transition to recover. When the recovery fence names
2190
+ // that live incumbent, the failed deploy never journaled its own transition, so the
2191
+ // only fence that describes the authoritative release is the completed one: recovery
2192
+ // is already done. In-flight phases keep the stale-fence guard failing closed.
2193
+ if (transition.phase === "committed" && transition.candidateReleaseId === identity.previousReleaseId && this.activeRelease?.releaseId === identity.previousReleaseId && this.releases.get(transition.candidateReleaseId) === this.activeRelease) {
2194
+ return {
2195
+ activeReleaseId: this.activeRelease.releaseId,
2196
+ candidateReleaseId: identity.releaseId,
2197
+ previousReleaseId: identity.previousReleaseId,
2198
+ recoveryStatus: "already_recovered"
2199
+ }
2200
+ }
1958
2201
  if (transition.previousReleaseId !== identity.previousReleaseId) {
1959
2202
  throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
1960
2203
  }
@@ -2401,11 +2644,38 @@ export default class RollbridgeDaemon {
2401
2644
  throw new Error(`Release "${target.releaseId}" is already active.`)
2402
2645
  }
2403
2646
 
2647
+ // A generation recorded as `stopped` whose processes are still alive is the shape a
2648
+ // restore cannot honour: re-deploying that release id re-registers its guardian process
2649
+ // keys under a new provenance while the old processes still hold them, and the guardian
2650
+ // answers with a bare provenance mismatch. Use the guardian's own fence here so the
2651
+ // refusal is decided before anything is touched, and name the divergence — including a
2652
+ // process group that outlived the tracked process and is invisible as a pid — together
2653
+ // with the release the guardian actually reports as active, so the deployment tool can
2654
+ // reconcile its own current pointer instead of restoring a generation that already drained.
2655
+ // A target that is still draining is not refused here: stopping it is what makes its id
2656
+ // reusable, and the deploy below applies the same fence after that stop.
2657
+ const heldGeneration = target.state === "stopped" ? heldGenerationHold(target) : undefined
2658
+
2659
+ if (heldGeneration) {
2660
+ const activeReleaseId = this.activeRelease?.releaseId ?? null
2661
+
2662
+ this.logger("rollback refused: drained generation still holds live processes", {
2663
+ activeReleaseId,
2664
+ heldProcesses: heldGeneration.processes,
2665
+ releaseId: target.releaseId,
2666
+ revision: target.revision
2667
+ })
2668
+
2669
+ throw new Error(`Release "${target.releaseId}" is recorded as drained while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; reconcile the deployment pointer with \`rollbridge status\` before restoring.`)
2670
+ }
2671
+
2404
2672
  // The target may still be draining a prior deploy (live processes). Stop it before the
2405
2673
  // deploy below re-uses its id in this.releases, otherwise the still-running instance
2406
2674
  // would be dropped from status/pruning/shutdown and could be orphaned.
2675
+ // Rollback is an owner mutation too, so this stop carries the same retirement bound as
2676
+ // `stop`: a process the supervisor must not kill must not be able to hold the fence.
2407
2677
  if (target.state !== "stopped" && target.state !== "failed") {
2408
- await target.stop()
2678
+ await target.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})
2409
2679
  }
2410
2680
 
2411
2681
  this.logger("rollback starting", {releaseId: target.releaseId, releasePath: target.releasePath})
@@ -2626,7 +2896,7 @@ export default class RollbridgeDaemon {
2626
2896
 
2627
2897
  /**
2628
2898
  * @param {string | undefined} releaseId - Release id, or active release when omitted.
2629
- * @returns {Promise<void>} Resolves when stopped.
2899
+ * @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
2630
2900
  */
2631
2901
  async stopRelease(releaseId) {
2632
2902
  this.assertNoUnresolvedGenerationTransition("stop")
@@ -2635,7 +2905,10 @@ export default class RollbridgeDaemon {
2635
2905
  if (!release) throw new Error(`Release not found: ${releaseId || "active"}`)
2636
2906
  if (release === this.activeRelease) this.activeRelease = undefined
2637
2907
 
2638
- await release.stop()
2908
+ // This command runs inside the owner mutation, so an unbudgeted wait for a process the
2909
+ // supervisor must not kill holds the fence for the whole drain and every later deploy is
2910
+ // refused. The retirement bound is the same one the deploy path already uses.
2911
+ await release.stop({retirementTimeoutMs: this.config.proxy.nonBlockingDrainTimeoutMs})
2639
2912
  this.logger("release stopped", {releaseId: release.releaseId})
2640
2913
  this.pruneStoppedReleases()
2641
2914
  this.persistState()
@@ -2840,11 +3113,16 @@ export default class RollbridgeDaemon {
2840
3113
  await this.guardian.retireOwner()
2841
3114
  this.guardian.disconnect()
2842
3115
  } else {
3116
+ // Without a guardian nobody else can retire these processes, so an unbounded stop would
3117
+ // leave the generation draining forever — the state that made the measured 2026-11-17
3118
+ // rollback refuse. The same retirement budget terminalizes it and names the residue.
3119
+ const retirementTimeoutMs = this.config.proxy.nonBlockingDrainTimeoutMs
3120
+
2843
3121
  void Promise.allSettled([
2844
- ...[...this.services.values()].map((processInstance) => processInstance.stop()),
2845
- ...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
2846
- ...[...this.startingReleases].map((release) => release.stop()),
2847
- ...[...this.releases.values()].map((release) => release.stop())
3122
+ ...[...this.services.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
3123
+ ...[...this.singletons.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
3124
+ ...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs})),
3125
+ ...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs}))
2848
3126
  ])
2849
3127
  }
2850
3128
  this.logger("external owner retired", {attestation, status: "draining"})
@@ -2878,12 +3156,19 @@ export default class RollbridgeDaemon {
2878
3156
  }
2879
3157
 
2880
3158
  await captureShutdownError(cleanupErrors, "proxy close", async () => this.proxy.close())
3159
+ // Shutdown must return even when a process refuses to exit. The bound is the
3160
+ // `proxy.nonBlockingDrainTimeoutMs` an explicit `rollbridge stop` already uses, and it means
3161
+ // "stop waiting", never "SIGKILL": a process whose graceful window is `"indefinite"` is left
3162
+ // draining rather than killed. Left unbudgeted, that stop keeps the daemon alive after its
3163
+ // control socket is already unlinked — alive, unreachable, and still holding the generation it
3164
+ // stopped: the wedged daemon STOE-2275 measured behind 4,800 `connect ECONNREFUSED` proxy errors.
3165
+ const retirementTimeoutMs = this.config.proxy.nonBlockingDrainTimeoutMs
2881
3166
  const dependentStopResults = await Promise.allSettled([
2882
- ...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
2883
- ...[...this.startingReleases].map((release) => release.stop()),
2884
- ...[...this.releases.values()].map((release) => release.stop())
3167
+ ...[...this.singletons.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})),
3168
+ ...[...this.startingReleases].map((release) => release.stop({retirementTimeoutMs})),
3169
+ ...[...this.releases.values()].map((release) => release.stop({retirementTimeoutMs}))
2885
3170
  ])
2886
- const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
3171
+ const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop({retirementTimeoutMs})))
2887
3172
  const stopResults = [...dependentStopResults, ...serviceStopResults]
2888
3173
  const guardian = this.guardian
2889
3174
  if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
@@ -3067,6 +3352,42 @@ function statusIncludeLogs(value) {
3067
3352
  return value
3068
3353
  }
3069
3354
 
3355
+ /**
3356
+ * Names the live work a retained release group still holds, using the guardian's own fence.
3357
+ *
3358
+ * A process counts as holding work when the guardian would refuse to hand its key to a new
3359
+ * provenance: a running or stopping process, or a record already `stopped` whose process group
3360
+ * outlived it and is invisible as a pid.
3361
+ * @param {ReleaseGroup} release - Retained generation to inspect.
3362
+ * @returns {HeldGenerationHold | undefined} The hold and its log payload, or undefined when every process is gone.
3363
+ */
3364
+ function heldGenerationHold(release) {
3365
+ const heldProcesses = release.status().processes.filter((processStatus) => !childRecordIsGone(processStatus))
3366
+
3367
+ if (heldProcesses.length === 0) return undefined
3368
+
3369
+ return {
3370
+ count: heldProcesses.length,
3371
+ hold: heldProcesses.map((processStatus) => {
3372
+ if (processStatus.heldProcessGroup) {
3373
+ const {liveMemberCount, pgid} = processStatus.heldProcessGroup
3374
+ const members = liveMemberCount === undefined ? "" : ` with ${liveMemberCount} live member(s)`
3375
+
3376
+ return `${processStatus.id}: process group ${pgid}${members} still alive after the tracked process exited`
3377
+ }
3378
+
3379
+ return `${processStatus.id}: state ${processStatus.state}, pid ${processStatus.pid ?? "unknown"}`
3380
+ }),
3381
+ processes: heldProcesses.map((processStatus) => ({
3382
+ heldProcessGroupPgid: processStatus.heldProcessGroup?.pgid ?? null,
3383
+ pid: processStatus.pid ?? null,
3384
+ processId: processStatus.id,
3385
+ state: processStatus.state
3386
+ })),
3387
+ state: release.state
3388
+ }
3389
+ }
3390
+
3070
3391
  /**
3071
3392
  * @param {PrivateOwnerState} transfer - Authenticated private incumbent state.
3072
3393
  * @param {GenerationTransition} transition - Unresolved generation transition.
@@ -3127,6 +3448,15 @@ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
3127
3448
  diagnostic === "Unknown guardian command: prepare-owner-replacement"
3128
3449
  }
3129
3450
 
3451
+ /**
3452
+ * Reports the capability diagnostics that identify a guardian predating daemon recovery.
3453
+ * @param {string} diagnostic - Guardian error message.
3454
+ * @returns {boolean} Whether the guardian predates the daemon recovery capability.
3455
+ */
3456
+ function isGuardianDaemonRecoveryDiagnostic(diagnostic) {
3457
+ return diagnostic === "Guardian capabilities requires a process key" || diagnostic === "Unknown guardian command: capabilities"
3458
+ }
3459
+
3130
3460
  /**
3131
3461
  * @param {DaemonStatus} snapshot - Serialized owner process snapshot.
3132
3462
  * @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
@@ -3449,7 +3779,7 @@ function requiredNonNegativeInteger(value, key) {
3449
3779
  /**
3450
3780
  * Selects stopped releases to prune by the retention policy, keeping the most recent.
3451
3781
  * @param {PrunableRelease[]} releases - Status of all tracked releases, in deploy order (oldest first).
3452
- * @param {import("./config.js").ReleaseRetentionConfig} policy - Retention policy.
3782
+ * @param {{keep: number, maxAgeMs: number}} policy - Retention policy; draining generations are never pruned here.
3453
3783
  * @param {number} now - Current epoch milliseconds.
3454
3784
  * @returns {string[]} Release ids to remove.
3455
3785
  */
@@ -3474,7 +3804,7 @@ export function releasesToPrune(releases, policy, now) {
3474
3804
  }
3475
3805
 
3476
3806
  /**
3477
- * @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, proxy?: {host: string, port: number}}} ControlSocketInspection
3807
+ * @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, guardian?: {ownerMutation: Record<string, JsonValue> | null, version: string | null}, proxy?: {host: string, port: number}}} ControlSocketInspection
3478
3808
  */
3479
3809
 
3480
3810
  /**
@@ -3549,7 +3879,7 @@ export async function inspectControlSocket(socketPath, timeoutMs = 1000) {
3549
3879
  /**
3550
3880
  * Parses a control status response line into a Rollbridge identity, if it is one.
3551
3881
  * @param {string} line - JSON response line.
3552
- * @returns {{application: string, activeReleaseId: string | null, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
3882
+ * @returns {{application: string, activeReleaseId: string | null, guardian: {ownerMutation: Record<string, JsonValue> | null, version: string | null} | undefined, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
3553
3883
  */
3554
3884
  function parseControlStatus(line) {
3555
3885
  /** @type {JsonValue} */
@@ -3566,5 +3896,8 @@ function parseControlStatus(line) {
3566
3896
 
3567
3897
  const proxy = "proxy" in parsed && parsed.proxy && typeof parsed.proxy === "object" && !Array.isArray(parsed.proxy) && typeof parsed.proxy.host === "string" && typeof parsed.proxy.port === "number" ? {host: parsed.proxy.host, port: parsed.proxy.port} : undefined
3568
3898
 
3569
- return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, proxy}
3899
+ const guardianValue = "guardian" in parsed && parsed.guardian && typeof parsed.guardian === "object" && !Array.isArray(parsed.guardian) ? parsed.guardian : undefined
3900
+ const guardian = guardianValue && typeof guardianValue.version === "string" || guardianValue && guardianValue.version === null ? {ownerMutation: guardianValue.ownerMutation && typeof guardianValue.ownerMutation === "object" && !Array.isArray(guardianValue.ownerMutation) ? guardianValue.ownerMutation : null, version: guardianValue.version} : undefined
3901
+
3902
+ return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, guardian, proxy}
3570
3903
  }