rollbridge 0.1.68 → 0.1.70

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.js CHANGED
@@ -11,7 +11,7 @@ import {loadConfig} from "./config.js"
11
11
  import {openControlSession} from "./control-client.js"
12
12
  import {admitGenerationRelease} from "./drain-policy.js"
13
13
  import EventLog from "./event-log.js"
14
- import GuardianClient from "./guardian-client.js"
14
+ import GuardianClient, {guardianVersionFromProcess, usableSocketPath} from "./guardian-client.js"
15
15
  import {childRecordIsGone} from "./managed-process.js"
16
16
  import ReleaseGroup from "./release-group.js"
17
17
  import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
@@ -25,10 +25,11 @@ const STATE_PERSIST_INTERVAL_MS = 5000
25
25
  * @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
26
26
  * @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
27
27
  * @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
28
- * @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
28
+ * @typedef {{disruptive: true, mode: "guardian-version-handoff" | "legacy-first-upgrade", reason: string}} OwnerTransition
29
+ * @typedef {"guardian-version-drift" | "legacy-protocol"} GuardianHandoffProtocol
29
30
  * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "degraded_active" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
30
31
  * @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, degradedIncumbent?: boolean, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
31
- * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
32
+ * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, guardian?: {guardianPid: number | null, ownerMutation: Record<string, JsonValue> | null, version: string | null}, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
32
33
  * @typedef {Omit<import("./managed-process.js").ManagedProcessStatus, "logs">} ManagedProcessStatusWithoutLogs
33
34
  * @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
34
35
  * @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
@@ -37,7 +38,7 @@ const STATE_PERSIST_INTERVAL_MS = 5000
37
38
  * @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
38
39
  * @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
39
40
  * @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
40
- * @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
41
+ * @typedef {{boundaryCrossed: boolean, guardianVersion?: string, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, protocol: GuardianHandoffProtocol, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
41
42
  */
42
43
 
43
44
  export default class RollbridgeDaemon {
@@ -206,7 +207,7 @@ export default class RollbridgeDaemon {
206
207
  if (!snapshot || snapshot.recovery?.guardian || !exactBootstrap) throw new Error("Retired owner reset requires an exact committed bootstrap with missing guardian identity")
207
208
  const control = await inspectControlSocket(this.config.control.path)
208
209
  if (control.alive) throw new Error(`Retired owner reset refuses responsive control owner at ${this.config.control.path}`)
209
- this.guardianIdentity = {socketPath: `${this.statePath}.guardian.sock`, token: crypto.randomBytes(32).toString("hex")}
210
+ this.guardianIdentity = {socketPath: usableSocketPath(`${this.statePath}.guardian.sock`), token: crypto.randomBytes(32).toString("hex")}
210
211
  this.guardian = new GuardianClient(this.guardianIdentity)
211
212
  await this.guardian.launch()
212
213
  this.guardianIdentity.pid = this.guardian.pid
@@ -224,10 +225,12 @@ export default class RollbridgeDaemon {
224
225
 
225
226
  if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
226
227
 
227
- const guardianIdentity = recovery?.guardian || {
228
- socketPath: `${this.statePath}.guardian.sock`,
229
- token: crypto.randomBytes(32).toString("hex")
230
- }
228
+ const guardianIdentity = recovery?.guardian
229
+ ? {...recovery.guardian, socketPath: usableSocketPath(recovery.guardian.socketPath)}
230
+ : {
231
+ socketPath: usableSocketPath(`${this.statePath}.guardian.sock`),
232
+ token: crypto.randomBytes(32).toString("hex")
233
+ }
231
234
  this.guardianIdentity = guardianIdentity
232
235
  this.guardian = new GuardianClient(guardianIdentity)
233
236
  if (recovery) {
@@ -503,15 +506,36 @@ export default class RollbridgeDaemon {
503
506
  /**
504
507
  * Prepares from private guardian state, starts listeners, then asks the committed owner
505
508
  * to atomically fence itself and transfer authority.
509
+ * @returns {Promise<void>} Resolves after the replacement is committed.
506
510
  */
507
511
  async replaceIncompatibleOwner() {
512
+ const maxAttempts = 3
513
+ let lastError
514
+
515
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
516
+ try {
517
+ return await this.replaceIncompatibleOwnerAttempt()
518
+ } catch (error) {
519
+ lastError = error
520
+ const diagnostic = error instanceof Error ? error.message : String(error)
521
+
522
+ if (!/^Guardian owner state changed after prepare \(prepared revision \d+, current revision \d+\); abort the stale candidate before preparing again$/.test(diagnostic) || attempt === maxAttempts) {
523
+ throw error
524
+ }
525
+ this.logger("owner replacement candidate became stale; re-preparing", {attempt, maxAttempts, error: diagnostic})
526
+ }
527
+ }
528
+ throw lastError
529
+ }
530
+
531
+ async replaceIncompatibleOwnerAttempt() {
508
532
  if (!this.statePath || !this.config.ownerRecovery) throw new Error("Atomic owner replacement requires ownerRecovery and the same statePath transaction anchor")
509
533
  this.controlCommandsReady = false
510
534
  const state = await readState(this.statePath)
511
535
  const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
512
536
 
513
537
  if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
514
- this.guardianIdentity = persisted.recovery.guardian
538
+ this.guardianIdentity = {...persisted.recovery.guardian, socketPath: usableSocketPath(persisted.recovery.guardian.socketPath)}
515
539
  this.guardian = new GuardianClient(this.guardianIdentity)
516
540
  await this.guardian.connect()
517
541
  this.watchOwnerReplacementEvents()
@@ -519,9 +543,16 @@ export default class RollbridgeDaemon {
519
543
  configDigest: persisted.recovery.configDigest,
520
544
  runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
521
545
  }
522
- const replacementProtocol = await this.guardian.ownerReplacementProtocol()
546
+ const guardianVersion = await this.retainedGuardianVersion()
547
+ const versionDrift = Boolean(this.runtime?.version && guardianVersion && guardianVersion !== this.runtime.version)
548
+ const replacementProtocol = versionDrift ? "legacy" : await this.guardian.ownerReplacementProtocol()
523
549
  const legacyBridge = replacementProtocol === "legacy"
524
- ? await this.prepareLegacyOwnerReplacement({persisted, persistedAuthority})
550
+ ? await this.prepareLegacyOwnerReplacement({
551
+ guardianVersion,
552
+ persisted,
553
+ persistedAuthority,
554
+ protocol: versionDrift ? "guardian-version-drift" : "legacy-protocol"
555
+ })
525
556
  : undefined
526
557
  const prepared = legacyBridge?.prepared ?? await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
527
558
  let preparedStatus
@@ -767,6 +798,33 @@ export default class RollbridgeDaemon {
767
798
  }
768
799
  }
769
800
 
801
+ /**
802
+ * Reads the retained guardian's package version without deciding the replacement protocol.
803
+ *
804
+ * The version is read from the guardian's own executable rather than its capabilities
805
+ * command: a pre-replacement guardian only answers that command once a replacement
806
+ * transaction is prepared, so asking here would move the probe ahead of the transaction
807
+ * every capability-gated guardian depends on. A guardian whose version cannot be read
808
+ * stays on the protocol classification path.
809
+ * @returns {Promise<string | undefined>} Retained guardian package version.
810
+ */
811
+ async retainedGuardianVersion() {
812
+ if (!this.guardian) throw new Error("Guardian version reconciliation requires the durable process guardian")
813
+ return await guardianVersionFromProcess(this.guardian.pid)
814
+ }
815
+
816
+ /**
817
+ * Names the last-resort refusal for a guardian this daemon cannot hand off automatically.
818
+ * @param {string | undefined} guardianVersion - Retained guardian package version.
819
+ * @param {string} cause - Measured reason the automatic handoff cannot run.
820
+ * @returns {Error} Refusal naming both versions and the exact repair command.
821
+ */
822
+ guardianVersionHandoffRefusal(guardianVersion, cause) {
823
+ const configArgument = this.configPath ? ` --config ${this.configPath}` : ""
824
+
825
+ return new Error(`The persistent Rollbridge guardian is version ${guardianVersion ?? "unknown"} while this daemon runs rollbridge ${this.runtime?.version ?? "unknown"}; ${cause}. Run "rollbridge shutdown${configArgument}" and then "rollbridge ensure-daemon${configArgument}" to replace the supervisor with this Rollbridge installation.`)
826
+ }
827
+
770
828
  /** Requires the persistent guardian protocol which can restart an accepted daemon. */
771
829
  async assertGuardianDaemonRecoveryCapability() {
772
830
  if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
@@ -775,10 +833,18 @@ export default class RollbridgeDaemon {
775
833
  try {
776
834
  capabilities = await this.guardian.capabilities()
777
835
  } catch (error) {
778
- if (!(error instanceof Error) || !["Guardian capabilities requires a process key", "Unknown guardian command: capabilities"].includes(error.message)) throw error
836
+ if (!(error instanceof Error) || !isGuardianDaemonRecoveryDiagnostic(error.message)) throw error
779
837
  throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
780
838
  }
781
839
  if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
840
+ const clientVersion = this.runtime?.version
841
+ const guardianVersion = capabilities.guardianVersion
842
+
843
+ if (!clientVersion) return
844
+ if (!guardianVersion) throw this.guardianVersionHandoffRefusal(guardianVersion, "its version cannot be attested, so no handoff can be authorised")
845
+ if (guardianVersion !== clientVersion) {
846
+ throw this.guardianVersionHandoffRefusal(guardianVersion, "this startup path has no incumbent daemon to hand guardian ownership from")
847
+ }
782
848
  }
783
849
 
784
850
  /**
@@ -899,13 +965,15 @@ export default class RollbridgeDaemon {
899
965
  }
900
966
 
901
967
  /**
902
- * Authenticates and prepares the one-time disruptive bridge for an exact legacy guardian protocol.
903
- * @param {{persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
968
+ * Authenticates and prepares the disruptive guardian-upgrade bridge, for a guardian that predates
969
+ * the replacement protocol or whose package version drifted from this daemon's runtime.
970
+ * @param {{guardianVersion?: string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}, protocol: GuardianHandoffProtocol}} options - Bridge evidence and why the bridge is required.
904
971
  * @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
905
972
  */
906
- async prepareLegacyOwnerReplacement({persisted, persistedAuthority}) {
973
+ async prepareLegacyOwnerReplacement({guardianVersion, persisted, persistedAuthority, protocol}) {
907
974
  const legacyProcessKey = ownerSnapshotProcessKeys(persisted, persisted.singletonReleaseIds)[0]
908
975
  const legacyGuardian = this.guardian
976
+ const versionHandoff = protocol === "guardian-version-drift"
909
977
  let legacyPrepared
910
978
 
911
979
  if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state")
@@ -938,12 +1006,23 @@ export default class RollbridgeDaemon {
938
1006
 
939
1007
  try {
940
1008
  if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
1009
+ if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "the handoff keeps the incumbent config identity unchanged and this deploy changes config authority")
941
1010
  throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
942
1011
  }
943
- if (!this.legacyIncumbentPid) throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
1012
+ if (!this.legacyIncumbentPid) {
1013
+ if (versionHandoff) throw this.guardianVersionHandoffRefusal(guardianVersion, "ensure-daemon reported no incumbent daemon to hand guardian ownership from")
1014
+ throw new Error("The authenticated legacy guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
1015
+ }
944
1016
  if (!this.guardianIdentity?.pid) throw new Error("The authenticated legacy guardian state is missing its exact guardian PID")
945
1017
  await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
946
- const incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
1018
+ let incumbentStartTime
1019
+
1020
+ try {
1021
+ incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
1022
+ } catch (error) {
1023
+ if (!versionHandoff) throw error
1024
+ throw this.guardianVersionHandoffRefusal(guardianVersion, `the incumbent daemon PID ${this.legacyIncumbentPid} is not the attested owner: ${error instanceof Error ? error.message : String(error)}`)
1025
+ }
947
1026
  if (legacyPrepared) assertLegacyPrivateOwnerState(legacyPrepared.ownerState, persisted, persistedAuthority)
948
1027
  try {
949
1028
  incumbentControl = await openControlSession(persisted.control.path)
@@ -970,7 +1049,9 @@ export default class RollbridgeDaemon {
970
1049
 
971
1050
  assertLegacyGuardianInventoryMembership(legacyInventory, /** @type {PrivateOwnerState} */ (ownerState))
972
1051
  const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
973
- socketPath: `${this.statePath}.split3-guardian.sock`,
1052
+ socketPath: usableSocketPath(versionHandoff
1053
+ ? `${this.statePath}.guardian-${this.runtime?.version}.sock`
1054
+ : `${this.statePath}.split3-guardian.sock`),
974
1055
  token: crypto.randomBytes(32).toString("hex")
975
1056
  })
976
1057
 
@@ -985,12 +1066,15 @@ export default class RollbridgeDaemon {
985
1066
  recovery: {...persisted.recovery, guardian: upgradedIdentity}
986
1067
  })
987
1068
 
988
- this.logger("legacy owner replacement bridge prepared", {
1069
+ this.logger("guardian upgrade bridge prepared", {
989
1070
  guardianPid: upgradedIdentity.pid ?? null,
1071
+ guardianVersion: guardianVersion ?? "unknown",
990
1072
  incumbentPid: this.legacyIncumbentPid,
991
- replacementId: prepared.replacementId
1073
+ protocol,
1074
+ replacementId: prepared.replacementId,
1075
+ runtimeVersion: this.runtime?.version ?? "unknown"
992
1076
  })
993
- return {boundaryCrossed: false, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, recoverySnapshot}
1077
+ return {boundaryCrossed: false, guardianVersion, incumbentControl, incumbentStartTime, legacyGuardian, legacyInventory, legacyPrepared, legacySnapshot: persisted, prepared, protocol, recoverySnapshot}
994
1078
  } catch (upgradeError) {
995
1079
  incumbentControl?.close()
996
1080
  let cleanupError
@@ -1070,10 +1154,20 @@ export default class RollbridgeDaemon {
1070
1154
 
1071
1155
  if (staged.committed) throw new Error("Legacy guardian committed replacement before the authenticated disruptive boundary")
1072
1156
  }
1073
- this.logger("legacy owner replacement disruptive boundary", {
1157
+ const boundaryReason = bridge.protocol === "guardian-version-drift"
1158
+ ? `retained guardian ${bridge.guardianVersion ?? "unknown"} predates this daemon runtime ${this.runtime?.version ?? "unknown"}`
1159
+ : "retained guardian and daemon lacked atomic replacement protocol"
1160
+
1161
+ this.logger("guardian upgrade bridge crossed", {
1074
1162
  disruptive: true,
1163
+ guardianVersion: bridge.guardianVersion ?? "unknown",
1075
1164
  incumbentPid: this.legacyIncumbentPid,
1076
- reason: "retained guardian and daemon lacked atomic replacement protocol"
1165
+ protocol: bridge.protocol,
1166
+ reason: boundaryReason,
1167
+ // The retained guardian keeps its own children; the boundary retires only the incumbent daemon.
1168
+ retainedGuardianPid: bridge.legacyGuardian.pid ?? null,
1169
+ retainedGuardianProcesses: bridge.legacyInventory.length,
1170
+ runtimeVersion: this.runtime?.version ?? "unknown"
1077
1171
  })
1078
1172
  await this.guardian.beginLegacyOwnerClaim(
1079
1173
  bridge.prepared.replacementId,
@@ -1089,8 +1183,8 @@ export default class RollbridgeDaemon {
1089
1183
  await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
1090
1184
  this.ownerTransition = /** @type {OwnerTransition} */ ({
1091
1185
  disruptive: true,
1092
- mode: "legacy-first-upgrade",
1093
- reason: "retained guardian and daemon lacked atomic replacement protocol"
1186
+ mode: bridge.protocol === "guardian-version-drift" ? "guardian-version-handoff" : "legacy-first-upgrade",
1187
+ reason: boundaryReason
1094
1188
  })
1095
1189
  }
1096
1190
 
@@ -1406,7 +1500,18 @@ export default class RollbridgeDaemon {
1406
1500
  }
1407
1501
 
1408
1502
  if (commandName === "status") {
1409
- return this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
1503
+ const status = this.status({includeLogs: statusIncludeLogs(data.includeLogs)})
1504
+ let guardian
1505
+
1506
+ try {
1507
+ const diagnostics = await this.guardian?.guardianStatus()
1508
+
1509
+ guardian = diagnostics ? {guardianPid: diagnostics.guardianPid, ownerMutation: diagnostics.ownerMutation, version: diagnostics.version} : {guardianPid: null, ownerMutation: null, version: null}
1510
+ } catch (error) {
1511
+ guardian = {error: error instanceof Error ? error.message : String(error), guardianPid: this.guardianIdentity?.pid ?? null, ownerMutation: null, version: null}
1512
+ }
1513
+
1514
+ return {...status, guardian}
1410
1515
  }
1411
1516
 
1412
1517
  if (commandName === "events") {
@@ -1824,8 +1929,10 @@ export default class RollbridgeDaemon {
1824
1929
  let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
1825
1930
 
1826
1931
  if (transition.phase === "candidate_ready") {
1827
- if (previousRelease && !transition.degradedIncumbent) await this.updateGenerationTransition("retiring_previous")
1828
- else await this.updateGenerationTransition("previous_retired")
1932
+ // The candidate has already started and passed its health gates. Keep the
1933
+ // incumbent authoritative until the candidate is activated and traffic is
1934
+ // switched; retirement is deliberately launched from committed_pending.
1935
+ await this.updateGenerationTransition("activating_candidate")
1829
1936
  }
1830
1937
 
1831
1938
  if (transition.phase === "retiring_previous") {
@@ -1899,22 +2006,20 @@ export default class RollbridgeDaemon {
1899
2006
 
1900
2007
  if (transition.phase === "committed_pending") {
1901
2008
  await this.refreshServiceDefinitions(release)
1902
- if (!activationLifecycle && previousRelease?.state === "active") {
1903
- try {
1904
- const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
1905
-
1906
- await previousRelease.beginRetirement(retirementConfig)
1907
- } catch (error) {
1908
- retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
1909
- this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
1910
- }
2009
+ if (previousRelease?.state === "active") {
2010
+ const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
2011
+
2012
+ // Traffic is already on the healthy candidate. Retirement must not hold
2013
+ // the deploy mutation or its owner fence; its own bounded drain reports
2014
+ // failures and escalates any process that misses the retirement budget.
2015
+ void previousRelease.beginRetirement(retirementConfig)
2016
+ .catch((error) => {
2017
+ retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
2018
+ this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
2019
+ })
2020
+ .then(() => this.drainAndPrune(previousRelease, retirementConfig))
1911
2021
  }
1912
2022
  await this.checkpointGenerationTransition()
1913
- if (previousRelease) {
1914
- const retirementConfig = previousRelease.config
1915
-
1916
- void this.drainAndPrune(previousRelease, retirementConfig)
1917
- }
1918
2023
  try {
1919
2024
  await this.replaceSingletons(release)
1920
2025
  } catch (error) {
@@ -1956,6 +2061,8 @@ export default class RollbridgeDaemon {
1956
2061
  if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
1957
2062
  if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
1958
2063
  if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
2064
+ /** @type {Promise<void> | undefined} */
2065
+ let candidateDrain
1959
2066
 
1960
2067
  if (transition.phase === "activating_candidate") {
1961
2068
  if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
@@ -1987,6 +2094,15 @@ export default class RollbridgeDaemon {
1987
2094
  transition.compensationError = undefined
1988
2095
  await this.updateGenerationTransition("restoring_previous")
1989
2096
  this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
2097
+
2098
+ // The candidate's bounded drain starts here, before the restoration attempt. The only other
2099
+ // compensation drain runs after the success path, so when incumbent restoration throws the
2100
+ // candidate is left `draining` with a live process: measured 2026-11-17, candidate
2101
+ // `20261117063703` still held `web` pid 3201928 at 07:30Z after its compensation threw at
2102
+ // 06:42:31Z, and reached `release drained` only when a later compensation reached the
2103
+ // success path at 08:47:00Z. A retained generation holding live work is what refuses every
2104
+ // later deploy, and a failed candidate is never a restoration target.
2105
+ candidateDrain = this.drainAndPrune(candidate, candidate.config)
1990
2106
  }
1991
2107
 
1992
2108
  if (transition.phase === "restoring_previous") {
@@ -1998,10 +2114,12 @@ export default class RollbridgeDaemon {
1998
2114
 
1999
2115
  transition.compensationError = failure
2000
2116
  await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
2117
+ await previous.stopProxyTraffic()
2001
2118
  this.logger("release generation compensation restoration failed", {
2002
2119
  activationError: transition.activationError,
2003
2120
  error: failure,
2004
- releaseId: previous.releaseId
2121
+ releaseId: previous.releaseId,
2122
+ traffic: "unavailable"
2005
2123
  })
2006
2124
  throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
2007
2125
  }
@@ -2040,7 +2158,7 @@ export default class RollbridgeDaemon {
2040
2158
  this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
2041
2159
  throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
2042
2160
  }
2043
- void this.drainAndPrune(candidate, candidate.config)
2161
+ if (candidateDrain === undefined) void this.drainAndPrune(candidate, candidate.config)
2044
2162
  this.logger("release generation compensation completed", result)
2045
2163
  return result
2046
2164
  }
@@ -3315,6 +3433,15 @@ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
3315
3433
  diagnostic === "Unknown guardian command: prepare-owner-replacement"
3316
3434
  }
3317
3435
 
3436
+ /**
3437
+ * Reports the capability diagnostics that identify a guardian predating daemon recovery.
3438
+ * @param {string} diagnostic - Guardian error message.
3439
+ * @returns {boolean} Whether the guardian predates the daemon recovery capability.
3440
+ */
3441
+ function isGuardianDaemonRecoveryDiagnostic(diagnostic) {
3442
+ return diagnostic === "Guardian capabilities requires a process key" || diagnostic === "Unknown guardian command: capabilities"
3443
+ }
3444
+
3318
3445
  /**
3319
3446
  * @param {DaemonStatus} snapshot - Serialized owner process snapshot.
3320
3447
  * @param {Record<string, string>} [singletonReleaseIds] - Exact singleton owner releases.
@@ -3662,7 +3789,7 @@ export function releasesToPrune(releases, policy, now) {
3662
3789
  }
3663
3790
 
3664
3791
  /**
3665
- * @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, proxy?: {host: string, port: number}}} ControlSocketInspection
3792
+ * @typedef {{alive: boolean, application?: string, activeReleaseId?: string | null, guardian?: {ownerMutation: Record<string, JsonValue> | null, version: string | null}, proxy?: {host: string, port: number}}} ControlSocketInspection
3666
3793
  */
3667
3794
 
3668
3795
  /**
@@ -3737,7 +3864,7 @@ export async function inspectControlSocket(socketPath, timeoutMs = 1000) {
3737
3864
  /**
3738
3865
  * Parses a control status response line into a Rollbridge identity, if it is one.
3739
3866
  * @param {string} line - JSON response line.
3740
- * @returns {{application: string, activeReleaseId: string | null, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
3867
+ * @returns {{application: string, activeReleaseId: string | null, guardian: {ownerMutation: Record<string, JsonValue> | null, version: string | null} | undefined, proxy: {host: string, port: number} | undefined} | undefined} Identity, or undefined when unrecognized.
3741
3868
  */
3742
3869
  function parseControlStatus(line) {
3743
3870
  /** @type {JsonValue} */
@@ -3754,5 +3881,8 @@ function parseControlStatus(line) {
3754
3881
 
3755
3882
  const proxy = "proxy" in parsed && parsed.proxy && typeof parsed.proxy === "object" && !Array.isArray(parsed.proxy) && typeof parsed.proxy.host === "string" && typeof parsed.proxy.port === "number" ? {host: parsed.proxy.host, port: parsed.proxy.port} : undefined
3756
3883
 
3757
- return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, proxy}
3884
+ const guardianValue = "guardian" in parsed && parsed.guardian && typeof parsed.guardian === "object" && !Array.isArray(parsed.guardian) ? parsed.guardian : undefined
3885
+ const guardian = guardianValue && typeof guardianValue.version === "string" || guardianValue && guardianValue.version === null ? {ownerMutation: guardianValue.ownerMutation && typeof guardianValue.ownerMutation === "object" && !Array.isArray(guardianValue.ownerMutation) ? guardianValue.ownerMutation : null, version: guardianValue.version} : undefined
3886
+
3887
+ return {activeReleaseId: typeof parsed.activeReleaseId === "string" ? parsed.activeReleaseId : null, application: parsed.application, guardian, proxy}
3758
3888
  }
package/src/doctor.js CHANGED
@@ -26,6 +26,13 @@ export async function runEnvironmentChecks(config) {
26
26
  const socketInspection = await inspectControlSocketSafely(config.control.path)
27
27
 
28
28
  checks.push(controlSocketCheck(config, socketInspection))
29
+ if (!("error" in socketInspection) && socketInspection.guardian) {
30
+ const {guardian} = socketInspection
31
+ const lease = guardian.ownerMutation
32
+ const leaseDetail = lease ? `, owner mutation ${String(lease.operation || "unnamed operation")} held by pid ${String(lease.pid || "unknown")}` : ", no active owner-mutation lease"
33
+
34
+ checks.push({detail: `guardian Rollbridge ${guardian.version || "unknown"}${leaseDetail}`, name: "guardian diagnostics", ok: guardian.version !== null})
35
+ }
29
36
  checks.push(await controlSocketDirectoryCheck(config))
30
37
  checks.push(await proxyPortCheck(config, socketInspection))
31
38
 
@@ -1,7 +1,7 @@
1
1
  // @ts-check
2
2
 
3
3
  /**
4
- * @typedef {"progressing" | "wedged" | "unproven"} DrainProgress
4
+ * @typedef {"progressing" | "timedOut" | "wedged" | "unproven"} DrainProgress
5
5
  * @typedef {{cpuMs: number | undefined, heldProcessGroupPgid: number | undefined, id: string, liveChildCount: number | undefined, pid: number | undefined, retirementTimedOut: boolean}} DrainProcessObservation
6
6
  * @typedef {{atMs: number, processes: DrainProcessObservation[]}} DrainLivenessSample
7
7
  * @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: DrainProgress}} DrainClassification
@@ -12,26 +12,40 @@
12
12
  */
13
13
 
14
14
  /**
15
- * Classifies one draining release group as still draining or wedged.
15
+ * Classifies one draining release group as still draining, past its drain budget, or wedged.
16
16
  *
17
17
  * A group is wedged only when its drain can no longer progress. Every inconclusive reading
18
18
  * — a live connection, a process still inside its retirement budget, an unmeasurable CPU
19
19
  * counter, or a sample window shorter than `minSampleWindowMs` — reports `progressing` or
20
20
  * `unproven`, never `wedged`: a legitimate multi-hour drain is valid and must not be
21
21
  * mislabelled as a wedge.
22
+ *
23
+ * A live connection inside `drainBudgetMs` is still a legitimate drain. Past that budget the
24
+ * connection wait has already been given up, so the group reports `timedOut` instead of
25
+ * `progressing` forever — the reading that lets one held websocket outlive the activation
26
+ * budget and stall every later deploy. `timedOut` names the deadline; it is not an escalation
27
+ * and never authorises a signal.
22
28
  * @param {object} args - Classification inputs.
23
29
  * @param {number} args.connectionCount - Live proxy connections this generation still owns.
24
30
  * @param {DrainLivenessSample} args.currentSample - Newest liveness observation.
31
+ * @param {number} args.drainBudgetMs - Configured drain budget (`proxy.drainTimeoutMs`); 0 records no budget.
32
+ * @param {number} args.drainElapsedMs - Time since this drain started, at classification time.
25
33
  * @param {number} args.minSampleWindowMs - Shortest window that can prove flat CPU.
26
34
  * @param {DrainLivenessSample | undefined} args.previousSample - Older observation, when one is in range.
27
35
  * @returns {DrainClassification} The classification and the evidence it rests on.
28
36
  */
29
- export function classifyDrainProgress({connectionCount, currentSample, minSampleWindowMs, previousSample}) {
37
+ export function classifyDrainProgress({connectionCount, currentSample, drainBudgetMs = 0, drainElapsedMs = 0, minSampleWindowMs, previousSample}) {
30
38
  const live = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
31
39
  const stillInsideBudget = live.filter((process) => !process.retirementTimedOut)
32
40
 
33
41
  if (connectionCount > 0) {
34
- return classify("progressing", `${connectionCount} live connection${connectionCount === 1 ? "" : "s"} still owned by the generation`, currentSample, previousSample)
42
+ const connections = `${connectionCount} live connection${connectionCount === 1 ? "" : "s"} still owned by the generation`
43
+
44
+ if (drainBudgetMs > 0 && drainElapsedMs > drainBudgetMs) {
45
+ return classify("timedOut", `drain budget ${drainBudgetMs}ms exceeded by ${drainElapsedMs - drainBudgetMs}ms with ${connections}`, currentSample, previousSample)
46
+ }
47
+
48
+ return classify("progressing", connections, currentSample, previousSample)
35
49
  }
36
50
  if (stillInsideBudget.length > 0) {
37
51
  return classify("progressing", `process ${stillInsideBudget[0].id} is still inside its retirement budget`, currentSample, previousSample)
@@ -2,17 +2,34 @@
2
2
 
3
3
  import crypto from "node:crypto"
4
4
  import {spawn} from "node:child_process"
5
+ import fs from "node:fs/promises"
5
6
  import net from "node:net"
7
+ import path from "node:path"
6
8
  import {fileURLToPath} from "node:url"
7
9
  import ManagedProcess from "./managed-process.js"
8
10
 
9
11
  const guardianPath = fileURLToPath(new URL("./process-guardian.js", import.meta.url))
12
+ const MAX_UNIX_SOCKET_PATH_BYTES = 107
13
+
14
+ /**
15
+ * Linux reserves one byte of sockaddr_un.sun_path for its terminator. Keep
16
+ * long task/workspace prefixes out of the pathname while retaining a stable,
17
+ * collision-resistant name for the configured path.
18
+ * @param {string} socketPath - Configured Unix socket pathname.
19
+ * @returns {string} Path that can be passed to net.createConnection/listen.
20
+ */
21
+ export function usableSocketPath(socketPath) {
22
+ if (Buffer.byteLength(socketPath) <= MAX_UNIX_SOCKET_PATH_BYTES) return socketPath
23
+
24
+ const digest = crypto.createHash("sha256").update(socketPath).digest("hex").slice(0, 32)
25
+ return path.join("/tmp", `rollbridge-${digest}.sock`)
26
+ }
10
27
 
11
28
  export default class GuardianClient {
12
29
  /** @param {{pid?: number, socketPath: string, token: string}} identity - Durable guardian identity. */
13
30
  constructor({pid, socketPath, token}) {
14
31
  this.pid = pid
15
- this.socketPath = socketPath
32
+ this.socketPath = usableSocketPath(socketPath)
16
33
  this.token = token
17
34
  this.socket = /** @type {net.Socket | undefined} */ (undefined)
18
35
  this.buffer = ""
@@ -150,14 +167,17 @@ export default class GuardianClient {
150
167
  return await response
151
168
  }
152
169
 
153
- /** Stops the guardian after every owned process has stopped. */
154
- async shutdown() {
170
+ /**
171
+ * Stops the guardian after every owned process has stopped or reached its retirement budget.
172
+ * @param {{retirementTimeoutMs?: number}} [options] - Optional non-blocking retirement budget.
173
+ */
174
+ async shutdown(options = {}) {
155
175
  if (this.pending.size > 0) {
156
176
  await new Promise((resolve) => {
157
177
  this.idleWaiters.push(() => { resolve(undefined) })
158
178
  })
159
179
  }
160
- await this.request({command: "shutdown"})
180
+ await this.request({command: "shutdown", options})
161
181
  const socket = this.socket
162
182
 
163
183
  if (!socket || socket.destroyed) throw new Error("Process guardian disconnected before shutdown acknowledgement")
@@ -186,9 +206,11 @@ export default class GuardianClient {
186
206
  await this.request({command: "owner-ready", ownerPid: process.pid})
187
207
  }
188
208
 
189
- /** @returns {Promise<{daemonRecovery: number, generationReactivation?: number}>} Guardian protocol capabilities. */
209
+ /** @returns {Promise<{daemonRecovery: number, generationReactivation?: number, guardianVersion?: string}>} Guardian protocol capabilities. */
190
210
  async capabilities() {
191
- const capabilities = /** @type {{daemonRecovery: number, generationReactivation?: number}} */ (await this.request({command: "capabilities"}))
211
+ const capabilities = /** @type {{daemonRecovery: number, generationReactivation?: number, guardianVersion?: string}} */ (await this.request({command: "capabilities"}))
212
+
213
+ if (!capabilities.guardianVersion) capabilities.guardianVersion = await guardianVersionFromProcess(this.pid)
192
214
 
193
215
  this.generationReactivation = capabilities.generationReactivation === 1
194
216
  return capabilities
@@ -347,6 +369,11 @@ export default class GuardianClient {
347
369
  await this.request({command: "report-owner-mutation-wait", mutationId, waitingFor})
348
370
  }
349
371
 
372
+ /** @returns {Promise<{guardianPid: number, ownerMutation: {elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null, version: string}>} Guardian diagnostics. */
373
+ async guardianStatus() {
374
+ return /** @type {{guardianPid: number, ownerMutation: {elapsedMs: number, leaseMs: number, mutationId: string, operation: string, pid: number | null, startedAt: string, waitingFor: string} | null, version: string}} */ (await this.request({command: "guardian-status"}))
375
+ }
376
+
350
377
  /** @returns {Promise<{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}>} Transaction status. */
351
378
  async replacementStatus() {
352
379
  return /** @type {{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}} */ (await this.request({command: "replacement-status"}))
@@ -450,6 +477,27 @@ export default class GuardianClient {
450
477
  }
451
478
  }
452
479
 
480
+ /**
481
+ * Reads the package version from a guardian's own executable path. Used when its
482
+ * capabilities response predates guardian version reporting, and by owner
483
+ * replacement, which must not send a capabilities command before a replacement
484
+ * transaction is prepared. An unreadable or non-Rollbridge process remains unknown.
485
+ * @param {number | undefined} pid - Existing guardian PID.
486
+ * @returns {Promise<string | undefined>} Guardian package version.
487
+ */
488
+ export async function guardianVersionFromProcess(pid) {
489
+ if (!pid) return undefined
490
+
491
+ try {
492
+ const commandLine = (await fs.readFile(`/proc/${pid}/cmdline`, "utf8")).split("\0").find((argument) => argument.endsWith("/src/process-guardian.js"))
493
+ if (!commandLine) return undefined
494
+ const metadata = JSON.parse(await fs.readFile(path.resolve(path.dirname(commandLine), "..", "package.json"), "utf8"))
495
+ return metadata && typeof metadata.version === "string" ? metadata.version : undefined
496
+ } catch {
497
+ return undefined
498
+ }
499
+ }
500
+
453
501
  class GuardianProcess extends ManagedProcess {
454
502
  /** @param {{client: GuardianClient, definition: ConstructorParameters<typeof ManagedProcess>[0], key: string}} args - Remote process args. */
455
503
  constructor({client, definition, key}) {