rollbridge 0.1.38 → 0.1.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.js CHANGED
@@ -4,6 +4,7 @@ import fs from "node:fs/promises"
4
4
  import http from "node:http"
5
5
  import net from "node:net"
6
6
  import crypto from "node:crypto"
7
+ import path from "node:path"
7
8
  import {isDeepStrictEqual} from "node:util"
8
9
  import httpProxy from "http-proxy"
9
10
  import {loadConfig} from "./config.js"
@@ -23,12 +24,12 @@ const STATE_PERSIST_INTERVAL_MS = 5000
23
24
  * @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
24
25
  * @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
25
26
  * @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
26
- * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed"} GenerationTransitionPhase
27
- * @typedef {{candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, configDigest: string, error?: string, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
28
- * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
27
+ * @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
28
+ * @typedef {{activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, configDigest: string, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
29
+ * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
29
30
  * @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
30
- * @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
31
- * @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
31
+ * @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
32
+ * @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
32
33
  * @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
33
34
  */
34
35
 
@@ -40,12 +41,14 @@ export default class RollbridgeDaemon {
40
41
  * @param {string} [args.configPath] - Config file path to reload before deploys.
41
42
  * @param {(message: string, data?: Record<string, JsonValue>) => void} [args.logger] - Logger.
42
43
  * @param {number} [args.legacyIncumbentPid] - Exact incumbent PID supplied by ensure-daemon.
44
+ * @param {{args: string[], cwd: string, env: Record<string, string>, executable: string, logPath?: string, pidPath?: string, startupTimeoutMs: number}} [args.recoveryCommand] - Exact command for guardian-owned daemon recovery.
43
45
  * @param {import("./daemon-runtime.js").DaemonRuntimeIdentity} [args.runtime] - Immutable daemon runtime identity.
44
46
  */
45
- constructor({bootstrap, config, configPath, legacyIncumbentPid, logger, runtime}) {
47
+ constructor({bootstrap, config, configPath, legacyIncumbentPid, logger, recoveryCommand, runtime}) {
46
48
  this.bootstrap = bootstrap ? {...bootstrap} : undefined
47
49
  this.config = config
48
50
  this.configPath = configPath
51
+ this.recoveryCommand = recoveryCommand
49
52
  this.runtime = runtime
50
53
  this.legacyIncumbentPid = legacyIncumbentPid
51
54
  this.eventLog = new EventLog(EVENT_HISTORY_LIMIT)
@@ -62,6 +65,7 @@ export default class RollbridgeDaemon {
62
65
 
63
66
  this.releases = /** @type {Map<string, ReleaseGroup>} */ (new Map())
64
67
  this.services = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
68
+ this.serviceReleaseIds = /** @type {Map<string, string>} */ (new Map())
65
69
  this.servicePorts = /** @type {Record<string, number>} */ ({})
66
70
  this.portReservations = /** @type {Set<number>} */ (new Set())
67
71
  this.singletons = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
@@ -72,6 +76,7 @@ export default class RollbridgeDaemon {
72
76
  this.proxyServer = /** @type {http.Server | undefined} */ (undefined)
73
77
  this.controlServer = /** @type {net.Server | undefined} */ (undefined)
74
78
  this.controlSocketOwned = false
79
+ this.boundControlIdentity = /** @type {{dev: number, ino: number} | undefined} */ (undefined)
75
80
  this.boundControlPath = /** @type {string | undefined} */ (undefined)
76
81
  this.controlSockets = /** @type {Set<net.Socket>} */ (new Set())
77
82
  this.proxyPort = /** @type {number | undefined} */ (undefined)
@@ -83,17 +88,26 @@ export default class RollbridgeDaemon {
83
88
  this.stateCleanupEnabled = false
84
89
  this.shutdownPromise = /** @type {Promise<void> | undefined} */ (undefined)
85
90
  this.retirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
91
+ this.committedRetirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
86
92
  this.ownerRetired = false
87
93
  this.controlClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
88
94
  this.proxyClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
89
- this.listenerHandoff = /** @type {{control: boolean, proxy: boolean, replacementId: string} | undefined} */ (undefined)
95
+ this.listenerHandoff = /** @type {{completionSocket: net.Socket, control: boolean, proxy: boolean, replacementId: string, retired: Promise<Record<string, JsonValue>>} | undefined} */ (undefined)
90
96
  this.listenerHandoffFailure = /** @type {Error | undefined} */ (undefined)
91
97
  this.incumbentListenerControl = /** @type {Awaited<ReturnType<typeof openControlSession>> | undefined} */ (undefined)
98
+ this.incumbentListenerSourceIds = /** @type {WeakMap<object, string>} */ (new WeakMap())
99
+ this.listenerConnectionSources = /** @type {Map<string, Map<string, {http: number, websocket: number}>>} */ (new Map())
100
+ this.listenerSourceId = crypto.randomUUID()
101
+ this.retiredConnectionPublication = Promise.resolve()
102
+ this.retiredReplacementId = /** @type {string | undefined} */ (undefined)
103
+ this.retiredListenerHandoff = /** @type {{releases: ReleaseGroup[], replacementId: string, replacementRetired: boolean, retired: Promise<Record<string, JsonValue>>} | undefined} */ (undefined)
92
104
  this.ownerTransition = /** @type {OwnerTransition | undefined} */ (undefined)
93
105
  this.controlCommandsReady = true
94
106
  this.startingReleases = /** @type {Set<ReleaseGroup>} */ (new Set())
95
107
  this.guardian = /** @type {GuardianClient | undefined} */ (undefined)
96
108
  this.guardianIdentity = /** @type {{pid?: number, socketPath: string, token: string} | undefined} */ (undefined)
109
+ this.ownerReady = false
110
+ this.generationTransitionRecovery = false
97
111
  // Still-alive managed processes left by a previous daemon (from statePath), captured at
98
112
  // startup and surfaced in status(). The daemon cannot re-manage them, only report them.
99
113
  this.orphans = /** @type {{id: string, pid: number, releaseId: string | null}[]} */ ([])
@@ -109,8 +123,67 @@ export default class RollbridgeDaemon {
109
123
  async start({exposeControl = true, reportOrphans = true} = {}) {
110
124
  if (this.config.ownerRecovery) await this.initializeOwnerRecovery()
111
125
  else if (reportOrphans) await this.reportOrphans()
126
+ if (this.ownerRetired) return
127
+ const transition = this.generationTransition
128
+ const recoveredReplacementProxy = this.guardian ? await this.finalizeRecoveredOwnerReplacement() : false
129
+
130
+ if (this.guardian && transition && transition.phase !== "committed" && !transition.error) {
131
+ this.generationTransitionRecovery = true
132
+ try {
133
+ await this.executeOwnerMutation("recover generation transition", async () => {
134
+ if (!recoveredReplacementProxy) await this.startProxy()
135
+ if (this.ownerRetired) return {}
136
+ if (exposeControl) await this.exposeControl()
137
+ if (this.ownerRetired) return {}
138
+ await this.markOwnerReady()
139
+ try {
140
+ await this.resumeGenerationTransition()
141
+ } catch (error) {
142
+ this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
143
+ }
144
+ return {}
145
+ })
146
+ } finally {
147
+ this.generationTransitionRecovery = false
148
+ }
149
+ if (!this.ownerRetired) this.logger("release generation transition recovery settled", {phase: transition.phase, releaseId: transition.candidateReleaseId})
150
+ return
151
+ }
152
+ if (!recoveredReplacementProxy) await this.startProxy()
153
+ if (!this.ownerRetired && exposeControl) await this.exposeControl()
154
+ }
155
+
156
+ /** @returns {Promise<boolean>} Whether startup bound a proxy for a pending committed replacement. */
157
+ async finalizeRecoveredOwnerReplacement() {
158
+ if (!this.guardian) return false
159
+ let status = await this.guardian.replacementStatus()
160
+
161
+ if (status.retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
162
+ if (!status.retirementPending) return false
163
+ if (!status.committedReplacementId) throw new Error("Guardian reports listener retirement without a committed replacement")
164
+ if (!status.retirementReady) {
165
+ const settled = Promise.race([
166
+ this.guardian.waitForEvent("replacement-listeners-retired").then(() => undefined, (error) => error instanceof Error ? error : new Error(String(error))),
167
+ this.guardian.waitForEvent("replacement-retirement-failed").then((event) => new Error(stringOrUndefined(event.reason) || "Retired listener state transfer failed"), (error) => error instanceof Error ? error : new Error(String(error))),
168
+ this.guardian.waitForEvent("replacement-committed").then(() => undefined, (error) => error instanceof Error ? error : new Error(String(error)))
169
+ ])
170
+
171
+ status = await this.guardian.replacementStatus()
172
+ if (status.retirementPending && !status.retirementReady) {
173
+ const settlementError = await settled
174
+
175
+ if (settlementError) throw settlementError
176
+ status = await this.guardian.replacementStatus()
177
+ }
178
+ if (!status.retirementPending) return false
179
+ if (!status.retirementReady) throw new Error("Guardian listener retirement did not publish complete state")
180
+ }
181
+ const replacementId = status.committedReplacementId
182
+
183
+ if (!replacementId) throw new Error("Guardian listener retirement lost its committed replacement")
112
184
  await this.startProxy()
113
- if (exposeControl) await this.exposeControl()
185
+ await this.guardian.finalizeOwnerReplacement(replacementId)
186
+ return true
114
187
  }
115
188
 
116
189
  /** Connects to the durable process guardian and reconstructs a matching persisted owner snapshot. */
@@ -119,13 +192,8 @@ export default class RollbridgeDaemon {
119
192
  const state = await readState(this.statePath)
120
193
  const snapshot = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
121
194
  const recovery = snapshot?.recovery
122
- const configDigest = this.ownerRecoveryConfigDigest()
123
195
 
124
196
  if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
125
- if (recovery && recovery.configDigest !== configDigest) throw new Error("Owner recovery config identity does not match the persisted owner; refusing cross-authority adoption.")
126
- if (snapshot && ((this.runtime?.digest ?? null) !== (snapshot.daemonRuntime?.digest ?? null))) {
127
- throw new Error("Owner recovery runtime identity does not match the persisted owner; use the exact same Rollbridge runtime.")
128
- }
129
197
 
130
198
  const guardianIdentity = recovery?.guardian || {
131
199
  socketPath: `${this.statePath}.guardian.sock`,
@@ -133,49 +201,73 @@ export default class RollbridgeDaemon {
133
201
  }
134
202
  this.guardianIdentity = guardianIdentity
135
203
  this.guardian = new GuardianClient(guardianIdentity)
136
- if (recovery) await this.guardian.connect()
204
+ if (recovery) {
205
+ await this.guardian.connect()
206
+ await this.assertGuardianDaemonRecoveryCapability()
207
+ }
137
208
  else {
138
209
  await this.guardian.launch()
139
210
  guardianIdentity.pid = this.guardian.pid
140
211
  }
141
- await this.guardian.claimOwner(this.config.ownerRecovery?.reconnectGraceMs ?? 30000, this.ownerAuthority())
142
212
  this.watchOwnerReplacementEvents()
213
+ await this.guardian.claimOwner(this.config.ownerRecovery?.reconnectGraceMs ?? 30000, this.ownerAuthority())
143
214
 
144
215
  if (snapshot) {
145
216
  if (!Array.isArray(snapshot.releases) || (snapshot.activeReleaseId !== null && typeof snapshot.activeReleaseId !== "string")) {
146
217
  throw new Error("Owner recovery state is partial or corrupt; active release metadata is required.")
147
218
  }
148
- let recoveryConfig = this.config
149
- let releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({})
150
- let singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({})
151
- let recoverySnapshot = snapshot
219
+ const ownerState = /** @type {PrivateOwnerState} */ (await this.guardian.ownerState())
152
220
 
153
- if ((recovery?.format ?? 0) >= 2) {
154
- const ownerState = /** @type {PrivateOwnerState} */ (await this.guardian.ownerState())
221
+ if (this.ownerRetired) return
222
+ if (!ownerState?.config || !ownerState.snapshot) throw new Error("Committed guardian published incomplete owner recovery state")
223
+ let recoverySnapshot = ownerState.snapshot
155
224
 
156
- if (!ownerState?.config || !ownerState.releaseConfigs || !ownerState.snapshot) {
157
- throw new Error("Durable guardian state is missing exact release definitions; refusing partial owner recovery.")
158
- }
159
- recoveryConfig = ownerState.config
160
- releaseConfigs = ownerState.releaseConfigs
161
- singletonReleaseIds = ownerState.singletonReleaseIds || singletonReleaseIds
162
- const guardianTransitionAt = Date.parse(ownerState.snapshot.generationTransition?.updatedAt || "")
163
- const persistedTransitionAt = Date.parse(snapshot.generationTransition?.updatedAt || "")
164
-
165
- if (Number.isFinite(guardianTransitionAt) && (!Number.isFinite(persistedTransitionAt) || guardianTransitionAt > persistedTransitionAt)) {
166
- recoverySnapshot = ownerState.snapshot
225
+ if ((recovery?.format ?? 0) >= 2) {
226
+ const releaseConfigs = ownerState.releaseConfigs
227
+
228
+ if (!releaseConfigs) throw new Error("Durable guardian state is missing exact release definitions; refusing partial owner recovery.")
229
+ const guardianTransition = ownerState.snapshot.generationTransition
230
+ const persistedTransition = snapshot.generationTransition
231
+ const sameTransition = Boolean(guardianTransition && persistedTransition &&
232
+ guardianTransition.candidateReleaseId === persistedTransition.candidateReleaseId &&
233
+ guardianTransition.candidateReleasePath === persistedTransition.candidateReleasePath &&
234
+ guardianTransition.candidateRevision === persistedTransition.candidateRevision &&
235
+ guardianTransition.configDigest === persistedTransition.configDigest &&
236
+ guardianTransition.previousReleaseId === persistedTransition.previousReleaseId &&
237
+ guardianTransition.startedAt === persistedTransition.startedAt)
238
+ const requiredReleaseIds = new Set([
239
+ ...ownerState.snapshot.releaseReferences.map((reference) => reference.releaseId),
240
+ ...snapshot.releaseReferences.map((reference) => reference.releaseId),
241
+ ...Object.values(ownerState.serviceReleaseIds || {}),
242
+ ...Object.values(snapshot.serviceReleaseIds || {}),
243
+ ...Object.values(ownerState.singletonReleaseIds || {}),
244
+ ...Object.values(snapshot.singletonReleaseIds || {}),
245
+ ...(ownerState.snapshot.activeReleaseId ? [ownerState.snapshot.activeReleaseId] : []),
246
+ ...(snapshot.activeReleaseId ? [snapshot.activeReleaseId] : []),
247
+ ...(guardianTransition ? [guardianTransition.candidateReleaseId, ...(guardianTransition.previousReleaseId ? [guardianTransition.previousReleaseId] : [])] : []),
248
+ ...(persistedTransition ? [persistedTransition.candidateReleaseId, ...(persistedTransition.previousReleaseId ? [persistedTransition.previousReleaseId] : [])] : [])
249
+ ])
250
+ const publicReleasesComplete = [...requiredReleaseIds].every((releaseId) => snapshot.releases.some((release) => release.releaseId === releaseId) && releaseConfigs[releaseId])
251
+
252
+ if (sameTransition && publicReleasesComplete &&
253
+ typeof persistedTransition?.journalRevision === "number" &&
254
+ typeof guardianTransition?.journalRevision === "number" &&
255
+ persistedTransition.journalRevision > guardianTransition.journalRevision) {
256
+ recoverySnapshot = snapshot
167
257
  }
168
258
  }
169
- await this.restoreOwnerState(recoverySnapshot, {config: recoveryConfig, releaseConfigs, resumeDrains: false, singletonReleaseIds})
259
+ await this.restoreOwnerState(recoverySnapshot, {
260
+ config: ownerState.config,
261
+ listenerConnectionSources: ownerState.listenerConnectionSources,
262
+ releaseConfigs: ownerState.releaseConfigs,
263
+ resumeDrains: false,
264
+ serviceReleaseIds: recoverySnapshot === snapshot ? snapshot.serviceReleaseIds || ownerState.serviceReleaseIds : ownerState.serviceReleaseIds || snapshot.serviceReleaseIds,
265
+ singletonReleaseIds: recoverySnapshot === snapshot ? snapshot.singletonReleaseIds || ownerState.singletonReleaseIds : ownerState.singletonReleaseIds || snapshot.singletonReleaseIds
266
+ })
267
+ if (this.ownerRetired) return
170
268
  this.persistenceEnabled = true
171
269
  await this.guardian.reconcileInventory()
172
- if (this.generationTransition && this.generationTransition.phase !== "committed" && !this.generationTransition.error) {
173
- try {
174
- await this.resumeGenerationTransition()
175
- } catch (error) {
176
- this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: this.generationTransition.candidateReleaseId})
177
- }
178
- }
270
+ if (this.ownerRetired) return
179
271
  for (const release of this.releases.values()) {
180
272
  if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
181
273
  }
@@ -184,6 +276,7 @@ export default class RollbridgeDaemon {
184
276
  this.persistenceEnabled = true
185
277
  await this.persistState({throwOnError: true})
186
278
  }
279
+ if (this.ownerRetired) return
187
280
  this.stateCleanupEnabled = true
188
281
  await this.publishOwnerState()
189
282
  }
@@ -196,16 +289,46 @@ export default class RollbridgeDaemon {
196
289
  /** Installs event-driven drain fencing for the next prepared owner transaction. */
197
290
  watchOwnerReplacementEvents() {
198
291
  if (!this.guardian) throw new Error("Owner replacement event fencing requires the durable guardian")
292
+ this.guardian.onEvent("owner-connection-state", (event) => this.handleIncumbentListenerEvent(event))
199
293
  this.guardian.onEvent("replacement-prepared", () => {
200
294
  for (const release of this.releases.values()) release.pauseDrainForOwnerHandoff()
201
295
  })
202
296
  this.guardian.onEvent("replacement-aborted", () => {
297
+ if (this.retiredListenerHandoff) {
298
+ void this.resumeControlLessOwnerListeners().catch((error) => {
299
+ this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
300
+ this.logger("control-less owner listener handoff recovery failed", {error: this.listenerHandoffFailure.message})
301
+ })
302
+ return
303
+ }
203
304
  if (this.listenerHandoff || this.ownerRetired) return
204
305
  for (const release of this.releases.values()) {
205
306
  release.resumeDrainAfterOwnerHandoff()
206
307
  if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
207
308
  }
208
309
  })
310
+ this.guardian.onEvent("replacement-listener-handoff-requested", (event) => {
311
+ if (this.ownerRetired) return
312
+ const replacementId = stringOrUndefined(event.replacementId)
313
+
314
+ if (!replacementId) throw new Error("Owner listener handoff request is missing its replacement id")
315
+ void this.yieldControlLessOwnerListeners(replacementId).catch((error) => {
316
+ this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
317
+ this.logger("control-less owner listener handoff failed", {error: this.listenerHandoffFailure.message, replacementId})
318
+ this.guardian?.disconnect()
319
+ })
320
+ })
321
+ this.guardian.onEvent("replacement-retirement-requested", (event) => {
322
+ if (this.ownerRetired) return
323
+ const replacementId = stringOrUndefined(event.replacementId)
324
+
325
+ if (!replacementId) throw new Error("Committed owner retirement request is missing its replacement id")
326
+ void this.completeControlLessOwnerRetirement(replacementId).catch((error) => {
327
+ this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
328
+ this.logger("committed owner retirement failed", {error: this.listenerHandoffFailure.message, replacementId})
329
+ this.guardian?.disconnect()
330
+ })
331
+ })
209
332
  }
210
333
 
211
334
  /** @returns {{configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}} Exact authority fence. */
@@ -217,26 +340,34 @@ export default class RollbridgeDaemon {
217
340
  * @param {OwnerRecoverySnapshot} snapshot - Validated persisted owner state.
218
341
  * @param {object} [options] - Recovery definition options.
219
342
  * @param {import("./config.js").RollbridgeConfig} [options.config] - Owner config for daemon-wide processes.
343
+ * @param {Record<string, Record<string, {http: number, websocket: number}>>} [options.listenerConnectionSources] - Exact inherited listener sources.
220
344
  * @param {Record<string, import("./config.js").RollbridgeConfig>} [options.releaseConfigs] - Exact generation configs.
221
345
  * @param {boolean} [options.resumeDrains] - Whether to resume draining generations immediately.
346
+ * @param {Record<string, string>} [options.serviceReleaseIds] - Exact release owner for each persistent service definition.
222
347
  * @param {Record<string, string>} [options.singletonReleaseIds] - Exact release owner for each singleton registration.
223
348
  * @param {boolean} [options.synchronizeLifecycleRoles] - Whether the caller owns guardian role mutation authority.
224
349
  */
225
- async restoreOwnerState(snapshot, {config = this.config, releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({}), resumeDrains = true, singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({}), synchronizeLifecycleRoles = true} = {}) {
350
+ async restoreOwnerState(snapshot, {config = this.config, listenerConnectionSources = /** @type {Record<string, Record<string, {http: number, websocket: number}>>} */ ({}), releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({}), resumeDrains = true, serviceReleaseIds = snapshot.serviceReleaseIds || /** @type {Record<string, string>} */ ({}), singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({}), synchronizeLifecycleRoles = true} = {}) {
226
351
  if (!Array.isArray(snapshot.releases) || (snapshot.activeReleaseId !== null && typeof snapshot.activeReleaseId !== "string")) {
227
352
  throw new Error("Owner recovery state is partial or corrupt; active release metadata is required.")
228
353
  }
229
354
  this.generationTransition = snapshot.generationTransition ? {...snapshot.generationTransition} : undefined
230
355
  if (snapshot.activeReleaseId === null && snapshot.releases.length === 0) return
231
- this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
356
+ if (!this.bootstrap) this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
232
357
  this.ownerTransition = snapshot.ownerTransition ? {...snapshot.ownerTransition} : undefined
358
+ const fallbackDefinitionReleaseId = snapshot.activeReleaseId || snapshot.releases.at(-1)?.releaseId
359
+ const serviceOwnerReleaseIds = new Set(Object.values(serviceReleaseIds))
233
360
  const singletonOwnerReleaseIds = new Set(Object.values(singletonReleaseIds))
361
+ const transitionReleaseIds = this.generationTransitionReleaseIds()
362
+
363
+ if (snapshot.services.some((service) => !serviceReleaseIds[service.id]) && fallbackDefinitionReleaseId) serviceOwnerReleaseIds.add(fallbackDefinitionReleaseId)
234
364
 
235
365
  for (const releaseStatus of snapshot.releases) {
236
- const transitionCandidate = this.generationTransition?.candidateReleaseId === releaseStatus.releaseId && this.generationTransition.phase !== "committed"
366
+ const transitionOwner = transitionReleaseIds.has(releaseStatus.releaseId)
367
+ const serviceOwner = serviceOwnerReleaseIds.has(releaseStatus.releaseId)
237
368
  const singletonOwner = singletonOwnerReleaseIds.has(releaseStatus.releaseId)
238
369
 
239
- if (releaseStatus.state !== "active" && releaseStatus.state !== "draining" && !transitionCandidate && !singletonOwner) continue
370
+ if (releaseStatus.state !== "active" && releaseStatus.state !== "draining" && !transitionOwner && !serviceOwner && !singletonOwner) continue
240
371
  const release = new ReleaseGroup({
241
372
  config: releaseConfigs[releaseStatus.releaseId] || config,
242
373
  logger: this.logger,
@@ -253,21 +384,34 @@ export default class RollbridgeDaemon {
253
384
  this.releases.set(release.releaseId, release)
254
385
  if (release.releaseId === snapshot.activeReleaseId) this.activeRelease = release
255
386
  }
387
+ for (const [sourceId, releases] of Object.entries(listenerConnectionSources)) {
388
+ for (const [releaseId, connections] of Object.entries(releases)) {
389
+ this.setListenerConnectionSource(sourceId, releaseId, connections)
390
+ }
391
+ }
256
392
 
257
393
  if (snapshot.activeReleaseId !== null && !this.activeRelease) throw new Error(`Owner recovery state does not contain active release ${snapshot.activeReleaseId}.`)
258
- const definitionRelease = this.activeRelease || [...this.releases.values()].at(-1)
394
+ const committedBootstrapRelease = this.committedBootstrapRelease()
395
+ const definitionRelease = this.activeRelease || committedBootstrapRelease || [...this.releases.values()].at(-1)
259
396
  if (!definitionRelease) throw new Error("Owner recovery state has no release definition for owned processes.")
260
- if (!this.activeRelease && snapshot.singletons.length > 0) throw new Error("Owner recovery state has release-owned singletons without an active release identity.")
397
+ if (!this.activeRelease && !committedBootstrapRelease && snapshot.singletons.some((singleton) => !singletonReleaseIds[singleton.id])) {
398
+ throw new Error("Owner recovery state has release-owned singletons without an exact release identity.")
399
+ }
261
400
  for (const serviceStatus of snapshot.services) {
262
- const processConfig = config.processes.find((candidate) => candidate.id === serviceStatus.id && candidate.policy === "service" && candidate.deployStrategy !== "handoff")
401
+ const serviceReleaseId = serviceReleaseIds[serviceStatus.id] || fallbackDefinitionReleaseId
402
+ const serviceRelease = serviceReleaseId ? this.releases.get(serviceReleaseId) : undefined
403
+
404
+ if (!serviceRelease) throw new Error(`Owner recovery state contains service ${serviceStatus.id} for an unknown release.`)
405
+ const processConfig = serviceRelease.config.processes.find((candidate) => candidate.id === serviceStatus.id && candidate.policy === "service" && candidate.deployStrategy !== "handoff")
263
406
 
264
- if (!processConfig) throw new Error(`Owner recovery state contains unknown service ${serviceStatus.id}.`)
265
- const service = definitionRelease.buildProcess(processConfig, {guardianKey: `service:${serviceStatus.id}`, shouldRestart: () => !this.stopping})
407
+ if (!processConfig) throw new Error(`Owner recovery state contains unknown service ${serviceStatus.id} for release ${serviceReleaseId}.`)
408
+ const service = serviceRelease.buildProcess(processConfig, {guardianKey: `service:${serviceStatus.id}`, shouldRestart: () => !this.stopping})
266
409
 
267
410
  await this.recoverGuardianProcess(service)
268
411
  this.services.set(serviceStatus.id, service)
269
- if (definitionRelease.ports[serviceStatus.id] !== undefined) {
270
- const port = definitionRelease.ports[serviceStatus.id]
412
+ this.serviceReleaseIds.set(serviceStatus.id, serviceRelease.releaseId)
413
+ if (serviceRelease.ports[serviceStatus.id] !== undefined) {
414
+ const port = serviceRelease.ports[serviceStatus.id]
271
415
 
272
416
  if (this.portReservations.has(port)) throw new Error(`Persisted daemon service ${serviceStatus.id} port ${port} is already reserved by a live generation`)
273
417
  this.portReservations.add(port)
@@ -297,13 +441,34 @@ export default class RollbridgeDaemon {
297
441
  /** Publishes full normalized definitions only over the authenticated guardian channel. */
298
442
  async publishOwnerState() {
299
443
  if (!this.guardian) return
300
- await this.guardian.publishOwnerState({
444
+ await this.guardian.publishOwnerState(this.transferableOwnerState())
445
+ }
446
+
447
+ /** Confirms guardian-owned startup only after every requested listener is available. */
448
+ async markOwnerReady() {
449
+ if (!this.guardian) return
450
+ await this.guardian.ownerReady()
451
+ this.ownerReady = true
452
+ }
453
+
454
+ /** @returns {Record<string, JsonValue>} Complete private guardian owner state. */
455
+ transferableOwnerState() {
456
+ return {
301
457
  authority: this.ownerAuthority(),
302
458
  config: this.config,
459
+ recovery: this.recoveryCommand ? {
460
+ command: this.recoveryCommand,
461
+ reconnectGraceMs: this.config.ownerRecovery?.reconnectGraceMs ?? 30000,
462
+ startupTimeoutMs: this.recoveryCommand.startupTimeoutMs
463
+ } : undefined,
464
+ listenerConnectionSources: this.serializedListenerConnectionSources(),
465
+ listenerSourceId: this.listenerSourceId,
303
466
  releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
467
+ retiredListenerHandoff: 2,
468
+ serviceReleaseIds: Object.fromEntries(this.serviceReleaseIds),
304
469
  singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds),
305
470
  snapshot: this.status()
306
- })
471
+ }
307
472
  }
308
473
 
309
474
  /**
@@ -338,12 +503,17 @@ export default class RollbridgeDaemon {
338
503
  preparedStatus = await this.guardian.replacementStatus()
339
504
  transfer = /** @type {PrivateOwnerState} */ (prepared.ownerState)
340
505
  if (!transfer?.config || !transfer.snapshot) throw new Error("Committed owner published incomplete replacement state")
506
+ const unresolvedTransition = transfer.snapshot.generationTransition
507
+
508
+ if (unresolvedTransition && unresolvedTransition.phase !== "committed" && unresolvedTransition.configDigest !== this.ownerRecoveryConfigDigest()) {
509
+ throw new Error(`Owner replacement cannot change config authority while unresolved generation transition ${unresolvedTransition.candidateReleaseId} remains at ${unresolvedTransition.phase}`)
510
+ }
341
511
  const registeredProcesses = new Map((await this.guardian.inventory()).map(({key, provenance}) => [key, provenance]))
342
512
 
343
513
  reservedProcessKey = legacyBridge ? undefined : reconstructableOwnerSnapshotProcessKeys(transfer.snapshot, transfer.singletonReleaseIds)
344
514
  .find((key) => registeredProcesses.has(key))
345
515
  if (reservedProcessKey) this.guardian.reserveProcessRecovery(reservedProcessKey, /** @type {string} */ (registeredProcesses.get(reservedProcessKey)))
346
- await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, releaseConfigs: transfer.releaseConfigs, resumeDrains: false, singletonReleaseIds: transfer.singletonReleaseIds, synchronizeLifecycleRoles: false})
516
+ await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, listenerConnectionSources: transfer.listenerConnectionSources, releaseConfigs: transfer.releaseConfigs, resumeDrains: false, serviceReleaseIds: transfer.serviceReleaseIds, singletonReleaseIds: transfer.singletonReleaseIds, synchronizeLifecycleRoles: false})
347
517
  } catch (error) {
348
518
  legacyBridge?.incumbentControl?.close()
349
519
  if (legacyBridge) {
@@ -357,6 +527,16 @@ export default class RollbridgeDaemon {
357
527
  }
358
528
  throw error
359
529
  }
530
+ if (!legacyBridge) {
531
+ try {
532
+ await this.assertGuardianDaemonRecoveryCapability()
533
+ } catch (error) {
534
+ // The immediately preceding guardian protocol notifies the incumbent of
535
+ // an abort only when the prepared candidate disconnects.
536
+ this.guardian.disconnect()
537
+ throw error
538
+ }
539
+ }
360
540
  for (const release of this.releases.values()) release.preserveConfigOnRetirement = true
361
541
  this.logger("owner replacement candidate prepared", {activeReleaseId: this.activeRelease?.releaseId ?? null, replacementId: prepared.replacementId})
362
542
 
@@ -411,40 +591,61 @@ export default class RollbridgeDaemon {
411
591
  listenersYielded = true
412
592
  }
413
593
  }
414
- if (sharedFixedProxy) await this.startProxy()
594
+ if (sharedFixedProxy && !retiredIncumbentControl) await this.startProxy()
415
595
  if (!retiredIncumbentControl) {
416
596
  await fs.rename(stagingControlPath, this.config.control.path)
417
597
  finalControlPublished = true
418
598
  this.boundControlPath = this.config.control.path
419
599
  }
420
- committed = this.guardian.waitForEvent("replacement-committed").then(
421
- () => undefined,
600
+ const retirementFailure = this.guardian.waitForEvent("replacement-retirement-failed").then(
601
+ (event) => new Error(stringOrUndefined(event.reason) || "Retired listener state transfer failed"),
422
602
  (error) => error instanceof Error ? error : new Error(String(error))
423
603
  )
424
- const staged = await this.guardian.stageOwnerReplacement(prepared.replacementId, {
425
- authority: this.ownerAuthority(),
426
- config: this.config,
427
- releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
428
- singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds),
429
- snapshot: this.status()
430
- })
604
+ committed = Promise.race([
605
+ this.guardian.waitForEvent("replacement-committed").then(
606
+ () => undefined,
607
+ (error) => error instanceof Error ? error : new Error(String(error))
608
+ ),
609
+ retirementFailure
610
+ ])
611
+ const listenerRetirement = retiredIncumbentControl
612
+ ? Promise.race([
613
+ this.guardian.waitForEvent("replacement-listeners-retired").then(() => undefined, (error) => error instanceof Error ? error : new Error(String(error))),
614
+ retirementFailure
615
+ ])
616
+ : undefined
617
+ const staged = await this.guardian.stageOwnerReplacement(prepared.replacementId, this.transferableOwnerState())
431
618
 
432
619
  if (staged.committed && reservedProcessKey) {
433
620
  committedAuthority = true
434
621
  await this.guardian.recoverReservedProcess(reservedProcessKey)
435
622
  }
623
+ if (staged.committed && retiredIncumbentControl && sharedFixedProxy) await this.startProxy()
436
624
 
437
625
  if (!staged.committed) {
438
626
  if (retiredIncumbentControl) {
439
627
  const processKey = reservedProcessKey
440
628
 
441
629
  if (!processKey || !this.guardian.processes.has(processKey)) throw new Error("Retired owner replacement requires an exact reserved process from committed owner state")
630
+ if (transfer.retiredListenerHandoff !== 2) {
631
+ throw new Error("Cannot safely complete atomic owner replacement through the older retained guardian while the incumbent control socket is absent; incumbent owner and connections were preserved")
632
+ }
442
633
  try {
634
+ await this.guardian.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
635
+ const listenerRetirementError = await listenerRetirement
636
+
637
+ if (listenerRetirementError) throw listenerRetirementError
638
+ if (sharedFixedProxy) await this.startProxy()
443
639
  await this.guardian.commitRetiredOwnerReplacement(prepared.replacementId, processKey)
444
640
  committedAuthority = true
445
641
  await this.guardian.recoverReservedProcess(processKey)
642
+ await this.guardian.finalizeOwnerReplacement(prepared.replacementId)
446
643
  } catch (error) {
447
- if (!(error instanceof Error) || error.message !== "Guardian commit-retired-owner-replacement requires the committed owner") throw error
644
+ if (!(error instanceof Error) || ![
645
+ "Guardian commit-retired-owner-replacement requires the committed owner",
646
+ "Guardian prepare-retired-owner-listener-handoff requires the committed owner",
647
+ "Unknown guardian command: prepare-retired-owner-listener-handoff"
648
+ ].includes(error.message)) throw error
448
649
  throw new Error(
449
650
  "Cannot safely complete atomic owner replacement through the older retained guardian while the incumbent control socket is absent; incumbent owner and connections were preserved",
450
651
  {cause: error}
@@ -454,10 +655,12 @@ export default class RollbridgeDaemon {
454
655
  try {
455
656
  if (!incumbentControl) throw new Error("Owner replacement incumbent control session is unavailable")
456
657
  await incumbentControl.request({command: "commit-owner-replacement", replacementId: prepared.replacementId})
658
+ committedAuthority = true
457
659
  } catch (error) {
458
660
  const status = await this.guardian.replacementStatus()
459
661
 
460
662
  if (status.committedReplacementId !== prepared.replacementId || !status.ownerClaimed) throw error
663
+ committedAuthority = true
461
664
  this.logger("owner replacement commit response lost; guardian commit confirmed", {replacementId: prepared.replacementId})
462
665
  }
463
666
  }
@@ -534,33 +737,109 @@ export default class RollbridgeDaemon {
534
737
  }
535
738
  }
536
739
 
740
+ /** Requires the persistent guardian protocol which can restart an accepted daemon. */
741
+ async assertGuardianDaemonRecoveryCapability() {
742
+ if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
743
+ let capabilities
744
+
745
+ try {
746
+ capabilities = await this.guardian.capabilities()
747
+ } catch (error) {
748
+ if (!(error instanceof Error) || !["Guardian capabilities requires a process key", "Unknown guardian command: capabilities"].includes(error.message)) throw error
749
+ throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
750
+ }
751
+ if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
752
+ }
753
+
537
754
  /**
538
755
  * Applies authenticated live-connection state from the retired listener owner.
539
756
  * @param {Record<string, JsonValue>} event - Incumbent control event.
540
- * @param {{close: () => void}} session - Exact incumbent session.
757
+ * @param {{close: () => void}} [session] - Exact incumbent control session.
541
758
  */
542
759
  handleIncumbentListenerEvent(event, session) {
543
760
  if (event.event !== "owner-connection-state") return
544
761
  const releaseId = stringOrUndefined(event.releaseId)
762
+ let sourceId = stringOrUndefined(event.sourceId)
545
763
  const connections = event.connections
546
764
 
547
- if (!releaseId || !connections || typeof connections !== "object" || Array.isArray(connections)) {
765
+ if (!sourceId && session) {
766
+ sourceId = this.incumbentListenerSourceIds.get(session)
767
+ if (!sourceId) {
768
+ sourceId = crypto.randomUUID()
769
+ this.incumbentListenerSourceIds.set(session, sourceId)
770
+ }
771
+ }
772
+ if (!sourceId || !releaseId || !connections || typeof connections !== "object" || Array.isArray(connections)) {
548
773
  throw new Error("Incumbent listener sent invalid connection state")
549
774
  }
550
- const transferredConnections = {
775
+ const normalized = {
551
776
  http: requiredNonNegativeInteger(connections.http, "connections.http"),
552
777
  websocket: requiredNonNegativeInteger(connections.websocket, "connections.websocket")
553
778
  }
554
- const release = this.releases.get(releaseId)
555
779
 
556
- if (!release && (transferredConnections.http > 0 || transferredConnections.websocket > 0)) throw new Error(`Incumbent listener reported unknown release ${releaseId}`)
557
- if (release) release.setTransferredConnections(transferredConnections)
558
- if (this.incumbentListenerControl === session && ![...this.releases.values()].some((candidate) => candidate.hasTransferredConnections())) {
780
+ this.setListenerConnectionSource(sourceId, releaseId, normalized)
781
+ const handoffSocket = this.listenerHandoff?.completionSocket
782
+
783
+ if (handoffSocket && !handoffSocket.destroyed) {
784
+ handoffSocket.write(`${JSON.stringify({connections: normalized, event: "owner-connection-state", releaseId, sourceId})}\n`)
785
+ }
786
+ if (this.retiredReplacementId) {
787
+ void this.publishRetiredConnectionState(this.retiredReplacementId, sourceId, releaseId, normalized).catch((error) => {
788
+ this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
789
+ this.logger("retired owner connection-state relay failed", {error: this.listenerHandoffFailure.message, releaseId, replacementId: this.retiredReplacementId, sourceId})
790
+ this.guardian?.disconnect()
791
+ })
792
+ }
793
+ if (session && this.incumbentListenerControl === session && ![...this.releases.values()].some((candidate) => candidate.hasTransferredConnections())) {
559
794
  this.incumbentListenerControl = undefined
560
795
  session.close()
561
796
  }
562
797
  }
563
798
 
799
+ /**
800
+ * Reconciles one physical listener source and applies its aggregate to the release.
801
+ * @param {string} sourceId - Stable listener source.
802
+ * @param {string} releaseId - Retained release.
803
+ * @param {{http: number, websocket: number}} connections - Exact source counts.
804
+ */
805
+ setListenerConnectionSource(sourceId, releaseId, connections) {
806
+ const release = this.releases.get(releaseId)
807
+
808
+ if (!release && (connections.http > 0 || connections.websocket > 0)) throw new Error(`Incumbent listener reported unknown release ${releaseId}`)
809
+ const sourceReleases = this.listenerConnectionSources.get(sourceId) || new Map()
810
+
811
+ if (connections.http + connections.websocket === 0) sourceReleases.delete(releaseId)
812
+ else sourceReleases.set(releaseId, connections)
813
+ if (sourceReleases.size === 0) this.listenerConnectionSources.delete(sourceId)
814
+ else this.listenerConnectionSources.set(sourceId, sourceReleases)
815
+ if (!release) return
816
+ let http = 0
817
+ let websocket = 0
818
+
819
+ for (const candidateReleases of this.listenerConnectionSources.values()) {
820
+ const counts = candidateReleases.get(releaseId)
821
+
822
+ if (!counts) continue
823
+ http += counts.http
824
+ websocket += counts.websocket
825
+ }
826
+ release.setTransferredConnections({http, websocket})
827
+ }
828
+
829
+ /** @returns {Record<string, Record<string, {http: number, websocket: number}>>} Exact live listener sources. */
830
+ serializedListenerConnectionSources() {
831
+ const sources = new Map([...this.listenerConnectionSources].map(([sourceId, releases]) => [sourceId, new Map(releases)]))
832
+ const local = new Map()
833
+
834
+ for (const release of this.releases.values()) {
835
+ const connections = release.localConnections()
836
+
837
+ if (connections.http + connections.websocket > 0) local.set(release.releaseId, connections)
838
+ }
839
+ if (local.size > 0) sources.set(this.listenerSourceId, local)
840
+ return Object.fromEntries([...sources].map(([sourceId, releases]) => [sourceId, Object.fromEntries(releases)]))
841
+ }
842
+
564
843
  /**
565
844
  * Authenticates and prepares the one-time disruptive bridge for an exact legacy guardian protocol.
566
845
  * @param {{persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
@@ -618,12 +897,14 @@ export default class RollbridgeDaemon {
618
897
 
619
898
  assertLegacyIncumbentStatus(incumbentStatus, persisted)
620
899
  }
900
+ const definitionReleaseId = persisted.activeReleaseId || persisted.releases.at(-1)?.releaseId
621
901
  const ownerState = legacyPrepared
622
902
  ? legacyPrepared.ownerState
623
903
  : {
624
904
  authority: persistedAuthority,
625
905
  config: this.config,
626
906
  releaseConfigs: Object.fromEntries(persisted.releases.map((release) => [release.releaseId, this.config])),
907
+ serviceReleaseIds: definitionReleaseId ? Object.fromEntries(persisted.services.map((service) => [service.id, definitionReleaseId])) : {},
627
908
  singletonReleaseIds: Object.fromEntries(persisted.singletons.map((singleton) => [singleton.id, persisted.activeReleaseId]).filter((entry) => entry[1] !== null)),
628
909
  snapshot: persisted
629
910
  }
@@ -714,6 +995,7 @@ export default class RollbridgeDaemon {
714
995
  authority: this.ownerAuthority(),
715
996
  config: this.config,
716
997
  releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
998
+ serviceReleaseIds: Object.fromEntries(this.serviceReleaseIds),
717
999
  singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds),
718
1000
  snapshot: this.status()
719
1001
  }
@@ -735,14 +1017,18 @@ export default class RollbridgeDaemon {
735
1017
  incumbentPid: this.legacyIncumbentPid,
736
1018
  reason: "retained guardian and daemon lacked atomic replacement protocol"
737
1019
  })
738
- await this.guardian.beginLegacyOwnerClaim(bridge.prepared.replacementId, bridge.recoverySnapshot.recovery.reconnectGraceMs)
1020
+ await this.guardian.beginLegacyOwnerClaim(
1021
+ bridge.prepared.replacementId,
1022
+ bridge.recoverySnapshot.recovery.reconnectGraceMs,
1023
+ this.statePath,
1024
+ bridge.recoverySnapshot
1025
+ )
739
1026
  process.kill(this.legacyIncumbentPid, "SIGKILL")
740
1027
  bridge.boundaryCrossed = true
741
1028
  await bridge.incumbentControl?.closed()
742
1029
  if (legacyCommitted) await legacyCommitted
743
1030
  bridge.legacyGuardian.disconnect()
744
1031
  await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
745
- await writeState(this.statePath, bridge.recoverySnapshot)
746
1032
  this.ownerTransition = /** @type {OwnerTransition} */ ({
747
1033
  disruptive: true,
748
1034
  mode: "legacy-first-upgrade",
@@ -774,10 +1060,16 @@ export default class RollbridgeDaemon {
774
1060
 
775
1061
  /** @returns {Promise<void>} Exposes control commands and begins periodic state persistence. */
776
1062
  async exposeControl() {
1063
+ if (this.ownerRetired) return
777
1064
  if (this.stopping) throw new Error("Rollbridge is shutting down")
778
1065
 
779
1066
  await this.startControlServer()
780
1067
 
1068
+ if (this.ownerRetired) {
1069
+ await this.closeServer(this.controlServer)
1070
+ await this.removeControlSocket()
1071
+ return
1072
+ }
781
1073
  if (this.stopping) {
782
1074
  await this.closeServer(this.controlServer)
783
1075
  await fs.rm(this.config.control.path, {force: true})
@@ -791,6 +1083,7 @@ export default class RollbridgeDaemon {
791
1083
  * @returns {Promise<void>} Starts the stable local proxy.
792
1084
  */
793
1085
  async startProxy() {
1086
+ if (this.ownerRetired) return
794
1087
  const server = http.createServer((request, response) => this.proxyHttp(request, response))
795
1088
 
796
1089
  server.on("upgrade", (request, socket, head) => this.proxyWebSocket(request, socket, head))
@@ -805,6 +1098,7 @@ export default class RollbridgeDaemon {
805
1098
  resolve(undefined)
806
1099
  })
807
1100
  })
1101
+ if (this.ownerRetired) this.proxyClosePromise = this.closeServer(server)
808
1102
  }
809
1103
 
810
1104
  /**
@@ -813,10 +1107,12 @@ export default class RollbridgeDaemon {
813
1107
  * @returns {Promise<void>} Starts the control socket.
814
1108
  */
815
1109
  async startControlServer(socketPath = this.config.control.path, applyMetadata = true) {
1110
+ if (this.ownerRetired) return
816
1111
  const server = net.createServer((socket) => this.handleControlSocket(socket))
817
1112
 
818
1113
  this.controlServer = server
819
1114
  await this.prepareControlSocketPath(socketPath)
1115
+ if (this.ownerRetired) return
820
1116
 
821
1117
  await new Promise((resolve, reject) => {
822
1118
  server.once("error", reject)
@@ -827,7 +1123,14 @@ export default class RollbridgeDaemon {
827
1123
  resolve(undefined)
828
1124
  })
829
1125
  })
1126
+ const socketIdentity = await fs.lstat(socketPath)
830
1127
 
1128
+ this.boundControlIdentity = {dev: socketIdentity.dev, ino: socketIdentity.ino}
1129
+ if (this.ownerRetired) {
1130
+ await this.closeServer(server)
1131
+ await this.removeControlSocket()
1132
+ return
1133
+ }
831
1134
  if (applyMetadata) await this.applyControlSocketMetadata(socketPath)
832
1135
  }
833
1136
 
@@ -1086,9 +1389,12 @@ export default class RollbridgeDaemon {
1086
1389
  await this.guardian.commitOwnerReplacement(replacementId)
1087
1390
  committed = true
1088
1391
  await this.retireCommittedOwner(controlSocket)
1392
+ await this.beginRetiredListenerPublication(replacementId, this.listenerHandoff?.retired)
1089
1393
  await this.guardian.finalizeOwnerReplacement(replacementId)
1090
- } finally {
1394
+ await this.completeRetiredListenerHandoff(replacementId)
1395
+ } catch (error) {
1091
1396
  if (committed) this.guardian.disconnect()
1397
+ throw error
1092
1398
  }
1093
1399
  return {message: "owner replacement committed"}
1094
1400
  }
@@ -1164,10 +1470,12 @@ export default class RollbridgeDaemon {
1164
1470
  if (!this.guardian) throw new Error("Owner listener handoff requires the durable process guardian")
1165
1471
  if (this.listenerHandoff) throw new Error("Owner listeners are already yielded to a replacement candidate")
1166
1472
  await this.guardian.validateOwnerReplacement(replacementId)
1167
- this.listenerHandoff = {control, proxy, replacementId}
1473
+ if (!completionSocket) throw new Error("Owner listener handoff requires its authenticated control session")
1474
+ const retired = this.guardian.waitForEvent("replacement-retired")
1475
+
1476
+ this.listenerHandoff = {completionSocket, control, proxy, replacementId, retired}
1168
1477
  this.listenerHandoffFailure = undefined
1169
1478
 
1170
- if (!completionSocket) throw new Error("Owner listener handoff requires its authenticated control session")
1171
1479
  for (const release of this.releases.values()) release.pauseDrainForOwnerHandoff()
1172
1480
  if (proxy) this.proxyClosePromise = this.closeServer(this.proxyServer)
1173
1481
  if (control) {
@@ -1175,22 +1483,27 @@ export default class RollbridgeDaemon {
1175
1483
  for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
1176
1484
  await this.removeControlSocket()
1177
1485
  }
1486
+ /** @type {(sourceId: string, releaseId: string, connections: {http: number, websocket: number}) => void} */
1487
+ const publishConnections = (sourceId, releaseId, connections) => {
1488
+ if (completionSocket.destroyed) return
1489
+ completionSocket.write(`${JSON.stringify({connections, event: "owner-connection-state", releaseId, sourceId})}\n`)
1490
+ }
1491
+
1492
+ for (const [sourceId, releases] of this.listenerConnectionSources) {
1493
+ for (const [releaseId, connections] of releases) publishConnections(sourceId, releaseId, connections)
1494
+ }
1178
1495
  for (const release of this.releases.values()) {
1179
- const publishConnections = () => {
1180
- if (completionSocket.destroyed) return
1181
- completionSocket.write(`${JSON.stringify({
1182
- connections: release.status().connections,
1183
- event: "owner-connection-state",
1184
- releaseId: release.releaseId
1185
- })}\n`)
1186
- }
1496
+ const connections = release.localConnections()
1187
1497
 
1188
- publishConnections()
1189
- if (release.status().connectionCount > 0) release.once("drained", publishConnections)
1498
+ if (release.state === "active" || release.state === "draining" || connections.http + connections.websocket > 0) {
1499
+ publishConnections(this.listenerSourceId, release.releaseId, connections)
1500
+ }
1501
+ if (connections.http + connections.websocket > 0) {
1502
+ release.once("drained", () => publishConnections(this.listenerSourceId, release.releaseId, release.localConnections()))
1503
+ }
1190
1504
  }
1191
1505
 
1192
1506
  const aborted = this.guardian.waitForEvent("replacement-aborted").then(async () => await this.resumeYieldedListeners())
1193
- const retired = this.guardian.waitForEvent("replacement-retired")
1194
1507
 
1195
1508
  void Promise.race([aborted, retired]).catch((error) => {
1196
1509
  if (this.ownerRetired) return
@@ -1210,9 +1523,11 @@ export default class RollbridgeDaemon {
1210
1523
  }
1211
1524
  if (handoff.proxy) {
1212
1525
  await this.startProxy()
1526
+ if (this.ownerRetired) return
1213
1527
  }
1214
1528
  if (handoff.control) {
1215
1529
  await this.startControlServer()
1530
+ if (this.ownerRetired) return
1216
1531
  }
1217
1532
  this.listenerHandoff = undefined
1218
1533
  this.listenerHandoffFailure = undefined
@@ -1242,6 +1557,13 @@ export default class RollbridgeDaemon {
1242
1557
  this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
1243
1558
  return {activeReleaseId: newReleaseId, previousReleaseId: transition.previousReleaseId}
1244
1559
  }
1560
+ if (transition?.phase === "committed" && !this.activeRelease && this.bootstrap && this.releases.get(transition.candidateReleaseId)?.state === "draining") {
1561
+ this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
1562
+ this.assertCommittedBootstrapRecoveryReady()
1563
+ this.config = nextConfig
1564
+ await this.updateGenerationTransition("restoring_committed")
1565
+ return await this.resumeGenerationTransition()
1566
+ }
1245
1567
  const release = new ReleaseGroup({
1246
1568
  config: nextConfig,
1247
1569
  logger: this.logger,
@@ -1274,56 +1596,27 @@ export default class RollbridgeDaemon {
1274
1596
 
1275
1597
  const previousRelease = this.activeRelease
1276
1598
 
1277
- if (nextConfig.processes.some((processConfig) => processConfig.lifecycle.activateCommand !== undefined)) {
1278
- this.config = nextConfig
1279
- this.releases.set(release.releaseId, release)
1280
- const now = new Date().toISOString()
1281
-
1282
- this.generationTransition = /** @type {GenerationTransition} */ ({
1283
- candidateReleaseId: release.releaseId,
1284
- candidateReleasePath: release.releasePath,
1285
- candidateRevision: release.revision,
1286
- configDigest: ownerConfigDigest(nextConfig),
1287
- phase: "candidate_ready",
1288
- previousReleaseId: previousRelease?.releaseId ?? null,
1289
- startedAt: now,
1290
- updatedAt: now
1291
- })
1292
- await this.checkpointGenerationTransition()
1293
- return await this.resumeGenerationTransition()
1294
- }
1295
-
1296
1599
  this.config = nextConfig
1297
1600
  this.releases.set(release.releaseId, release)
1298
- release.activate()
1299
- this.activeRelease = release
1300
- this.logger("traffic switched", {previousReleaseId: previousRelease ? previousRelease.releaseId : null, releaseId: release.releaseId})
1301
-
1302
- this.refreshServiceDefinitions(release)
1303
- let retirementFailure
1304
-
1305
- if (previousRelease) {
1306
- try {
1307
- const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : nextConfig
1308
-
1309
- await previousRelease.beginRetirement(retirementConfig)
1310
- void this.drainAndPrune(previousRelease, retirementConfig)
1311
- } catch (error) {
1312
- retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
1313
- this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
1314
- }
1315
- }
1316
-
1317
- await this.replaceSingletons(release)
1318
-
1319
- await this.persistState()
1320
- await this.publishOwnerState()
1321
-
1322
- return {
1323
- activeReleaseId: release.releaseId,
1324
- previousReleaseId: previousRelease ? previousRelease.releaseId : null,
1325
- ...(retirementFailure && previousRelease ? {retirement: {error: retirementFailure, releaseId: previousRelease.releaseId, status: "quiescence_failed"}} : {})
1326
- }
1601
+ for (const serviceId of startedServices) this.serviceReleaseIds.set(serviceId, release.releaseId)
1602
+
1603
+ const activationLifecycle = nextConfig.processes.some((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
1604
+ const now = new Date().toISOString()
1605
+
1606
+ this.generationTransition = /** @type {GenerationTransition} */ ({
1607
+ activationLifecycle,
1608
+ candidateReleaseId: release.releaseId,
1609
+ candidateReleasePath: release.releasePath,
1610
+ candidateRevision: release.revision,
1611
+ configDigest: ownerConfigDigest(nextConfig),
1612
+ journalRevision: 0,
1613
+ phase: activationLifecycle ? "candidate_ready" : "activating_candidate",
1614
+ previousReleaseId: previousRelease?.releaseId ?? null,
1615
+ startedAt: now,
1616
+ updatedAt: now
1617
+ })
1618
+ await this.checkpointGenerationTransition()
1619
+ return await this.resumeGenerationTransition()
1327
1620
  }
1328
1621
 
1329
1622
  /**
@@ -1338,7 +1631,9 @@ export default class RollbridgeDaemon {
1338
1631
  const previousRelease = transition.previousReleaseId ? this.releases.get(transition.previousReleaseId) : undefined
1339
1632
 
1340
1633
  if (!release) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
1341
- if (transition.previousReleaseId && !previousRelease) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
1634
+ if (transition.previousReleaseId && !previousRelease && transition.phase !== "restoring_committed") throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
1635
+ const activationLifecycle = transition.activationLifecycle ?? release.config.processes.some((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
1636
+ let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
1342
1637
 
1343
1638
  if (transition.phase === "candidate_ready") {
1344
1639
  if (previousRelease) await this.updateGenerationTransition("retiring_previous")
@@ -1383,13 +1678,34 @@ export default class RollbridgeDaemon {
1383
1678
  this.activeRelease = release
1384
1679
  transition.phase = "committed_pending"
1385
1680
  transition.error = undefined
1681
+ transition.journalRevision = (transition.journalRevision ?? 0) + 1
1386
1682
  transition.updatedAt = new Date().toISOString()
1387
1683
  this.logger("traffic switched", {previousReleaseId: previousRelease?.releaseId ?? null, releaseId: release.releaseId})
1388
1684
  }
1389
1685
 
1686
+ if (transition.phase === "restoring_committed") {
1687
+ await this.resumeCommittedBootstrapGeneration(release)
1688
+ release.activate()
1689
+ this.activeRelease = release
1690
+ transition.phase = "committed_pending"
1691
+ transition.error = undefined
1692
+ transition.updatedAt = new Date().toISOString()
1693
+ this.logger("committed bootstrap generation restored", {releaseId: release.releaseId})
1694
+ }
1695
+
1390
1696
  if (transition.phase === "committed_pending") {
1697
+ await this.refreshServiceDefinitions(release)
1698
+ if (!activationLifecycle && previousRelease?.state === "active") {
1699
+ try {
1700
+ const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
1701
+
1702
+ await previousRelease.beginRetirement(retirementConfig)
1703
+ } catch (error) {
1704
+ retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
1705
+ this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
1706
+ }
1707
+ }
1391
1708
  await this.checkpointGenerationTransition()
1392
- this.refreshServiceDefinitions(release)
1393
1709
  if (previousRelease) {
1394
1710
  const retirementConfig = previousRelease.config
1395
1711
 
@@ -1403,11 +1719,17 @@ export default class RollbridgeDaemon {
1403
1719
  }
1404
1720
  transition.phase = "committed"
1405
1721
  transition.error = undefined
1722
+ transition.journalRevision = (transition.journalRevision ?? 0) + 1
1406
1723
  transition.updatedAt = new Date().toISOString()
1724
+ this.pruneStoppedReleases()
1407
1725
  await this.checkpointGenerationTransition()
1408
1726
  }
1409
1727
 
1410
- return {activeReleaseId: release.releaseId, previousReleaseId: previousRelease?.releaseId ?? null}
1728
+ return {
1729
+ activeReleaseId: release.releaseId,
1730
+ previousReleaseId: previousRelease?.releaseId ?? null,
1731
+ ...(retirementFailure && previousRelease ? {retirement: {error: retirementFailure, releaseId: previousRelease.releaseId, status: "quiescence_failed"}} : {})
1732
+ }
1411
1733
  }
1412
1734
 
1413
1735
  /**
@@ -1420,11 +1742,97 @@ export default class RollbridgeDaemon {
1420
1742
  }
1421
1743
  }
1422
1744
 
1745
+ /**
1746
+ * Returns the retained candidate only when the foreground bootstrap exactly proves the
1747
+ * committed transition that an external owner retirement left without an active role.
1748
+ * @returns {ReleaseGroup | undefined} Exact committed bootstrap candidate.
1749
+ */
1750
+ committedBootstrapRelease() {
1751
+ const bootstrap = this.bootstrap
1752
+ const transition = this.generationTransition
1753
+
1754
+ if (!bootstrap || !transition || (transition.phase !== "committed" && transition.phase !== "restoring_committed") || transition.candidateReleaseId !== bootstrap.releaseId || transition.candidateReleasePath !== bootstrap.releasePath || transition.candidateRevision !== bootstrap.revision || transition.configDigest !== ownerConfigDigest(this.config)) return undefined
1755
+ const release = this.releases.get(bootstrap.releaseId)
1756
+
1757
+ if (!release || release.releasePath !== bootstrap.releasePath || release.revision !== bootstrap.revision) return undefined
1758
+ return release
1759
+ }
1760
+
1761
+ /**
1762
+ * Fails closed before journaling recovery unless external retirement has fully stopped
1763
+ * the exact candidate and daemon-owned processes.
1764
+ * @returns {void}
1765
+ */
1766
+ assertCommittedBootstrapRecoveryReady() {
1767
+ const release = this.committedBootstrapRelease()
1768
+
1769
+ if (!release) throw new Error("Committed generation has no exact foreground bootstrap recovery proof")
1770
+ release.assertCommittedGenerationStopped()
1771
+ const ownedProcesses = [
1772
+ ...this.services.values(),
1773
+ ...this.singletons.values()
1774
+ ]
1775
+ const stillRetiring = ownedProcesses.find((processInstance) => {
1776
+ const {pid, state} = processInstance.status()
1777
+ return pid !== undefined || (state !== "stopped" && state !== "failed")
1778
+ })
1779
+
1780
+ if (stillRetiring) throw new Error(`Committed generation daemon process ${stillRetiring.id} is still retiring; exact bootstrap recovery will retry after it stops`)
1781
+ for (const serviceId of this.services.keys()) {
1782
+ const serviceReleaseId = this.serviceReleaseIds.get(serviceId)
1783
+
1784
+ if (serviceReleaseId !== release.releaseId) throw new Error(`Committed generation service ${serviceId} belongs to retained release ${serviceReleaseId}`)
1785
+ }
1786
+ for (const [singletonId, singletonReleaseId] of this.singletonReleaseIds) {
1787
+ if (singletonReleaseId !== release.releaseId) throw new Error(`Committed generation singleton ${singletonId} belongs to retained release ${singletonReleaseId}`)
1788
+ }
1789
+ }
1790
+
1791
+ /**
1792
+ * Resumes a durably journaled committed-candidate restart. Running processes are
1793
+ * necessarily owned by this exact recovery phase; stopped processes are restarted.
1794
+ * Singleton completion remains in the established committed_pending phase.
1795
+ * @param {ReleaseGroup} release - Exact committed candidate.
1796
+ * @returns {Promise<void>}
1797
+ */
1798
+ async resumeCommittedBootstrapGeneration(release) {
1799
+ if (this.generationTransition?.phase !== "restoring_committed" || release !== this.committedBootstrapRelease()) {
1800
+ throw new Error("Committed bootstrap recovery is not durably journaled for this exact candidate")
1801
+ }
1802
+ const resumableStates = new Set(["failed", "running", "stopped"])
1803
+ const invalidProcess = [...this.services.values(), ...this.singletons.values()].find((processInstance) => {
1804
+ const {pid, state} = processInstance.status()
1805
+ return !resumableStates.has(state) || (state === "running") !== (pid !== undefined)
1806
+ })
1807
+
1808
+ if (invalidProcess) throw new Error(`Committed bootstrap recovery found daemon process ${invalidProcess.id} outside its journaled restart states`)
1809
+ release.assertCommittedGenerationRecoverable()
1810
+
1811
+ try {
1812
+ for (const processInstance of this.services.values()) {
1813
+ if (this.stopping) throw new Error("Rollbridge is shutting down")
1814
+ await processInstance.start("deploy")
1815
+ }
1816
+ if (this.stopping) throw new Error("Rollbridge is shutting down")
1817
+ await release.restartCommittedGeneration()
1818
+ if (this.stopping) throw new Error("Rollbridge is shutting down")
1819
+ await release.activateGeneration()
1820
+ if (this.stopping) throw new Error("Rollbridge is shutting down")
1821
+ } catch (error) {
1822
+ await Promise.allSettled([
1823
+ release.abortCommittedGenerationRestart(),
1824
+ ...[...this.services.values()].map((processInstance) => processInstance.stop())
1825
+ ])
1826
+ throw error
1827
+ }
1828
+ }
1829
+
1423
1830
  /** @param {GenerationTransitionPhase} phase - Durable phase to enter. */
1424
1831
  async updateGenerationTransition(phase) {
1425
1832
  if (!this.generationTransition) throw new Error("No release generation transition to update")
1426
1833
  this.generationTransition.phase = phase
1427
1834
  this.generationTransition.error = undefined
1835
+ this.generationTransition.journalRevision = (this.generationTransition.journalRevision ?? 0) + 1
1428
1836
  this.generationTransition.updatedAt = new Date().toISOString()
1429
1837
  await this.checkpointGenerationTransition()
1430
1838
  }
@@ -1433,6 +1841,7 @@ export default class RollbridgeDaemon {
1433
1841
  async failGenerationTransition(error) {
1434
1842
  if (!this.generationTransition) throw new Error("No release generation transition to fail")
1435
1843
  this.generationTransition.error = error
1844
+ this.generationTransition.journalRevision = (this.generationTransition.journalRevision ?? 0) + 1
1436
1845
  this.generationTransition.updatedAt = new Date().toISOString()
1437
1846
  await this.checkpointGenerationTransition()
1438
1847
  }
@@ -1455,6 +1864,7 @@ export default class RollbridgeDaemon {
1455
1864
  shouldResumeDrain(release) {
1456
1865
  const transition = this.generationTransition
1457
1866
 
1867
+ if (release === this.committedBootstrapRelease()) return false
1458
1868
  return !transition || transition.phase === "committed_pending" || transition.phase === "committed" || transition.previousReleaseId !== release.releaseId
1459
1869
  }
1460
1870
 
@@ -1464,19 +1874,143 @@ export default class RollbridgeDaemon {
1464
1874
  * @param {net.Socket | undefined} completionSocket - Commit response connection.
1465
1875
  */
1466
1876
  async retireCommittedOwner(completionSocket) {
1877
+ if (!this.committedRetirementPromise) this.committedRetirementPromise = this.performCommittedOwnerRetirement(completionSocket)
1878
+ await this.committedRetirementPromise
1879
+ }
1880
+
1881
+ /** @param {net.Socket | undefined} completionSocket - Commit response connection. */
1882
+ async performCommittedOwnerRetirement(completionSocket) {
1467
1883
  this.ownerRetired = true
1884
+ this.controlCommandsReady = false
1468
1885
  if (this.persistTimer) clearInterval(this.persistTimer)
1469
1886
  this.persistTimer = undefined
1470
1887
  this.persistenceEnabled = false
1471
1888
  if (this.pendingWrite) await this.pendingWrite
1472
1889
  this.stateCleanupEnabled = false
1473
1890
  this.controlClosePromise = this.closeServer(this.controlServer)
1891
+ this.proxyClosePromise = this.closeServer(this.proxyServer)
1474
1892
  for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
1475
1893
  await this.removeControlSocket()
1476
- void this.closeServer(this.proxyServer)
1477
1894
  this.logger("owner authority transferred", {activeReleaseId: this.activeRelease?.releaseId ?? null})
1478
1895
  }
1479
1896
 
1897
+ /** @param {string} replacementId - Prepared same-authority transaction. */
1898
+ async yieldControlLessOwnerListeners(replacementId) {
1899
+ if (!this.guardian) throw new Error("Control-less owner listener handoff requires the durable process guardian")
1900
+ this.proxyClosePromise = this.closeServer(this.proxyServer)
1901
+ await this.beginRetiredListenerPublication(replacementId)
1902
+ }
1903
+
1904
+ /**
1905
+ * @param {string} replacementId - Transaction receiving durable source updates.
1906
+ * @param {Promise<Record<string, JsonValue>>} [retired] - Existing direct-listener retirement event.
1907
+ */
1908
+ async beginRetiredListenerPublication(replacementId, retired) {
1909
+ if (!this.guardian) throw new Error("Retired listener publication requires the durable process guardian")
1910
+ if (this.retiredListenerHandoff) throw new Error("Another retired listener handoff is already pending")
1911
+ const guardian = this.guardian
1912
+ const releases = [...this.releases.values()]
1913
+
1914
+ this.retiredListenerHandoff = {
1915
+ releases,
1916
+ replacementId,
1917
+ replacementRetired: false,
1918
+ retired: retired || guardian.waitForEvent("replacement-retired")
1919
+ }
1920
+ this.retiredReplacementId = replacementId
1921
+ for (const release of releases) {
1922
+ if (release.status().connectionCount > 0) {
1923
+ release.once("drained", () => {
1924
+ if (this.retiredReplacementId !== replacementId) return
1925
+ const publication = this.publishRetiredConnectionState(replacementId, this.listenerSourceId, release.releaseId, release.localConnections(), true)
1926
+
1927
+ void publication.then(async () => {
1928
+ await this.retiredConnectionPublication
1929
+ this.disconnectRetiredOwnerIfDrained()
1930
+ }).catch((error) => {
1931
+ this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
1932
+ this.logger("retired owner connection-state publication failed", {error: this.listenerHandoffFailure.message, releaseId: release.releaseId, replacementId})
1933
+ guardian.disconnect()
1934
+ })
1935
+ })
1936
+ }
1937
+ }
1938
+ for (const [sourceId, sourceReleases] of this.listenerConnectionSources) {
1939
+ for (const [releaseId, connections] of sourceReleases) {
1940
+ await this.publishRetiredConnectionState(replacementId, sourceId, releaseId, connections)
1941
+ }
1942
+ }
1943
+ for (const release of releases) {
1944
+ const connections = release.localConnections()
1945
+
1946
+ if (release.state === "active" || release.state === "draining" || connections.http + connections.websocket > 0) {
1947
+ await this.publishRetiredConnectionState(replacementId, this.listenerSourceId, release.releaseId, connections, true)
1948
+ }
1949
+ }
1950
+ await guardian.completeOwnerListenerRetirement(replacementId)
1951
+ }
1952
+
1953
+ /** Resumes the incumbent listener after a prepared candidate aborts before commit. */
1954
+ async resumeControlLessOwnerListeners() {
1955
+ const handoff = this.retiredListenerHandoff
1956
+
1957
+ if (!handoff || this.ownerRetired) return
1958
+ void handoff.retired.catch(() => undefined)
1959
+ this.retiredListenerHandoff = undefined
1960
+ this.retiredReplacementId = undefined
1961
+ for (const release of handoff.releases) {
1962
+ release.resumeDrainAfterOwnerHandoff()
1963
+ if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
1964
+ }
1965
+ await this.startProxy()
1966
+ this.listenerHandoffFailure = undefined
1967
+ this.logger("control-less owner listeners resumed", {replacementId: handoff.replacementId})
1968
+ }
1969
+
1970
+ /** @param {string} replacementId - Committed same-authority transaction. */
1971
+ async completeControlLessOwnerRetirement(replacementId) {
1972
+ const handoff = this.retiredListenerHandoff
1973
+
1974
+ if (!handoff || handoff.replacementId !== replacementId) throw new Error("Committed owner retirement does not match the prepared listener handoff")
1975
+ await this.retireCommittedOwner(undefined)
1976
+ await this.completeRetiredListenerHandoff(replacementId)
1977
+ }
1978
+
1979
+ /** @param {string} replacementId - Finalized retired-listener transaction. */
1980
+ async completeRetiredListenerHandoff(replacementId) {
1981
+ const handoff = this.retiredListenerHandoff
1982
+
1983
+ if (!handoff || handoff.replacementId !== replacementId) throw new Error("Retired listener completion does not match its prepared handoff")
1984
+ await handoff.retired
1985
+ handoff.replacementRetired = true
1986
+ this.disconnectRetiredOwnerIfDrained()
1987
+ }
1988
+
1989
+ /** Disconnects the retired daemon only after every retained listener source drains. */
1990
+ disconnectRetiredOwnerIfDrained() {
1991
+ const handoff = this.retiredListenerHandoff
1992
+
1993
+ if (handoff?.replacementRetired && handoff.releases.every((release) => release.status().connectionCount === 0)) this.guardian?.disconnect()
1994
+ }
1995
+
1996
+ /**
1997
+ * Serializes one retired-listener source update over the authenticated guardian channel.
1998
+ * @param {string} replacementId - Committed replacement transaction.
1999
+ * @param {string} sourceId - Stable physical listener source.
2000
+ * @param {string} releaseId - Retained release.
2001
+ * @param {{http: number, websocket: number}} connections - Exact source counts.
2002
+ * @param {boolean} [localSource] - Whether this daemon physically owns the source.
2003
+ */
2004
+ async publishRetiredConnectionState(replacementId, sourceId, releaseId, connections, localSource = false) {
2005
+ if (!this.guardian) throw new Error("Retired listener state publication requires the durable process guardian")
2006
+ const guardian = this.guardian
2007
+
2008
+ this.retiredConnectionPublication = this.retiredConnectionPublication.then(async () => {
2009
+ await guardian.publishOwnerConnectionState(replacementId, sourceId, releaseId, connections, localSource)
2010
+ })
2011
+ await this.retiredConnectionPublication
2012
+ }
2013
+
1480
2014
  /**
1481
2015
  * Rejects config changes that require rebinding daemon-owned resources or changing process topology.
1482
2016
  * @param {import("./config.js").RollbridgeConfig} nextConfig - Freshly loaded config.
@@ -1490,6 +2024,10 @@ export default class RollbridgeDaemon {
1490
2024
  if (!isDeepStrictEqual(nextConfig.control, this.config.control)) restartRequired.push("control")
1491
2025
  if (nextConfig.statePath !== this.config.statePath) restartRequired.push("statePath")
1492
2026
  if (!isDeepStrictEqual(nextConfig.ownerRecovery, this.config.ownerRecovery)) restartRequired.push("ownerRecovery")
2027
+ const activationProcessIds = this.config.processes.filter((processConfig) => processConfig.lifecycle.activateCommand !== undefined).map((processConfig) => processConfig.id)
2028
+ const nextActivationProcessIds = nextConfig.processes.filter((processConfig) => processConfig.lifecycle.activateCommand !== undefined).map((processConfig) => processConfig.id)
2029
+
2030
+ if (!isDeepStrictEqual(nextActivationProcessIds, activationProcessIds)) restartRequired.push("processes.lifecycle.activateCommand")
1493
2031
 
1494
2032
  if (nextConfig.proxy.host !== this.config.proxy.host) restartRequired.push("proxy.host")
1495
2033
  if (nextConfig.proxy.port !== this.config.proxy.port) restartRequired.push("proxy.port")
@@ -1592,6 +2130,7 @@ export default class RollbridgeDaemon {
1592
2130
  startedServices.push(processConfig.id)
1593
2131
  } catch (error) {
1594
2132
  this.services.delete(processConfig.id)
2133
+ this.serviceReleaseIds.delete(processConfig.id)
1595
2134
  delete this.servicePorts[processConfig.id]
1596
2135
  throw error
1597
2136
  }
@@ -1613,6 +2152,7 @@ export default class RollbridgeDaemon {
1613
2152
 
1614
2153
  await service.stop()
1615
2154
  this.services.delete(serviceId)
2155
+ this.serviceReleaseIds.delete(serviceId)
1616
2156
  const port = this.servicePorts[serviceId]
1617
2157
 
1618
2158
  if (port !== undefined) this.portReservations.delete(port)
@@ -1623,9 +2163,9 @@ export default class RollbridgeDaemon {
1623
2163
  /**
1624
2164
  * Updates daemon-wide service restart templates after a successful deploy.
1625
2165
  * @param {ReleaseGroup} release - Active release.
1626
- * @returns {void}
2166
+ * @returns {Promise<void>} Resolves once every persistent service definition is committed.
1627
2167
  */
1628
- refreshServiceDefinitions(release) {
2168
+ async refreshServiceDefinitions(release) {
1629
2169
  for (const processConfig of this.config.processes) {
1630
2170
  if (processConfig.policy !== "service") continue
1631
2171
 
@@ -1634,8 +2174,16 @@ export default class RollbridgeDaemon {
1634
2174
  if (!service) continue
1635
2175
 
1636
2176
  const nextDefinition = release.buildProcess(processConfig, {shouldRestart: () => !this.stopping})
2177
+ const previousReleaseId = this.serviceReleaseIds.get(processConfig.id)
1637
2178
 
1638
- service.updateDefinition(nextDefinition)
2179
+ this.serviceReleaseIds.set(processConfig.id, release.releaseId)
2180
+ try {
2181
+ await service.updateDefinition(nextDefinition, this.guardian ? this.transferableOwnerState() : undefined)
2182
+ } catch (error) {
2183
+ if (previousReleaseId) this.serviceReleaseIds.set(processConfig.id, previousReleaseId)
2184
+ else this.serviceReleaseIds.delete(processConfig.id)
2185
+ throw error
2186
+ }
1639
2187
  }
1640
2188
  }
1641
2189
 
@@ -1727,6 +2275,7 @@ export default class RollbridgeDaemon {
1727
2275
  */
1728
2276
  runningInstances(processConfig) {
1729
2277
  if (processConfig.policy === "service") {
2278
+ if (processConfig.deployStrategy === "handoff") return this.activeRelease?.getProcesses(processConfig.id) || []
1730
2279
  const service = this.services.get(processConfig.id)
1731
2280
 
1732
2281
  return service ? [{id: processConfig.id, process: service}] : []
@@ -1775,6 +2324,15 @@ export default class RollbridgeDaemon {
1775
2324
  }
1776
2325
  }
1777
2326
 
2327
+ /** @returns {Set<string>} Release definitions owned by an unresolved durable transition. */
2328
+ generationTransitionReleaseIds() {
2329
+ const transition = this.generationTransition
2330
+
2331
+ return new Set(transition && transition.phase !== "committed"
2332
+ ? [transition.candidateReleaseId, ...(transition.previousReleaseId ? [transition.previousReleaseId] : [])]
2333
+ : [])
2334
+ }
2335
+
1778
2336
  /**
1779
2337
  * Drains and stops a retired release in the background, then prunes stopped releases.
1780
2338
  * @param {ReleaseGroup} release - Release to drain and stop.
@@ -1797,9 +2355,11 @@ export default class RollbridgeDaemon {
1797
2355
 
1798
2356
  /** @returns {void} Removes stopped releases beyond the retention policy. */
1799
2357
  pruneStoppedReleases() {
2358
+ const serviceOwnerReleaseIds = new Set(this.serviceReleaseIds.values())
1800
2359
  const singletonOwnerReleaseIds = new Set(this.singletonReleaseIds.values())
2360
+ const transitionReleaseIds = this.generationTransitionReleaseIds()
1801
2361
  const statuses = [...this.releases.values()]
1802
- .filter((release) => !singletonOwnerReleaseIds.has(release.releaseId))
2362
+ .filter((release) => !serviceOwnerReleaseIds.has(release.releaseId) && !singletonOwnerReleaseIds.has(release.releaseId) && !transitionReleaseIds.has(release.releaseId))
1803
2363
  .map((release) => release.status())
1804
2364
 
1805
2365
  for (const releaseId of releasesToPrune(statuses, this.config.releaseRetention, Date.now())) {
@@ -1835,6 +2395,7 @@ export default class RollbridgeDaemon {
1835
2395
  ...status,
1836
2396
  events,
1837
2397
  persistedAt: new Date().toISOString(),
2398
+ serviceReleaseIds: Object.fromEntries(this.serviceReleaseIds),
1838
2399
  ...(this.hasActivationLifecycle() ? {singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds)} : {}),
1839
2400
  ...(this.guardianIdentity ? {recovery: {
1840
2401
  configDigest: this.ownerRecoveryConfigDigest(),
@@ -1848,8 +2409,8 @@ export default class RollbridgeDaemon {
1848
2409
  // clearing the file — otherwise a write started before shutdown could recreate it afterward.
1849
2410
  this.pendingWrite = Promise.resolve(this.pendingWrite)
1850
2411
  .catch(() => {})
1851
- .then(() => writeState(statePath, snapshot))
1852
2412
  .then(() => this.publishOwnerState())
2413
+ .then(() => writeState(statePath, snapshot))
1853
2414
  .catch((error) => {
1854
2415
  this.logger("state persist failed", {error: error instanceof Error ? error.message : String(error)})
1855
2416
  if (throwOnError) throw error
@@ -1895,6 +2456,7 @@ export default class RollbridgeDaemon {
1895
2456
  * @returns {Promise<void>} Resolves when owned resources are stopped (and, by default, control connections close).
1896
2457
  */
1897
2458
  async shutdown({completionSocket, waitForControlConnections = true} = {}) {
2459
+ this.assertNoGenerationTransitionRecovery("shut down")
1898
2460
  if (!this.shutdownPromise) this.shutdownPromise = this.performShutdown(completionSocket)
1899
2461
 
1900
2462
  await this.shutdownPromise
@@ -1908,11 +2470,17 @@ export default class RollbridgeDaemon {
1908
2470
  * @returns {Promise<void>} Resolves once a replacement can exclusively bind listeners.
1909
2471
  */
1910
2472
  async retireOwner({attestation, completionSocket}) {
2473
+ this.assertNoGenerationTransitionRecovery("retire owner")
1911
2474
  if (this.retirementPromise) return await this.retirementPromise
1912
2475
  this.retirementPromise = this.performOwnerRetirement(attestation, completionSocket)
1913
2476
  return await this.retirementPromise
1914
2477
  }
1915
2478
 
2479
+ /** @param {string} operation - Terminal owner operation. */
2480
+ assertNoGenerationTransitionRecovery(operation) {
2481
+ if (this.generationTransitionRecovery) throw new Error(`Cannot ${operation} while generation transition recovery is in progress`)
2482
+ }
2483
+
1916
2484
  /**
1917
2485
  * @param {string} attestation - Replacement boot attestation.
1918
2486
  * @param {net.Socket | undefined} completionSocket - Requesting handoff connection.
@@ -2022,9 +2590,23 @@ export default class RollbridgeDaemon {
2022
2590
  /** @returns {Promise<void>} Removes the configured control socket path. */
2023
2591
  async removeControlSocket() {
2024
2592
  if (!this.controlSocketOwned) return
2593
+ const socketPath = this.boundControlPath || this.config.control.path
2594
+ const expectedIdentity = this.boundControlIdentity
2595
+ let ownsCurrentPath = true
2596
+
2597
+ if (expectedIdentity) {
2598
+ try {
2599
+ const currentIdentity = await fs.lstat(socketPath)
2025
2600
 
2026
- await fs.rm(this.boundControlPath || this.config.control.path, {force: true})
2601
+ ownsCurrentPath = currentIdentity.dev === expectedIdentity.dev && currentIdentity.ino === expectedIdentity.ino
2602
+ } catch (error) {
2603
+ if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
2604
+ ownsCurrentPath = false
2605
+ }
2606
+ }
2607
+ if (ownsCurrentPath) await fs.rm(socketPath, {force: true})
2027
2608
  this.controlSocketOwned = false
2609
+ this.boundControlIdentity = undefined
2028
2610
  this.boundControlPath = undefined
2029
2611
  }
2030
2612
 
@@ -2050,7 +2632,9 @@ export default class RollbridgeDaemon {
2050
2632
  // a cleared orphan must not reappear if the OS later recycles its pid for an unrelated process.
2051
2633
  this.orphans = this.orphans.filter((orphan) => isProcessAlive(orphan.pid))
2052
2634
 
2635
+ const serviceOwnerReleaseIds = new Set(this.serviceReleaseIds.values())
2053
2636
  const singletonOwnerReleaseIds = new Set(this.singletonReleaseIds.values())
2637
+ const transitionReleaseIds = this.generationTransitionReleaseIds()
2054
2638
 
2055
2639
  return {
2056
2640
  activeReleaseId: this.activeRelease ? this.activeRelease.releaseId : null,
@@ -2060,7 +2644,7 @@ export default class RollbridgeDaemon {
2060
2644
  daemonPid: process.pid,
2061
2645
  daemonRuntime: this.runtime ? {...this.runtime} : undefined,
2062
2646
  generationTransition: this.generationTransition ? {...this.generationTransition} : undefined,
2063
- ownerRecovery: this.guardian ? {configDigest: this.ownerRecoveryConfigDigest()} : undefined,
2647
+ ownerRecovery: this.guardian ? {configDigest: this.ownerRecoveryConfigDigest(), ready: this.ownerReady} : undefined,
2064
2648
  ownerTransition: this.ownerTransition ? {...this.ownerTransition} : undefined,
2065
2649
  orphans: [...this.orphans],
2066
2650
  proxy: {
@@ -2069,13 +2653,15 @@ export default class RollbridgeDaemon {
2069
2653
  upstreamHost: this.config.proxy.upstreamHost
2070
2654
  },
2071
2655
  releaseReferences: [...this.releases.values()]
2072
- .filter((release) => release.state === "active" || release.state === "draining" || singletonOwnerReleaseIds.has(release.releaseId) || (this.generationTransition?.phase !== "committed" && this.generationTransition?.candidateReleaseId === release.releaseId))
2656
+ .filter((release) => release.state === "active" || release.state === "draining" || serviceOwnerReleaseIds.has(release.releaseId) || singletonOwnerReleaseIds.has(release.releaseId) || transitionReleaseIds.has(release.releaseId))
2073
2657
  .map((release) => ({releaseId: release.releaseId, releasePath: release.releasePath})),
2074
2658
  releases: [...this.releases.values()].map((release) => release.status()),
2075
- services: [...this.services.entries()].map(([id, processInstance]) => ({
2076
- id,
2077
- process: processInstance.status()
2078
- })),
2659
+ services: [...this.services.entries()]
2660
+ .filter(([id]) => this.serviceReleaseIds.has(id))
2661
+ .map(([id, processInstance]) => ({
2662
+ id,
2663
+ process: processInstance.status()
2664
+ })),
2079
2665
  singletons: [...this.singletons.entries()].map(([id, processInstance]) => ({
2080
2666
  id,
2081
2667
  process: processInstance.status()
@@ -2203,8 +2789,10 @@ async function verifyLegacyDaemonProcess(pid, configPath, socketPath) {
2203
2789
  const args = await processArguments(pid, "legacy daemon")
2204
2790
  const daemonIndex = args.indexOf("daemon")
2205
2791
  const configIndex = args.indexOf("--config")
2792
+ const configuredPath = args[configIndex + 1]
2793
+ const configuredAbsolutePath = configuredPath ? path.resolve(await fs.readlink(`/proc/${pid}/cwd`), configuredPath) : undefined
2206
2794
 
2207
- if (daemonIndex < 0 || configIndex < 0 || args[configIndex + 1] !== configPath) {
2795
+ if (daemonIndex < 0 || configIndex < 0 || configuredAbsolutePath !== path.resolve(configPath)) {
2208
2796
  throw new Error(`Daemon PID ${pid} does not match the exact retained daemon config command`)
2209
2797
  }
2210
2798
  await verifyProcessUser(pid, "legacy daemon")