rollbridge 0.1.28 → 0.1.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/AGENTS.md +14 -0
  2. package/README.md +69 -14
  3. package/TODO.md +5 -2
  4. package/changelog.d/20260828-atomic-owner-replacement.md +22 -0
  5. package/changelog.d/20260828-durable-owner-recovery.md +7 -0
  6. package/changelog.d/20260828-same-owner-jobs-generations.md +6 -0
  7. package/docs/cli.md +43 -21
  8. package/docs/config.md +71 -18
  9. package/docs/logging.md +8 -3
  10. package/docs/tensorbuzz-runbook.md +7 -6
  11. package/docs/troubleshooting.md +28 -12
  12. package/docs/velocious.md +11 -4
  13. package/docs/workers.md +8 -2
  14. package/examples/tensorbuzz.com.js +12 -4
  15. package/package.json +1 -1
  16. package/src/cli.js +209 -36
  17. package/src/config.js +8 -2
  18. package/src/control-client.js +118 -1
  19. package/src/daemon.js +939 -53
  20. package/src/guardian-client.js +434 -0
  21. package/src/managed-process.js +45 -15
  22. package/src/process-guardian.js +601 -0
  23. package/src/release-group.js +190 -15
  24. package/src/state-store.js +1 -1
  25. package/test/config-validation.test.js +22 -0
  26. package/test/fixtures/pre-split3-daemon-runner.js +30 -0
  27. package/test/fixtures/pre-split3-daemon.js +1336 -0
  28. package/test/fixtures/pre-split3-guardian-client.js +293 -0
  29. package/test/fixtures/pre-split3-process-guardian.js +292 -0
  30. package/test/fixtures/service-app.js +32 -2
  31. package/test/guardian-client.test.js +304 -0
  32. package/test/owner-recovery.test.js +950 -0
  33. package/test/owner-replacement.test.js +772 -0
  34. package/test/release-runtime-retention.test.js +1 -1
  35. package/test/rollbridge.test.js +178 -5
  36. package/test/shutdown-completion.test.js +1 -1
  37. package/test/state-store.test.js +12 -0
package/src/daemon.js CHANGED
@@ -3,10 +3,13 @@
3
3
  import fs from "node:fs/promises"
4
4
  import http from "node:http"
5
5
  import net from "node:net"
6
+ import crypto from "node:crypto"
6
7
  import {isDeepStrictEqual} from "node:util"
7
8
  import httpProxy from "http-proxy"
8
9
  import {loadConfig} from "./config.js"
10
+ import {openControlSession} from "./control-client.js"
9
11
  import EventLog from "./event-log.js"
12
+ import GuardianClient from "./guardian-client.js"
10
13
  import ReleaseGroup from "./release-group.js"
11
14
  import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
12
15
  import {resolveGroupId, resolveUserId} from "./system-ids.js"
@@ -19,7 +22,11 @@ const STATE_PERSIST_INTERVAL_MS = 5000
19
22
  * @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
20
23
  * @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
21
24
  * @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
22
- * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
25
+ * @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
26
+ * @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, ownerRecovery: {configDigest: string} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
27
+ * @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
28
+ * @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata}} OwnerRecoverySnapshot
29
+ * @typedef {{boundaryCrossed: boolean, incumbentControl: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
23
30
  */
24
31
 
25
32
  export default class RollbridgeDaemon {
@@ -29,13 +36,15 @@ export default class RollbridgeDaemon {
29
36
  * @param {import("./config.js").RollbridgeConfig} args.config - Rollbridge config.
30
37
  * @param {string} [args.configPath] - Config file path to reload before deploys.
31
38
  * @param {(message: string, data?: Record<string, JsonValue>) => void} [args.logger] - Logger.
39
+ * @param {number} [args.legacyIncumbentPid] - Exact incumbent PID supplied by ensure-daemon.
32
40
  * @param {import("./daemon-runtime.js").DaemonRuntimeIdentity} [args.runtime] - Immutable daemon runtime identity.
33
41
  */
34
- constructor({bootstrap, config, configPath, logger, runtime}) {
42
+ constructor({bootstrap, config, configPath, legacyIncumbentPid, logger, runtime}) {
35
43
  this.bootstrap = bootstrap ? {...bootstrap} : undefined
36
44
  this.config = config
37
45
  this.configPath = configPath
38
46
  this.runtime = runtime
47
+ this.legacyIncumbentPid = legacyIncumbentPid
39
48
  this.eventLog = new EventLog(EVENT_HISTORY_LIMIT)
40
49
 
41
50
  const baseLogger = logger || ((message, data = {}) => console.log(JSON.stringify({at: new Date().toISOString(), data, message})))
@@ -51,12 +60,14 @@ export default class RollbridgeDaemon {
51
60
  this.releases = /** @type {Map<string, ReleaseGroup>} */ (new Map())
52
61
  this.services = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
53
62
  this.servicePorts = /** @type {Record<string, number>} */ ({})
63
+ this.portReservations = /** @type {Set<number>} */ (new Set())
54
64
  this.singletons = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
55
65
  this.activeRelease = /** @type {ReleaseGroup | undefined} */ (undefined)
56
66
  this.proxy = httpProxy.createProxyServer({ws: true, xfwd: true})
57
67
  this.proxyServer = /** @type {http.Server | undefined} */ (undefined)
58
68
  this.controlServer = /** @type {net.Server | undefined} */ (undefined)
59
69
  this.controlSocketOwned = false
70
+ this.boundControlPath = /** @type {string | undefined} */ (undefined)
60
71
  this.controlSockets = /** @type {Set<net.Socket>} */ (new Set())
61
72
  this.proxyPort = /** @type {number | undefined} */ (undefined)
62
73
  this.stopping = false
@@ -67,8 +78,17 @@ export default class RollbridgeDaemon {
67
78
  this.stateCleanupEnabled = false
68
79
  this.shutdownPromise = /** @type {Promise<void> | undefined} */ (undefined)
69
80
  this.retirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
81
+ this.ownerRetired = false
70
82
  this.controlClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
83
+ this.proxyClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
84
+ this.listenerHandoff = /** @type {{control: boolean, proxy: boolean, replacementId: string} | undefined} */ (undefined)
85
+ this.listenerHandoffFailure = /** @type {Error | undefined} */ (undefined)
86
+ this.incumbentListenerControl = /** @type {Awaited<ReturnType<typeof openControlSession>> | undefined} */ (undefined)
87
+ this.ownerTransition = /** @type {OwnerTransition | undefined} */ (undefined)
88
+ this.controlCommandsReady = true
71
89
  this.startingReleases = /** @type {Set<ReleaseGroup>} */ (new Set())
90
+ this.guardian = /** @type {GuardianClient | undefined} */ (undefined)
91
+ this.guardianIdentity = /** @type {{pid?: number, socketPath: string, token: string} | undefined} */ (undefined)
72
92
  // Still-alive managed processes left by a previous daemon (from statePath), captured at
73
93
  // startup and surfaced in status(). The daemon cannot re-manage them, only report them.
74
94
  this.orphans = /** @type {{id: string, pid: number, releaseId: string | null}[]} */ ([])
@@ -82,11 +102,473 @@ export default class RollbridgeDaemon {
82
102
  * @returns {Promise<void>} Resolves when the requested listeners are ready.
83
103
  */
84
104
  async start({exposeControl = true, reportOrphans = true} = {}) {
85
- if (reportOrphans) await this.reportOrphans()
105
+ if (this.config.ownerRecovery) await this.initializeOwnerRecovery()
106
+ else if (reportOrphans) await this.reportOrphans()
86
107
  await this.startProxy()
87
108
  if (exposeControl) await this.exposeControl()
88
109
  }
89
110
 
111
+ /** Connects to the durable process guardian and reconstructs a matching persisted owner snapshot. */
112
+ async initializeOwnerRecovery() {
113
+ if (!this.statePath) throw new Error("ownerRecovery requires statePath")
114
+ const state = await readState(this.statePath)
115
+ const snapshot = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
116
+ const recovery = snapshot?.recovery
117
+ const configDigest = this.ownerRecoveryConfigDigest()
118
+
119
+ if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
120
+ if (recovery && recovery.configDigest !== configDigest) throw new Error("Owner recovery config identity does not match the persisted owner; refusing cross-authority adoption.")
121
+ if (snapshot && ((this.runtime?.digest ?? null) !== (snapshot.daemonRuntime?.digest ?? null))) {
122
+ throw new Error("Owner recovery runtime identity does not match the persisted owner; use the exact same Rollbridge runtime.")
123
+ }
124
+
125
+ const guardianIdentity = recovery?.guardian || {
126
+ socketPath: `${this.statePath}.guardian.sock`,
127
+ token: crypto.randomBytes(32).toString("hex")
128
+ }
129
+ this.guardianIdentity = guardianIdentity
130
+ this.guardian = new GuardianClient(guardianIdentity)
131
+ if (recovery) await this.guardian.connect()
132
+ else {
133
+ await this.guardian.launch()
134
+ guardianIdentity.pid = this.guardian.pid
135
+ }
136
+ await this.guardian.claimOwner(this.config.ownerRecovery?.reconnectGraceMs ?? 30000, this.ownerAuthority())
137
+ this.watchOwnerReplacementEvents()
138
+
139
+ if (snapshot) {
140
+ await this.restoreOwnerState(snapshot, {resumeDrains: false})
141
+ await this.guardian.reconcileInventory()
142
+ for (const release of this.releases.values()) {
143
+ if (release.state === "draining") void this.drainAndPrune(release, release.config)
144
+ }
145
+ }
146
+ else {
147
+ this.persistenceEnabled = true
148
+ await this.persistState({throwOnError: true})
149
+ }
150
+ this.stateCleanupEnabled = true
151
+ await this.publishOwnerState()
152
+ }
153
+
154
+ /** @returns {string} Stable identity for same-authority recovery. */
155
+ ownerRecoveryConfigDigest() {
156
+ return ownerConfigDigest(this.config)
157
+ }
158
+
159
+ /** Installs event-driven drain fencing for the next prepared owner transaction. */
160
+ watchOwnerReplacementEvents() {
161
+ if (!this.guardian) throw new Error("Owner replacement event fencing requires the durable guardian")
162
+ this.guardian.onEvent("replacement-prepared", () => {
163
+ for (const release of this.releases.values()) release.pauseDrainForOwnerHandoff()
164
+ })
165
+ this.guardian.onEvent("replacement-aborted", () => {
166
+ if (this.listenerHandoff || this.ownerRetired) return
167
+ for (const release of this.releases.values()) {
168
+ release.resumeDrainAfterOwnerHandoff()
169
+ if (release.state === "draining") void this.drainAndPrune(release, release.config)
170
+ }
171
+ })
172
+ }
173
+
174
+ /** @returns {{configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}} Exact authority fence. */
175
+ ownerAuthority() {
176
+ return {configDigest: this.ownerRecoveryConfigDigest(), runtime: this.runtime ? {...this.runtime} : null}
177
+ }
178
+
179
+ /**
180
+ * @param {OwnerRecoverySnapshot} snapshot - Validated persisted owner state.
181
+ * @param {object} [options] - Recovery definition options.
182
+ * @param {import("./config.js").RollbridgeConfig} [options.config] - Owner config for daemon-wide processes.
183
+ * @param {Record<string, import("./config.js").RollbridgeConfig>} [options.releaseConfigs] - Exact generation configs.
184
+ * @param {boolean} [options.resumeDrains] - Whether to resume draining generations immediately.
185
+ */
186
+ async restoreOwnerState(snapshot, {config = this.config, releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({}), resumeDrains = true} = {}) {
187
+ if (!Array.isArray(snapshot.releases) || (snapshot.activeReleaseId !== null && typeof snapshot.activeReleaseId !== "string")) {
188
+ throw new Error("Owner recovery state is partial or corrupt; active release metadata is required.")
189
+ }
190
+ if (snapshot.activeReleaseId === null && snapshot.releases.length === 0) return
191
+ this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
192
+ this.ownerTransition = snapshot.ownerTransition ? {...snapshot.ownerTransition} : undefined
193
+
194
+ for (const releaseStatus of snapshot.releases) {
195
+ if (releaseStatus.state !== "active" && releaseStatus.state !== "draining") continue
196
+ const release = new ReleaseGroup({
197
+ config: releaseConfigs[releaseStatus.releaseId] || config,
198
+ logger: this.logger,
199
+ portReservations: this.portReservations,
200
+ processFactory: (key, definition) => this.guardianProcess(key, definition),
201
+ releaseId: releaseStatus.releaseId,
202
+ releasePath: releaseStatus.releasePath,
203
+ revision: releaseStatus.revision,
204
+ servicePorts: this.servicePorts,
205
+ shouldStart: () => !this.stopping
206
+ })
207
+
208
+ await release.restore(releaseStatus)
209
+ this.releases.set(release.releaseId, release)
210
+ if (release.releaseId === snapshot.activeReleaseId) this.activeRelease = release
211
+ }
212
+
213
+ if (snapshot.activeReleaseId !== null && !this.activeRelease) throw new Error(`Owner recovery state does not contain active release ${snapshot.activeReleaseId}.`)
214
+ const definitionRelease = this.activeRelease || [...this.releases.values()].at(-1)
215
+ if (!definitionRelease) throw new Error("Owner recovery state has no release definition for owned processes.")
216
+ if (!this.activeRelease && snapshot.singletons.length > 0) throw new Error("Owner recovery state has release-owned singletons without an active release identity.")
217
+ for (const serviceStatus of snapshot.services) {
218
+ const processConfig = config.processes.find((candidate) => candidate.id === serviceStatus.id && candidate.policy === "service" && candidate.deployStrategy !== "handoff")
219
+
220
+ if (!processConfig) throw new Error(`Owner recovery state contains unknown service ${serviceStatus.id}.`)
221
+ const service = definitionRelease.buildProcess(processConfig, {guardianKey: `service:${serviceStatus.id}`, shouldRestart: () => !this.stopping})
222
+
223
+ await this.recoverGuardianProcess(service)
224
+ this.services.set(serviceStatus.id, service)
225
+ if (definitionRelease.ports[serviceStatus.id] !== undefined) {
226
+ const port = definitionRelease.ports[serviceStatus.id]
227
+
228
+ if (this.portReservations.has(port)) throw new Error(`Persisted daemon service ${serviceStatus.id} port ${port} is already reserved by a live generation`)
229
+ this.portReservations.add(port)
230
+ this.servicePorts[serviceStatus.id] = port
231
+ }
232
+ }
233
+ for (const singletonStatus of snapshot.singletons) {
234
+ const processConfig = config.processes.find((candidate) => candidate.id === singletonStatus.id && candidate.policy === "singleton")
235
+
236
+ if (!processConfig) throw new Error(`Owner recovery state contains unknown singleton ${singletonStatus.id}.`)
237
+ const singleton = definitionRelease.buildProcess(processConfig, {guardianKey: `singleton:${definitionRelease.releaseId}:${singletonStatus.id}`})
238
+
239
+ await this.recoverGuardianProcess(singleton)
240
+ this.singletons.set(singletonStatus.id, singleton)
241
+ }
242
+ for (const release of this.releases.values()) {
243
+ if (resumeDrains && release.state === "draining") void this.drainAndPrune(release, release.config)
244
+ }
245
+ this.logger("owner state recovered", {activeReleaseId: this.activeRelease?.releaseId ?? null, releases: this.releases.size})
246
+ }
247
+
248
+ /** Publishes full normalized definitions only over the authenticated guardian channel. */
249
+ async publishOwnerState() {
250
+ if (!this.guardian) return
251
+ await this.guardian.publishOwnerState({
252
+ authority: this.ownerAuthority(),
253
+ config: this.config,
254
+ releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
255
+ snapshot: this.status()
256
+ })
257
+ }
258
+
259
+ /**
260
+ * Prepares from private guardian state, starts listeners, then asks the committed owner
261
+ * to atomically fence itself and transfer authority.
262
+ */
263
+ async replaceIncompatibleOwner() {
264
+ if (!this.statePath || !this.config.ownerRecovery) throw new Error("Atomic owner replacement requires ownerRecovery and the same statePath transaction anchor")
265
+ this.controlCommandsReady = false
266
+ const state = await readState(this.statePath)
267
+ const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
268
+
269
+ if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
270
+ this.guardianIdentity = persisted.recovery.guardian
271
+ this.guardian = new GuardianClient(this.guardianIdentity)
272
+ await this.guardian.connect()
273
+ this.watchOwnerReplacementEvents()
274
+ const persistedAuthority = {
275
+ configDigest: persisted.recovery.configDigest,
276
+ runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
277
+ }
278
+ let legacyBridge
279
+ let prepared
280
+
281
+ try {
282
+ prepared = await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
283
+ } catch (error) {
284
+ legacyBridge = await this.prepareLegacyOwnerReplacement({
285
+ error: error instanceof Error ? error : String(error),
286
+ persisted,
287
+ persistedAuthority
288
+ })
289
+ prepared = legacyBridge.prepared
290
+ }
291
+ const preparedStatus = await this.guardian.replacementStatus()
292
+ const transfer = /** @type {{config: import("./config.js").RollbridgeConfig, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, snapshot: OwnerRecoverySnapshot}} */ (prepared.ownerState)
293
+
294
+ if (!transfer?.config || !transfer.snapshot) throw new Error("Committed owner published incomplete replacement state")
295
+ await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, releaseConfigs: transfer.releaseConfigs, resumeDrains: false})
296
+ for (const release of this.releases.values()) release.preserveConfigOnRetirement = true
297
+ this.logger("owner replacement candidate prepared", {activeReleaseId: this.activeRelease?.releaseId ?? null, replacementId: prepared.replacementId})
298
+
299
+ let committedAuthority = false
300
+ let stagingControlPath
301
+ let finalControlPublished = false
302
+ let listenersYielded = false
303
+ let retainIncumbentControl = false
304
+ let incumbentControl = legacyBridge?.incumbentControl
305
+
306
+ try {
307
+ if (this.config.control.path !== transfer.snapshot.control.path) {
308
+ const finalSocket = await inspectControlSocket(this.config.control.path)
309
+
310
+ if (finalSocket.alive) throw new Error(`Owner replacement final control socket ${this.config.control.path} already answers another live process`)
311
+ try {
312
+ await fs.lstat(this.config.control.path)
313
+ throw new Error(`Owner replacement final control socket ${this.config.control.path} already exists; refusing to replace it`)
314
+ } catch (error) {
315
+ if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
316
+ }
317
+ }
318
+
319
+ stagingControlPath = `${this.config.control.path}.replacement-${process.pid}`
320
+ await this.startControlServer(stagingControlPath)
321
+ const sharedFixedProxy = this.config.proxy.port !== 0 &&
322
+ transfer.snapshot.proxy.host === this.config.proxy.host && transfer.snapshot.proxy.port === this.config.proxy.port
323
+
324
+ if (!sharedFixedProxy) await this.startProxy()
325
+ if (legacyBridge) {
326
+ await this.crossLegacyDisruptiveBoundary(legacyBridge)
327
+ } else if (preparedStatus.ownerClaimed) {
328
+ incumbentControl = await openControlSession(transfer.snapshot.control.path)
329
+ const listenerSession = incumbentControl
330
+
331
+ incumbentControl.onEvent((event) => this.handleIncumbentListenerEvent(event, listenerSession))
332
+ await incumbentControl.request({
333
+ command: "yield-owner-listeners",
334
+ control: transfer.snapshot.control.path === this.config.control.path,
335
+ proxy: true,
336
+ replacementId: prepared.replacementId
337
+ })
338
+ listenersYielded = true
339
+ }
340
+ if (sharedFixedProxy) await this.startProxy()
341
+ await fs.rename(stagingControlPath, this.config.control.path)
342
+ finalControlPublished = true
343
+ this.boundControlPath = this.config.control.path
344
+ const committed = this.guardian.waitForEvent("replacement-committed")
345
+ const staged = await this.guardian.stageOwnerReplacement(prepared.replacementId, {
346
+ authority: this.ownerAuthority(),
347
+ config: this.config,
348
+ releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
349
+ snapshot: this.status()
350
+ })
351
+
352
+ if (!staged.committed) {
353
+ try {
354
+ if (!incumbentControl) throw new Error("Owner replacement incumbent control session is unavailable")
355
+ await incumbentControl.request({command: "commit-owner-replacement", replacementId: prepared.replacementId})
356
+ } catch (error) {
357
+ const status = await this.guardian.replacementStatus()
358
+
359
+ if (status.committedReplacementId !== prepared.replacementId || !status.ownerClaimed) throw error
360
+ this.logger("owner replacement commit response lost; guardian commit confirmed", {replacementId: prepared.replacementId})
361
+ }
362
+ }
363
+ await committed
364
+ committedAuthority = true
365
+ if (!legacyBridge && incumbentControl && [...this.releases.values()].some((release) => release.hasTransferredConnections())) {
366
+ this.incumbentListenerControl = incumbentControl
367
+ retainIncumbentControl = true
368
+ }
369
+ for (const release of this.releases.values()) {
370
+ if (release.state === "draining") void this.drainAndPrune(release, release.config)
371
+ }
372
+ this.startStatePersistence()
373
+ await this.persistState({throwOnError: true})
374
+ await this.publishOwnerState()
375
+ this.controlCommandsReady = true
376
+ this.logger("owner replacement committed", {replacementId: prepared.replacementId})
377
+ } catch (error) {
378
+ if (committedAuthority) {
379
+ this.logger("owner replacement post-commit finalization failed", {error: error instanceof Error ? error.message : String(error), replacementId: prepared.replacementId})
380
+ throw error
381
+ }
382
+ await this.closeServer(this.controlServer)
383
+ await this.closeServer(this.proxyServer)
384
+ if (finalControlPublished) await this.removeControlSocket()
385
+ else if (stagingControlPath) await fs.rm(stagingControlPath, {force: true})
386
+ let abortError
387
+
388
+ if (listenersYielded && incumbentControl) {
389
+ try {
390
+ await incumbentControl.request({command: "abort-owner-listener-handoff", replacementId: prepared.replacementId})
391
+ } catch (failure) {
392
+ abortError = failure instanceof Error ? failure : new Error(String(failure))
393
+ }
394
+ }
395
+ if (legacyBridge && !legacyBridge.boundaryCrossed) {
396
+ try {
397
+ await this.guardian.abandonLegacyUpgrade()
398
+ } catch (failure) {
399
+ abortError = failure instanceof Error ? failure : new Error(String(failure))
400
+ }
401
+ } else {
402
+ this.guardian.disconnect()
403
+ }
404
+ incumbentControl?.close()
405
+ if (legacyBridge?.boundaryCrossed) {
406
+ this.logger("legacy disruptive owner replacement failed after incumbent exit", {
407
+ error: error instanceof Error ? error.message : String(error),
408
+ recoveryStatePath: this.statePath,
409
+ replacementId: prepared.replacementId
410
+ })
411
+ }
412
+ if (abortError) throw new AggregateError([error, abortError], `Owner replacement failed and recovery cleanup failed: ${abortError.message}`, {cause: error})
413
+ throw error
414
+ } finally {
415
+ if (!retainIncumbentControl) incumbentControl?.close()
416
+ }
417
+ }
418
+
419
+ /**
420
+ * Applies authenticated live-connection state from the retired listener owner.
421
+ * @param {Record<string, JsonValue>} event - Incumbent control event.
422
+ * @param {{close: () => void}} session - Exact incumbent session.
423
+ */
424
+ handleIncumbentListenerEvent(event, session) {
425
+ if (event.event !== "owner-connection-state") return
426
+ const releaseId = stringOrUndefined(event.releaseId)
427
+ const connections = event.connections
428
+
429
+ if (!releaseId || !connections || typeof connections !== "object" || Array.isArray(connections)) {
430
+ throw new Error("Incumbent listener sent invalid connection state")
431
+ }
432
+ const release = this.releases.get(releaseId)
433
+
434
+ if (!release) throw new Error(`Incumbent listener reported unknown release ${releaseId}`)
435
+ release.setTransferredConnections({
436
+ http: requiredNonNegativeInteger(connections.http, "connections.http"),
437
+ websocket: requiredNonNegativeInteger(connections.websocket, "connections.websocket")
438
+ })
439
+ if (this.incumbentListenerControl === session && ![...this.releases.values()].some((candidate) => candidate.hasTransferredConnections())) {
440
+ this.incumbentListenerControl = undefined
441
+ session.close()
442
+ }
443
+ }
444
+
445
+ /**
446
+ * Authenticates and prepares the one-time disruptive bridge for a genuine pre-split guardian.
447
+ * @param {{error: Error | string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
448
+ * @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
449
+ */
450
+ async prepareLegacyOwnerReplacement({error, persisted, persistedAuthority}) {
451
+ const diagnostic = error instanceof Error ? error.message : String(error)
452
+ const legacyProcessKey = legacyGuardianKeys(persisted)[0]
453
+
454
+ if (!isLegacyGuardianPrepareDiagnostic(diagnostic)) throw error
455
+ if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state", {cause: error})
456
+ if (diagnostic !== "Unknown guardian command: prepare-owner-replacement") {
457
+ try {
458
+ await this.guardian?.request({
459
+ authority: persistedAuthority,
460
+ command: "prepare-owner-replacement",
461
+ key: legacyProcessKey,
462
+ nextAuthority: this.ownerAuthority()
463
+ })
464
+ throw new Error("Legacy guardian protocol probe unexpectedly accepted owner replacement")
465
+ } catch (probeError) {
466
+ if (!(probeError instanceof Error) || probeError.message !== "Unknown guardian command: prepare-owner-replacement") {
467
+ throw new Error("Guardian does not match the authenticated pre-split replacement protocol signature", {cause: probeError})
468
+ }
469
+ }
470
+ }
471
+ if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
472
+ throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
473
+ }
474
+ if (!this.legacyIncumbentPid) throw new Error("The authenticated pre-split guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
475
+ if (!this.guardianIdentity?.pid) throw new Error("The authenticated pre-split guardian state is missing its exact guardian PID")
476
+ const legacyGuardian = this.guardian
477
+
478
+ if (!legacyGuardian) throw new Error("Legacy disruptive replacement is missing its authenticated guardian connection")
479
+
480
+ await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
481
+ const incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
482
+ const incumbentControl = await openControlSession(persisted.control.path)
483
+ let upgraded
484
+
485
+ try {
486
+ const incumbentStatus = await incumbentControl.request({command: "status"})
487
+
488
+ assertLegacyIncumbentStatus(incumbentStatus, persisted)
489
+ const ownerState = {
490
+ authority: persistedAuthority,
491
+ config: this.config,
492
+ releaseConfigs: Object.fromEntries(persisted.releases.map((release) => [release.releaseId, this.config])),
493
+ snapshot: persisted
494
+ }
495
+ const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
496
+ socketPath: `${this.statePath}.split3-guardian.sock`,
497
+ token: crypto.randomBytes(32).toString("hex")
498
+ })
499
+
500
+ await assertPathAbsent(upgradedIdentity.socketPath, "Legacy upgrade guardian socket")
501
+ upgraded = await legacyGuardian.upgradeLegacyGuardian({ownerState, ...upgradedIdentity})
502
+ upgradedIdentity.pid = upgraded.pid
503
+ legacyGuardian.disconnect()
504
+ this.guardian = upgraded
505
+ this.guardianIdentity = upgradedIdentity
506
+ const prepared = await upgraded.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
507
+ const recoverySnapshot = /** @type {OwnerRecoverySnapshot} */ ({
508
+ ...persisted,
509
+ recovery: {...persisted.recovery, guardian: upgradedIdentity}
510
+ })
511
+
512
+ this.logger("legacy owner replacement bridge prepared", {
513
+ guardianPid: upgradedIdentity.pid ?? null,
514
+ incumbentPid: this.legacyIncumbentPid,
515
+ replacementId: prepared.replacementId
516
+ })
517
+ return {boundaryCrossed: false, incumbentControl, incumbentStartTime, prepared, recoverySnapshot}
518
+ } catch (upgradeError) {
519
+ incumbentControl.close()
520
+ if (upgraded) await upgraded.abandonLegacyUpgrade()
521
+ throw upgradeError
522
+ }
523
+ }
524
+
525
+ /**
526
+ * Crosses the explicitly disruptive legacy-only boundary after candidate reconstruction.
527
+ * @param {LegacyOwnerBridge} bridge - Prepared bridge.
528
+ */
529
+ async crossLegacyDisruptiveBoundary(bridge) {
530
+ if (!this.legacyIncumbentPid || !this.statePath) throw new Error("Legacy disruptive boundary is missing its exact incumbent identity")
531
+ const currentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, bridge.recoverySnapshot.control.path)
532
+
533
+ if (currentStartTime !== bridge.incumbentStartTime) throw new Error("Legacy incumbent PID identity changed before the disruptive boundary")
534
+ this.logger("legacy owner replacement disruptive boundary", {
535
+ disruptive: true,
536
+ incumbentPid: this.legacyIncumbentPid,
537
+ reason: "pre-split guardian and daemon lacked atomic replacement protocol"
538
+ })
539
+ process.kill(this.legacyIncumbentPid, "SIGKILL")
540
+ bridge.boundaryCrossed = true
541
+ await bridge.incumbentControl.closed()
542
+ await writeState(this.statePath, bridge.recoverySnapshot)
543
+ this.ownerTransition = /** @type {OwnerTransition} */ ({
544
+ disruptive: true,
545
+ mode: "legacy-first-upgrade",
546
+ reason: "pre-split guardian and daemon lacked atomic replacement protocol"
547
+ })
548
+ }
549
+
550
+ /**
551
+ * @param {string} key - Guardian key.
552
+ * @param {Parameters<GuardianClient["process"]>[1]} definition - Process definition.
553
+ * @returns {import("./managed-process.js").default} Guardian-backed managed process.
554
+ */
555
+ guardianProcess(key, definition) {
556
+ if (!this.guardian) throw new Error("Process guardian is not initialized")
557
+ return this.guardian.process(key, definition)
558
+ }
559
+
560
+ /** @param {import("./managed-process.js").default} processInstance - Guardian-backed process. */
561
+ async recoverGuardianProcess(processInstance) {
562
+ if (!("recover" in processInstance) || typeof processInstance.recover !== "function") throw new Error(`Managed process ${processInstance.id} is not guardian-backed`)
563
+ await processInstance.recover()
564
+ }
565
+
566
+ /** Releases only resources created by a fenced startup loser. */
567
+ async abandonOwnerRecoveryAttempt() {
568
+ this.guardian?.disconnect()
569
+ await this.closeServer(this.proxyServer)
570
+ }
571
+
90
572
  /** @returns {Promise<void>} Exposes control commands and begins periodic state persistence. */
91
573
  async exposeControl() {
92
574
  if (this.stopping) throw new Error("Rollbridge is shutting down")
@@ -102,7 +584,9 @@ export default class RollbridgeDaemon {
102
584
  this.startStatePersistence()
103
585
  }
104
586
 
105
- /** @returns {Promise<void>} Starts the stable local proxy. */
587
+ /**
588
+ * @returns {Promise<void>} Starts the stable local proxy.
589
+ */
106
590
  async startProxy() {
107
591
  const server = http.createServer((request, response) => this.proxyHttp(request, response))
108
592
 
@@ -111,7 +595,7 @@ export default class RollbridgeDaemon {
111
595
 
112
596
  await new Promise((resolve, reject) => {
113
597
  server.once("error", reject)
114
- server.listen(this.config.proxy.port, this.config.proxy.host, () => {
598
+ server.listen({host: this.config.proxy.host, port: this.config.proxy.port}, () => {
115
599
  const address = server.address()
116
600
  this.proxyPort = address && typeof address === "object" ? address.port : this.config.proxy.port
117
601
  this.logger("proxy listening", {host: this.config.proxy.host, port: this.proxyPort})
@@ -120,35 +604,46 @@ export default class RollbridgeDaemon {
120
604
  })
121
605
  }
122
606
 
123
- /** @returns {Promise<void>} Starts the control socket. */
124
- async startControlServer() {
607
+ /**
608
+ * @param {string} [socketPath] - Socket path to bind.
609
+ * @param {boolean} [applyMetadata] - Whether to apply configured mode and ownership.
610
+ * @returns {Promise<void>} Starts the control socket.
611
+ */
612
+ async startControlServer(socketPath = this.config.control.path, applyMetadata = true) {
125
613
  const server = net.createServer((socket) => this.handleControlSocket(socket))
126
614
 
127
615
  this.controlServer = server
128
- await this.prepareControlSocketPath()
616
+ await this.prepareControlSocketPath(socketPath)
129
617
 
130
618
  await new Promise((resolve, reject) => {
131
619
  server.once("error", reject)
132
- server.listen(this.config.control.path, () => {
620
+ server.listen(socketPath, () => {
133
621
  this.controlSocketOwned = true
134
- this.logger("control socket listening", {path: this.config.control.path})
622
+ this.boundControlPath = socketPath
623
+ this.logger("control socket listening", {path: socketPath})
135
624
  resolve(undefined)
136
625
  })
137
626
  })
138
627
 
139
- if (this.config.control.mode !== undefined) {
140
- await fs.chmod(this.config.control.path, this.config.control.mode)
141
- }
628
+ if (applyMetadata) await this.applyControlSocketMetadata(socketPath)
629
+ }
142
630
 
143
- await this.applyControlSocketOwnership()
631
+ /**
632
+ * @param {string} socketPath - Bound socket path.
633
+ * @returns {Promise<void>} Metadata application completion.
634
+ */
635
+ async applyControlSocketMetadata(socketPath) {
636
+ if (this.config.control.mode !== undefined) await fs.chmod(socketPath, this.config.control.mode)
637
+ await this.applyControlSocketOwnership(socketPath)
144
638
  }
145
639
 
146
640
  /**
147
641
  * Applies control.owner/control.group to the bound socket via chown, resolving names to ids.
642
+ * @param {string} [socketPath] - Bound socket path.
148
643
  * @returns {Promise<void>} Resolves once ownership is applied (no-op when neither is set).
149
644
  */
150
- async applyControlSocketOwnership() {
151
- const {group, owner, path: socketPath} = this.config.control
645
+ async applyControlSocketOwnership(socketPath = this.config.control.path) {
646
+ const {group, owner} = this.config.control
152
647
 
153
648
  if (owner === undefined && group === undefined) return
154
649
 
@@ -165,16 +660,19 @@ export default class RollbridgeDaemon {
165
660
  }
166
661
  }
167
662
 
168
- /** @returns {Promise<void>} Removes a stale Unix socket before binding, or fails clearly when a daemon is alive. */
169
- async prepareControlSocketPath() {
170
- const existing = await inspectControlSocket(this.config.control.path)
663
+ /**
664
+ * @param {string} [socketPath] - Socket path to inspect and prepare.
665
+ * @returns {Promise<void>} Removes a stale Unix socket before binding, or fails clearly when a daemon is alive.
666
+ */
667
+ async prepareControlSocketPath(socketPath = this.config.control.path) {
668
+ const existing = await inspectControlSocket(socketPath)
171
669
 
172
670
  if (existing.alive) {
173
- throw new Error(controlSocketBusyMessage(this.config.control.path, existing))
671
+ throw new Error(controlSocketBusyMessage(socketPath, existing))
174
672
  }
175
673
 
176
674
  try {
177
- await fs.rm(this.config.control.path, {force: true})
675
+ await fs.rm(socketPath, {force: true})
178
676
  } catch (error) {
179
677
  if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return
180
678
  throw error
@@ -322,12 +820,15 @@ export default class RollbridgeDaemon {
322
820
  const data = /** @type {Record<string, JsonValue>} */ (command)
323
821
  const commandName = data.command
324
822
 
823
+ if (!this.controlCommandsReady) throw new Error("Rollbridge replacement candidate is not committed and ready")
824
+ if (this.ownerRetired && commandName !== "status") throw new Error("Rollbridge owner authority has been transferred")
825
+
325
826
  if (commandName === "deploy") {
326
- return await this.deploy({
327
- releaseId: stringOrUndefined(data.releaseId),
328
- releasePath: requiredString(data.releasePath, "releasePath"),
329
- revision: stringOrUndefined(data.revision)
330
- })
827
+ return await this.executeOwnerMutation("deploy", async () => await this.deploy({
828
+ releaseId: stringOrUndefined(data.releaseId),
829
+ releasePath: requiredString(data.releasePath, "releasePath"),
830
+ revision: stringOrUndefined(data.revision)
831
+ }))
331
832
  }
332
833
 
333
834
  if (commandName === "status") {
@@ -339,19 +840,21 @@ export default class RollbridgeDaemon {
339
840
  }
340
841
 
341
842
  if (commandName === "stop") {
342
- await this.stopRelease(stringOrUndefined(data.releaseId))
343
- return this.status()
843
+ return await this.executeOwnerMutation("stop", async () => {
844
+ await this.stopRelease(stringOrUndefined(data.releaseId))
845
+ return this.status()
846
+ })
344
847
  }
345
848
 
346
849
  if (commandName === "restart") {
347
- return await this.restartProcesses({
850
+ return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
348
851
  policy: stringOrUndefined(data.policy),
349
852
  processId: stringOrUndefined(data.processId)
350
- })
853
+ }))
351
854
  }
352
855
 
353
856
  if (commandName === "rollback") {
354
- return await this.rollback({releaseId: stringOrUndefined(data.releaseId)})
857
+ return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
355
858
  }
356
859
 
357
860
  if (commandName === "shutdown") {
@@ -370,9 +873,149 @@ export default class RollbridgeDaemon {
370
873
  return {message: "owner retired"}
371
874
  }
372
875
 
876
+ if (commandName === "commit-owner-replacement") {
877
+ const replacementId = requiredString(data.replacementId, "replacementId")
878
+
879
+ if (!this.guardian) throw new Error("Atomic owner replacement requires the durable process guardian")
880
+ let committed = false
881
+
882
+ try {
883
+ await this.guardian.commitOwnerReplacement(replacementId)
884
+ committed = true
885
+ await this.retireCommittedOwner(controlSocket)
886
+ await this.guardian.finalizeOwnerReplacement(replacementId)
887
+ } finally {
888
+ if (committed) this.guardian.disconnect()
889
+ }
890
+ return {message: "owner replacement committed"}
891
+ }
892
+
893
+ if (commandName === "yield-owner-listeners") {
894
+ await this.yieldOwnerListeners({
895
+ completionSocket: controlSocket,
896
+ control: data.control === true,
897
+ proxy: data.proxy === true,
898
+ replacementId: requiredString(data.replacementId, "replacementId")
899
+ })
900
+ return {message: "owner listeners yielded"}
901
+ }
902
+
903
+ if (commandName === "abort-owner-listener-handoff") {
904
+ const replacementId = requiredString(data.replacementId, "replacementId")
905
+
906
+ if (!this.listenerHandoff || this.listenerHandoff.replacementId !== replacementId) throw new Error("Owner listener handoff is not the prepared transaction")
907
+ await this.resumeYieldedListeners()
908
+ return {message: "owner listeners resumed"}
909
+ }
910
+
373
911
  throw new Error(`Unknown command: ${String(commandName)}`)
374
912
  }
375
913
 
914
+ /**
915
+ * Runs one owner mutation under the guardian's prepare/commit exclusion fence.
916
+ * @param {string} operation - Mutation diagnostic name.
917
+ * @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
918
+ * @returns {Promise<Record<string, JsonValue>>} Operation result.
919
+ */
920
+ async executeOwnerMutation(operation, callback) {
921
+ if (!this.guardian) return await callback()
922
+ const mutationId = await this.guardian.beginOwnerMutation(operation)
923
+ let result
924
+ let operationError
925
+
926
+ try {
927
+ result = await callback()
928
+ } catch (error) {
929
+ operationError = error instanceof Error ? error : new Error(String(error))
930
+ }
931
+
932
+ const finalizationErrors = /** @type {Error[]} */ ([])
933
+
934
+ try {
935
+ await this.publishOwnerState()
936
+ } catch (error) {
937
+ finalizationErrors.push(error instanceof Error ? error : new Error(String(error)))
938
+ }
939
+ try {
940
+ await this.guardian.endOwnerMutation(mutationId)
941
+ } catch (error) {
942
+ finalizationErrors.push(error instanceof Error ? error : new Error(String(error)))
943
+ }
944
+
945
+ if (operationError || finalizationErrors.length > 0) {
946
+ const errors = [...(operationError ? [operationError] : []), ...finalizationErrors]
947
+
948
+ if (errors.length === 1) throw errors[0]
949
+ throw new AggregateError(errors, `Owner mutation ${operation} failed: ${errors.map((error) => error.message).join("; ")}`)
950
+ }
951
+
952
+ return /** @type {Record<string, JsonValue>} */ (result)
953
+ }
954
+
955
+ /**
956
+ * Stops accepting only the listener endpoints that the prepared candidate must bind.
957
+ * Existing proxy connections remain owned by this daemon until they close.
958
+ * @param {{completionSocket?: net.Socket, control: boolean, proxy: boolean, replacementId: string}} options - Handoff request.
959
+ */
960
+ async yieldOwnerListeners({completionSocket, control, proxy, replacementId}) {
961
+ if (!this.guardian) throw new Error("Owner listener handoff requires the durable process guardian")
962
+ if (this.listenerHandoff) throw new Error("Owner listeners are already yielded to a replacement candidate")
963
+ await this.guardian.validateOwnerReplacement(replacementId)
964
+ this.listenerHandoff = {control, proxy, replacementId}
965
+ this.listenerHandoffFailure = undefined
966
+
967
+ if (!completionSocket) throw new Error("Owner listener handoff requires its authenticated control session")
968
+ for (const release of this.releases.values()) release.pauseDrainForOwnerHandoff()
969
+ if (proxy) this.proxyClosePromise = this.closeServer(this.proxyServer)
970
+ if (control) {
971
+ this.controlClosePromise = this.closeServer(this.controlServer)
972
+ for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
973
+ await this.removeControlSocket()
974
+ }
975
+ for (const release of this.releases.values()) {
976
+ const publishConnections = () => {
977
+ if (completionSocket.destroyed) return
978
+ completionSocket.write(`${JSON.stringify({
979
+ connections: release.status().connections,
980
+ event: "owner-connection-state",
981
+ releaseId: release.releaseId
982
+ })}\n`)
983
+ }
984
+
985
+ publishConnections()
986
+ if (release.status().connectionCount > 0) release.once("drained", publishConnections)
987
+ }
988
+
989
+ const aborted = this.guardian.waitForEvent("replacement-aborted").then(async () => await this.resumeYieldedListeners())
990
+ const retired = this.guardian.waitForEvent("replacement-retired")
991
+
992
+ void Promise.race([aborted, retired]).catch((error) => {
993
+ if (this.ownerRetired) return
994
+ this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
995
+ this.logger("owner listener handoff recovery failed", {error: this.listenerHandoffFailure.message, replacementId})
996
+ })
997
+ }
998
+
999
+ /** Restores listener endpoints after a prepared candidate aborts before commit. */
1000
+ async resumeYieldedListeners() {
1001
+ const handoff = this.listenerHandoff
1002
+
1003
+ if (!handoff || this.ownerRetired) return
1004
+ for (const release of this.releases.values()) {
1005
+ release.resumeDrainAfterOwnerHandoff()
1006
+ if (release.state === "draining") void this.drainAndPrune(release, release.config)
1007
+ }
1008
+ if (handoff.proxy) {
1009
+ await this.startProxy()
1010
+ }
1011
+ if (handoff.control) {
1012
+ await this.startControlServer()
1013
+ }
1014
+ this.listenerHandoff = undefined
1015
+ this.listenerHandoffFailure = undefined
1016
+ this.logger("owner listeners resumed", {replacementId: handoff.replacementId})
1017
+ }
1018
+
376
1019
  /**
377
1020
  * Starts a new release, switches traffic, and drains the previous release.
378
1021
  * @param {DeployArgs} args - Deploy args.
@@ -389,6 +1032,8 @@ export default class RollbridgeDaemon {
389
1032
  const release = new ReleaseGroup({
390
1033
  config: nextConfig,
391
1034
  logger: this.logger,
1035
+ portReservations: this.portReservations,
1036
+ ...(this.guardian ? {processFactory: (key, definition) => this.guardianProcess(key, definition)} : {}),
392
1037
  releaseId: newReleaseId,
393
1038
  releasePath,
394
1039
  revision,
@@ -423,20 +1068,51 @@ export default class RollbridgeDaemon {
423
1068
  this.logger("traffic switched", {previousReleaseId: previousRelease ? previousRelease.releaseId : null, releaseId: release.releaseId})
424
1069
 
425
1070
  this.refreshServiceDefinitions(release)
426
- await this.replaceSingletons(release)
1071
+ let retirementFailure
427
1072
 
428
1073
  if (previousRelease) {
429
- void this.drainAndPrune(previousRelease, nextConfig)
1074
+ try {
1075
+ const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : nextConfig
1076
+
1077
+ await previousRelease.beginRetirement(retirementConfig)
1078
+ void this.drainAndPrune(previousRelease, retirementConfig)
1079
+ } catch (error) {
1080
+ retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
1081
+ this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
1082
+ }
430
1083
  }
431
1084
 
432
- this.persistState()
1085
+ await this.replaceSingletons(release)
1086
+
1087
+ await this.persistState()
1088
+ await this.publishOwnerState()
433
1089
 
434
1090
  return {
435
1091
  activeReleaseId: release.releaseId,
436
- previousReleaseId: previousRelease ? previousRelease.releaseId : null
1092
+ previousReleaseId: previousRelease ? previousRelease.releaseId : null,
1093
+ ...(retirementFailure && previousRelease ? {retirement: {error: retirementFailure, releaseId: previousRelease.releaseId, status: "quiescence_failed"}} : {})
437
1094
  }
438
1095
  }
439
1096
 
1097
+ /**
1098
+ * Relinquishes only daemon authority/listeners after the guardian has committed a
1099
+ * prepared replacement. Guardian-owned processes and drains are never stopped.
1100
+ * @param {net.Socket | undefined} completionSocket - Commit response connection.
1101
+ */
1102
+ async retireCommittedOwner(completionSocket) {
1103
+ this.ownerRetired = true
1104
+ if (this.persistTimer) clearInterval(this.persistTimer)
1105
+ this.persistTimer = undefined
1106
+ this.persistenceEnabled = false
1107
+ if (this.pendingWrite) await this.pendingWrite
1108
+ this.stateCleanupEnabled = false
1109
+ this.controlClosePromise = this.closeServer(this.controlServer)
1110
+ for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
1111
+ await this.removeControlSocket()
1112
+ void this.closeServer(this.proxyServer)
1113
+ this.logger("owner authority transferred", {activeReleaseId: this.activeRelease?.releaseId ?? null})
1114
+ }
1115
+
440
1116
  /**
441
1117
  * Rejects config changes that require rebinding daemon-owned resources or changing process topology.
442
1118
  * @param {import("./config.js").RollbridgeConfig} nextConfig - Freshly loaded config.
@@ -449,6 +1125,7 @@ export default class RollbridgeDaemon {
449
1125
  if (nextConfig.application !== this.config.application) restartRequired.push("application")
450
1126
  if (!isDeepStrictEqual(nextConfig.control, this.config.control)) restartRequired.push("control")
451
1127
  if (nextConfig.statePath !== this.config.statePath) restartRequired.push("statePath")
1128
+ if (!isDeepStrictEqual(nextConfig.ownerRecovery, this.config.ownerRecovery)) restartRequired.push("ownerRecovery")
452
1129
 
453
1130
  if (nextConfig.proxy.host !== this.config.proxy.host) restartRequired.push("proxy.host")
454
1131
  if (nextConfig.proxy.port !== this.config.proxy.port) restartRequired.push("proxy.port")
@@ -536,7 +1213,7 @@ export default class RollbridgeDaemon {
536
1213
  if (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff") continue
537
1214
  if (this.services.has(processConfig.id)) continue
538
1215
 
539
- const service = release.buildProcess(processConfig, {shouldRestart: () => !this.stopping})
1216
+ const service = release.buildProcess(processConfig, {guardianKey: `service:${processConfig.id}`, shouldRestart: () => !this.stopping})
540
1217
 
541
1218
  this.services.set(processConfig.id, service)
542
1219
 
@@ -546,6 +1223,7 @@ export default class RollbridgeDaemon {
546
1223
 
547
1224
  try {
548
1225
  await service.start("deploy")
1226
+ release.transferPortReservation(processConfig.id)
549
1227
  startedServices.push(processConfig.id)
550
1228
  } catch (error) {
551
1229
  this.services.delete(processConfig.id)
@@ -570,6 +1248,9 @@ export default class RollbridgeDaemon {
570
1248
 
571
1249
  await service.stop()
572
1250
  this.services.delete(serviceId)
1251
+ const port = this.servicePorts[serviceId]
1252
+
1253
+ if (port !== undefined) this.portReservations.delete(port)
573
1254
  delete this.servicePorts[serviceId]
574
1255
  }
575
1256
  }
@@ -610,7 +1291,7 @@ export default class RollbridgeDaemon {
610
1291
  if (this.stopping) throw new Error("Rollbridge is shutting down")
611
1292
  }
612
1293
 
613
- const singleton = release.buildProcess(processConfig)
1294
+ const singleton = release.buildProcess(processConfig, {guardianKey: `singleton:${release.releaseId}:${processConfig.id}`})
614
1295
 
615
1296
  this.singletons.set(processConfig.id, singleton)
616
1297
  await singleton.start("deploy")
@@ -726,8 +1407,10 @@ export default class RollbridgeDaemon {
726
1407
  } catch (error) {
727
1408
  this.logger("release drain failed", {error: error instanceof Error ? error.message : String(error), releaseId: release.releaseId})
728
1409
  } finally {
729
- this.pruneStoppedReleases()
730
- this.persistState()
1410
+ if (!release.isDrainPausedForOwnerHandoff()) {
1411
+ this.pruneStoppedReleases()
1412
+ this.persistState()
1413
+ }
731
1414
  }
732
1415
  }
733
1416
 
@@ -753,25 +1436,40 @@ export default class RollbridgeDaemon {
753
1436
 
754
1437
  /**
755
1438
  * Persists a state snapshot (status plus recent events) to statePath, atomically and
756
- * fire-and-forget. A failed write is logged but never blocks daemon operation.
757
- * @returns {void}
1439
+ * fire-and-forget unless the caller awaits the returned write. A failed write is logged.
1440
+ * @param {{allowStopping?: boolean, throwOnError?: boolean}} [options] - Write behavior.
1441
+ * @returns {Promise<void> | undefined} The queued write, or undefined when persistence is disabled.
758
1442
  */
759
- persistState() {
760
- if (!this.statePath || !this.persistenceEnabled || this.stopping) return
1443
+ persistState({allowStopping = false, throwOnError = false} = {}) {
1444
+ if (!this.statePath || !this.persistenceEnabled || (this.stopping && !allowStopping)) return
761
1445
 
762
1446
  const statePath = this.statePath
763
1447
  const status = /** @type {Record<string, JsonValue>} */ (secretSafeStateValue(this.status()))
764
1448
  const events = secretSafeStateValue(this.eventLog.recent())
765
- const snapshot = {...status, events, persistedAt: new Date().toISOString()}
1449
+ const snapshot = {
1450
+ ...status,
1451
+ events,
1452
+ persistedAt: new Date().toISOString(),
1453
+ ...(this.guardianIdentity ? {recovery: {
1454
+ configDigest: this.ownerRecoveryConfigDigest(),
1455
+ format: 1,
1456
+ guardian: this.guardianIdentity,
1457
+ reconnectGraceMs: this.config.ownerRecovery?.reconnectGraceMs
1458
+ }} : {})
1459
+ }
766
1460
 
767
1461
  // Serialize writes (and track the tail) so shutdown can wait for an in-flight write before
768
1462
  // clearing the file — otherwise a write started before shutdown could recreate it afterward.
769
1463
  this.pendingWrite = Promise.resolve(this.pendingWrite)
770
1464
  .catch(() => {})
771
1465
  .then(() => writeState(statePath, snapshot))
1466
+ .then(() => this.publishOwnerState())
772
1467
  .catch((error) => {
773
1468
  this.logger("state persist failed", {error: error instanceof Error ? error.message : String(error)})
1469
+ if (throwOnError) throw error
774
1470
  })
1471
+
1472
+ return this.pendingWrite
775
1473
  }
776
1474
 
777
1475
  /**
@@ -835,25 +1533,35 @@ export default class RollbridgeDaemon {
835
1533
  clearInterval(this.persistTimer)
836
1534
  this.persistTimer = undefined
837
1535
  }
838
- this.persistenceEnabled = false
839
1536
  if (this.pendingWrite) await this.pendingWrite
840
1537
  this.stateCleanupEnabled = false
841
1538
  this.controlClosePromise = this.closeServer(this.controlServer)
842
1539
  for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
1540
+ if (this.activeRelease) {
1541
+ await this.activeRelease.beginRetirement(this.activeRelease.config)
1542
+ this.activeRelease = undefined
1543
+ }
843
1544
  await Promise.all([
844
1545
  ...[...this.services.values()].map((processInstance) => processInstance.quiesce()),
845
1546
  ...[...this.singletons.values()].map((processInstance) => processInstance.quiesce()),
846
1547
  ...[...this.startingReleases].map((release) => release.quiesce()),
847
1548
  ...[...this.releases.values()].map((release) => release.quiesce())
848
1549
  ])
1550
+ await this.persistState({allowStopping: true, throwOnError: true})
1551
+ this.persistenceEnabled = false
849
1552
  await this.removeControlSocket()
850
1553
  void this.closeServer(this.proxyServer)
851
- void Promise.allSettled([
852
- ...[...this.services.values()].map((processInstance) => processInstance.stop()),
853
- ...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
854
- ...[...this.startingReleases].map((release) => release.stop()),
855
- ...[...this.releases.values()].map((release) => release.stop())
856
- ])
1554
+ if (this.guardian) {
1555
+ await this.guardian.retireOwner()
1556
+ this.guardian.disconnect()
1557
+ } else {
1558
+ void Promise.allSettled([
1559
+ ...[...this.services.values()].map((processInstance) => processInstance.stop()),
1560
+ ...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
1561
+ ...[...this.startingReleases].map((release) => release.stop()),
1562
+ ...[...this.releases.values()].map((release) => release.stop())
1563
+ ])
1564
+ }
857
1565
  this.logger("external owner retired", {attestation, status: "draining"})
858
1566
  }
859
1567
 
@@ -865,6 +1573,9 @@ export default class RollbridgeDaemon {
865
1573
  this.stopping = true
866
1574
  const cleanupErrors = /** @type {Error[]} */ ([])
867
1575
 
1576
+ this.incumbentListenerControl?.close()
1577
+ this.incumbentListenerControl = undefined
1578
+
868
1579
  // server.close() stops new connections synchronously. Unlink immediately afterward so a
869
1580
  // replacement can bind as soon as cleanup completes; existing connections remain usable for
870
1581
  // the shutdown completion/error response.
@@ -889,6 +1600,8 @@ export default class RollbridgeDaemon {
889
1600
  ])
890
1601
  const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
891
1602
  const stopResults = [...dependentStopResults, ...serviceStopResults]
1603
+ const guardian = this.guardian
1604
+ if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
892
1605
  await captureShutdownError(cleanupErrors, "proxy server close", () => this.closeServer(this.proxyServer))
893
1606
 
894
1607
  // Wait for any in-flight write first so it can't recreate or overwrite the final state (no
@@ -919,8 +1632,9 @@ export default class RollbridgeDaemon {
919
1632
  async removeControlSocket() {
920
1633
  if (!this.controlSocketOwned) return
921
1634
 
922
- await fs.rm(this.config.control.path, {force: true})
1635
+ await fs.rm(this.boundControlPath || this.config.control.path, {force: true})
923
1636
  this.controlSocketOwned = false
1637
+ this.boundControlPath = undefined
924
1638
  }
925
1639
 
926
1640
  /**
@@ -950,13 +1664,19 @@ export default class RollbridgeDaemon {
950
1664
  application: this.config.application,
951
1665
  bootstrap: this.bootstrap ? {...this.bootstrap} : undefined,
952
1666
  control: {...this.config.control},
1667
+ daemonPid: process.pid,
953
1668
  daemonRuntime: this.runtime ? {...this.runtime} : undefined,
1669
+ ownerRecovery: this.guardian ? {configDigest: this.ownerRecoveryConfigDigest()} : undefined,
1670
+ ownerTransition: this.ownerTransition ? {...this.ownerTransition} : undefined,
954
1671
  orphans: [...this.orphans],
955
1672
  proxy: {
956
1673
  host: this.config.proxy.host,
957
1674
  port: this.proxyPort ?? this.config.proxy.port,
958
1675
  upstreamHost: this.config.proxy.upstreamHost
959
1676
  },
1677
+ releaseReferences: [...this.releases.values()]
1678
+ .filter((release) => release.state === "active" || release.state === "draining")
1679
+ .map((release) => ({releaseId: release.releaseId, releasePath: release.releasePath})),
960
1680
  releases: [...this.releases.values()].map((release) => release.status()),
961
1681
  services: [...this.services.entries()].map(([id, processInstance]) => ({
962
1682
  id,
@@ -970,6 +1690,162 @@ export default class RollbridgeDaemon {
970
1690
  }
971
1691
  }
972
1692
 
1693
+ /**
1694
+ * @param {import("./config.js").RollbridgeConfig} config - Normalized config.
1695
+ * @returns {string} Stable config authority digest.
1696
+ */
1697
+ export function ownerConfigDigest(config) {
1698
+ return crypto.createHash("sha256").update(JSON.stringify(config)).digest("hex")
1699
+ }
1700
+
1701
+ /**
1702
+ * Accepts only the two exact authenticated diagnostics emitted by pre-split guardians.
1703
+ * @param {string} diagnostic - Guardian response diagnostic.
1704
+ * @returns {boolean} Whether the response can proceed to full legacy process attestation.
1705
+ */
1706
+ export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
1707
+ return diagnostic === "Guardian prepare-owner-replacement requires a process key" ||
1708
+ diagnostic === "Unknown guardian command: prepare-owner-replacement"
1709
+ }
1710
+
1711
+ /**
1712
+ * @param {OwnerRecoverySnapshot} snapshot - Durable pre-split snapshot.
1713
+ * @returns {string[]} Exact guardian registration keys present in the snapshot.
1714
+ */
1715
+ function legacyGuardianKeys(snapshot) {
1716
+ const keys = []
1717
+
1718
+ for (const release of snapshot.releases) {
1719
+ for (const processStatus of release.processes) keys.push(`release:${release.releaseId}:${processStatus.id}`)
1720
+ }
1721
+ for (const service of snapshot.services) keys.push(`service:${service.id}`)
1722
+ if (snapshot.activeReleaseId) {
1723
+ for (const singleton of snapshot.singletons) keys.push(`singleton:${snapshot.activeReleaseId}:${singleton.id}`)
1724
+ }
1725
+ return keys
1726
+ }
1727
+
1728
+ /**
1729
+ * Verifies the exact authenticated guardian process and socket without scanning other PIDs.
1730
+ * @param {number} pid - Persisted guardian PID.
1731
+ * @param {string} socketPath - Persisted guardian socket.
1732
+ */
1733
+ async function verifyLegacyGuardianProcess(pid, socketPath) {
1734
+ const args = await processArguments(pid, "legacy guardian")
1735
+ const script = args.find((argument) => argument.endsWith("process-guardian.js"))
1736
+
1737
+ if (!script || !args.includes(socketPath)) throw new Error(`Persisted guardian PID ${pid} does not match the pre-split guardian command and socket`)
1738
+ await verifyProcessUser(pid, "legacy guardian")
1739
+ await verifyUnixSocketOwner(pid, socketPath, "legacy guardian")
1740
+ }
1741
+
1742
+ /**
1743
+ * Verifies the exact incumbent daemon process and control socket without scanning other PIDs.
1744
+ * @param {number} pid - Incumbent PID from the daemon PID file.
1745
+ * @param {string | undefined} configPath - Exact daemon config path.
1746
+ * @param {string} socketPath - Persisted control socket.
1747
+ * @returns {Promise<string>} Linux process start-time identity.
1748
+ */
1749
+ async function verifyLegacyDaemonProcess(pid, configPath, socketPath) {
1750
+ if (!configPath) throw new Error("Legacy disruptive replacement requires the daemon's exact config path")
1751
+ const args = await processArguments(pid, "legacy daemon")
1752
+ const daemonIndex = args.indexOf("daemon")
1753
+ const configIndex = args.indexOf("--config")
1754
+
1755
+ if (daemonIndex < 0 || configIndex < 0 || args[configIndex + 1] !== configPath) {
1756
+ throw new Error(`Daemon PID ${pid} does not match the exact pre-split daemon config command`)
1757
+ }
1758
+ await verifyProcessUser(pid, "legacy daemon")
1759
+ await verifyUnixSocketOwner(pid, socketPath, "legacy daemon")
1760
+ const stat = await fs.readFile(`/proc/${pid}/stat`, "utf8")
1761
+ const closingParenthesis = stat.lastIndexOf(")")
1762
+ const fields = stat.slice(closingParenthesis + 2).split(" ")
1763
+ const startTime = fields[19]
1764
+
1765
+ if (!startTime) throw new Error(`Could not attest start time for legacy daemon PID ${pid}`)
1766
+ return startTime
1767
+ }
1768
+
1769
+ /**
1770
+ * @param {number} pid - Exact process PID.
1771
+ * @param {string} label - Diagnostic label.
1772
+ * @returns {Promise<string[]>} NUL-delimited argv.
1773
+ */
1774
+ async function processArguments(pid, label) {
1775
+ try {
1776
+ return (await fs.readFile(`/proc/${pid}/cmdline`, "utf8")).split("\0").filter(Boolean)
1777
+ } catch (error) {
1778
+ throw new Error(`Could not verify exact ${label} PID ${pid}`, {cause: error})
1779
+ }
1780
+ }
1781
+
1782
+ /**
1783
+ * @param {number} pid - Exact process PID.
1784
+ * @param {string} label - Diagnostic label.
1785
+ */
1786
+ async function verifyProcessUser(pid, label) {
1787
+ if (typeof process.getuid !== "function") throw new Error(`Cannot verify ${label} ownership on this platform`)
1788
+ const stats = await fs.stat(`/proc/${pid}`)
1789
+
1790
+ if (stats.uid !== process.getuid()) throw new Error(`Refusing ${label} PID ${pid} owned by another user`)
1791
+ }
1792
+
1793
+ /**
1794
+ * @param {number} pid - Exact process PID.
1795
+ * @param {string} socketPath - Exact Unix socket pathname.
1796
+ * @param {string} label - Diagnostic label.
1797
+ */
1798
+ async function verifyUnixSocketOwner(pid, socketPath, label) {
1799
+ const rows = (await fs.readFile("/proc/net/unix", "utf8")).split("\n").map((line) => line.trim().split(/\s+/))
1800
+ const row = rows.find((columns) => columns[7] === socketPath)
1801
+
1802
+ if (!row?.[6]) throw new Error(`Could not find exact ${label} socket ${socketPath} in the kernel socket table`)
1803
+ const expected = `socket:[${row[6]}]`
1804
+ const descriptors = await fs.readdir(`/proc/${pid}/fd`)
1805
+ let owned = false
1806
+
1807
+ for (const descriptor of descriptors) {
1808
+ try {
1809
+ if (await fs.readlink(`/proc/${pid}/fd/${descriptor}`) === expected) {
1810
+ owned = true
1811
+ break
1812
+ }
1813
+ } catch (error) {
1814
+ if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
1815
+ }
1816
+ }
1817
+ if (!owned) throw new Error(`Exact ${label} PID ${pid} does not own socket ${socketPath}`)
1818
+ }
1819
+
1820
+ /**
1821
+ * @param {Record<string, JsonValue>} status - Authenticated incumbent status response.
1822
+ * @param {OwnerRecoverySnapshot} persisted - Durable expected identity.
1823
+ */
1824
+ function assertLegacyIncumbentStatus(status, persisted) {
1825
+ const control = status.control
1826
+ const runtime = status.daemonRuntime
1827
+ const ownerRecovery = status.ownerRecovery
1828
+
1829
+ if (status.application !== persisted.application || !control || typeof control !== "object" || Array.isArray(control) || control.path !== persisted.control.path ||
1830
+ !runtime || typeof runtime !== "object" || Array.isArray(runtime) || runtime.digest !== persisted.daemonRuntime?.digest ||
1831
+ !ownerRecovery || typeof ownerRecovery !== "object" || Array.isArray(ownerRecovery) || ownerRecovery.configDigest !== persisted.recovery.configDigest) {
1832
+ throw new Error("Responsive incumbent does not match the exact persisted pre-split daemon authority")
1833
+ }
1834
+ }
1835
+
1836
+ /**
1837
+ * @param {string} candidatePath - Path that must not preexist.
1838
+ * @param {string} label - Diagnostic label.
1839
+ */
1840
+ async function assertPathAbsent(candidatePath, label) {
1841
+ try {
1842
+ await fs.lstat(candidatePath)
1843
+ throw new Error(`${label} ${candidatePath} already exists; refusing legacy upgrade`)
1844
+ } catch (error) {
1845
+ if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
1846
+ }
1847
+ }
1848
+
973
1849
  /**
974
1850
  * @param {JsonValue} value - Value.
975
1851
  * @returns {string | undefined} String value.
@@ -989,7 +1865,7 @@ function isShutdownControlLine(line) {
989
1865
  try {
990
1866
  const command = JSON.parse(line)
991
1867
 
992
- return Boolean(command && typeof command === "object" && ["retire-owner", "shutdown"].includes(command.command))
1868
+ return Boolean(command && typeof command === "object" && ["commit-owner-replacement", "retire-owner", "shutdown"].includes(command.command))
993
1869
  } catch {
994
1870
  return false
995
1871
  }
@@ -1049,6 +1925,16 @@ function requiredString(value, key) {
1049
1925
  return value
1050
1926
  }
1051
1927
 
1928
+ /**
1929
+ * @param {JsonValue} value - Value.
1930
+ * @param {string} key - Key.
1931
+ * @returns {number} Non-negative integer.
1932
+ */
1933
+ function requiredNonNegativeInteger(value, key) {
1934
+ if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) throw new Error(`${key} must be a non-negative integer`)
1935
+ return value
1936
+ }
1937
+
1052
1938
  /**
1053
1939
  * @typedef {{releaseId: string, state: string, stoppedAt: string | undefined}} PrunableRelease
1054
1940
  */