rollbridge 0.1.39 → 0.1.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +12 -1
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +16 -2
- package/changelog.d/20260830055159-guardian-daemon-restart.md +2 -0
- package/docs/cli.md +23 -9
- package/docs/config.md +37 -13
- package/docs/logging.md +4 -3
- package/docs/troubleshooting.md +7 -5
- package/examples/tensorbuzz.com.js +5 -2
- package/package.json +1 -1
- package/src/cli.js +118 -24
- package/src/config.js +35 -8
- package/src/daemon.js +822 -152
- package/src/guardian-client.js +121 -16
- package/src/managed-process.js +117 -15
- package/src/process-guardian.js +734 -43
- package/src/release-group.js +45 -7
- package/test/completion.test.js +4 -2
- package/test/config-examples.test.js +1 -0
- package/test/config-validation.test.js +22 -0
- package/test/fixtures/guardian-recovery-owner.js +86 -0
- package/test/fixtures/pre-split3-process-guardian.js +14 -0
- package/test/guardian-client.test.js +1420 -62
- package/test/managed-process.test.js +163 -7
- package/test/owner-recovery.test.js +575 -73
- package/test/owner-replacement.test.js +525 -28
- package/test/release-group.test.js +19 -2
- package/test/release-runtime-retention.test.js +121 -4
- package/test/rollbridge.test.js +263 -13
- package/test/support/process.js +41 -0
package/src/daemon.js
CHANGED
|
@@ -4,6 +4,7 @@ import fs from "node:fs/promises"
|
|
|
4
4
|
import http from "node:http"
|
|
5
5
|
import net from "node:net"
|
|
6
6
|
import crypto from "node:crypto"
|
|
7
|
+
import path from "node:path"
|
|
7
8
|
import {isDeepStrictEqual} from "node:util"
|
|
8
9
|
import httpProxy from "http-proxy"
|
|
9
10
|
import {loadConfig} from "./config.js"
|
|
@@ -23,12 +24,12 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
23
24
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
24
25
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
25
26
|
* @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
26
|
-
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
27
|
-
* @typedef {{candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, configDigest: string, error?: string, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
28
|
-
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
27
|
+
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "restoring_previous" | "retiring_failed_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
28
|
+
* @typedef {{activationError?: string, activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, compensationError?: string, configDigest: string, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
29
|
+
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
29
30
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
30
|
-
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
31
|
-
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
31
|
+
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
32
|
+
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
32
33
|
* @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
33
34
|
*/
|
|
34
35
|
|
|
@@ -40,12 +41,14 @@ export default class RollbridgeDaemon {
|
|
|
40
41
|
* @param {string} [args.configPath] - Config file path to reload before deploys.
|
|
41
42
|
* @param {(message: string, data?: Record<string, JsonValue>) => void} [args.logger] - Logger.
|
|
42
43
|
* @param {number} [args.legacyIncumbentPid] - Exact incumbent PID supplied by ensure-daemon.
|
|
44
|
+
* @param {{args: string[], cwd: string, env: Record<string, string>, executable: string, logPath?: string, pidPath?: string, startupTimeoutMs: number}} [args.recoveryCommand] - Exact command for guardian-owned daemon recovery.
|
|
43
45
|
* @param {import("./daemon-runtime.js").DaemonRuntimeIdentity} [args.runtime] - Immutable daemon runtime identity.
|
|
44
46
|
*/
|
|
45
|
-
constructor({bootstrap, config, configPath, legacyIncumbentPid, logger, runtime}) {
|
|
47
|
+
constructor({bootstrap, config, configPath, legacyIncumbentPid, logger, recoveryCommand, runtime}) {
|
|
46
48
|
this.bootstrap = bootstrap ? {...bootstrap} : undefined
|
|
47
49
|
this.config = config
|
|
48
50
|
this.configPath = configPath
|
|
51
|
+
this.recoveryCommand = recoveryCommand
|
|
49
52
|
this.runtime = runtime
|
|
50
53
|
this.legacyIncumbentPid = legacyIncumbentPid
|
|
51
54
|
this.eventLog = new EventLog(EVENT_HISTORY_LIMIT)
|
|
@@ -62,6 +65,7 @@ export default class RollbridgeDaemon {
|
|
|
62
65
|
|
|
63
66
|
this.releases = /** @type {Map<string, ReleaseGroup>} */ (new Map())
|
|
64
67
|
this.services = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
|
|
68
|
+
this.serviceReleaseIds = /** @type {Map<string, string>} */ (new Map())
|
|
65
69
|
this.servicePorts = /** @type {Record<string, number>} */ ({})
|
|
66
70
|
this.portReservations = /** @type {Set<number>} */ (new Set())
|
|
67
71
|
this.singletons = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
|
|
@@ -72,6 +76,7 @@ export default class RollbridgeDaemon {
|
|
|
72
76
|
this.proxyServer = /** @type {http.Server | undefined} */ (undefined)
|
|
73
77
|
this.controlServer = /** @type {net.Server | undefined} */ (undefined)
|
|
74
78
|
this.controlSocketOwned = false
|
|
79
|
+
this.boundControlIdentity = /** @type {{dev: number, ino: number} | undefined} */ (undefined)
|
|
75
80
|
this.boundControlPath = /** @type {string | undefined} */ (undefined)
|
|
76
81
|
this.controlSockets = /** @type {Set<net.Socket>} */ (new Set())
|
|
77
82
|
this.proxyPort = /** @type {number | undefined} */ (undefined)
|
|
@@ -83,17 +88,26 @@ export default class RollbridgeDaemon {
|
|
|
83
88
|
this.stateCleanupEnabled = false
|
|
84
89
|
this.shutdownPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
85
90
|
this.retirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
91
|
+
this.committedRetirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
86
92
|
this.ownerRetired = false
|
|
87
93
|
this.controlClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
88
94
|
this.proxyClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
89
|
-
this.listenerHandoff = /** @type {{control: boolean, proxy: boolean, replacementId: string} | undefined} */ (undefined)
|
|
95
|
+
this.listenerHandoff = /** @type {{completionSocket: net.Socket, control: boolean, proxy: boolean, replacementId: string, retired: Promise<Record<string, JsonValue>>} | undefined} */ (undefined)
|
|
90
96
|
this.listenerHandoffFailure = /** @type {Error | undefined} */ (undefined)
|
|
91
97
|
this.incumbentListenerControl = /** @type {Awaited<ReturnType<typeof openControlSession>> | undefined} */ (undefined)
|
|
98
|
+
this.incumbentListenerSourceIds = /** @type {WeakMap<object, string>} */ (new WeakMap())
|
|
99
|
+
this.listenerConnectionSources = /** @type {Map<string, Map<string, {http: number, websocket: number}>>} */ (new Map())
|
|
100
|
+
this.listenerSourceId = crypto.randomUUID()
|
|
101
|
+
this.retiredConnectionPublication = Promise.resolve()
|
|
102
|
+
this.retiredReplacementId = /** @type {string | undefined} */ (undefined)
|
|
103
|
+
this.retiredListenerHandoff = /** @type {{releases: ReleaseGroup[], replacementId: string, replacementRetired: boolean, retired: Promise<Record<string, JsonValue>>} | undefined} */ (undefined)
|
|
92
104
|
this.ownerTransition = /** @type {OwnerTransition | undefined} */ (undefined)
|
|
93
105
|
this.controlCommandsReady = true
|
|
94
106
|
this.startingReleases = /** @type {Set<ReleaseGroup>} */ (new Set())
|
|
95
107
|
this.guardian = /** @type {GuardianClient | undefined} */ (undefined)
|
|
96
108
|
this.guardianIdentity = /** @type {{pid?: number, socketPath: string, token: string} | undefined} */ (undefined)
|
|
109
|
+
this.ownerReady = false
|
|
110
|
+
this.generationTransitionRecovery = false
|
|
97
111
|
// Still-alive managed processes left by a previous daemon (from statePath), captured at
|
|
98
112
|
// startup and surfaced in status(). The daemon cannot re-manage them, only report them.
|
|
99
113
|
this.orphans = /** @type {{id: string, pid: number, releaseId: string | null}[]} */ ([])
|
|
@@ -109,8 +123,67 @@ export default class RollbridgeDaemon {
|
|
|
109
123
|
async start({exposeControl = true, reportOrphans = true} = {}) {
|
|
110
124
|
if (this.config.ownerRecovery) await this.initializeOwnerRecovery()
|
|
111
125
|
else if (reportOrphans) await this.reportOrphans()
|
|
126
|
+
if (this.ownerRetired) return
|
|
127
|
+
const transition = this.generationTransition
|
|
128
|
+
const recoveredReplacementProxy = this.guardian ? await this.finalizeRecoveredOwnerReplacement() : false
|
|
129
|
+
|
|
130
|
+
if (this.guardian && transition && transition.phase !== "committed" && !transition.error) {
|
|
131
|
+
this.generationTransitionRecovery = true
|
|
132
|
+
try {
|
|
133
|
+
await this.executeOwnerMutation("recover generation transition", async () => {
|
|
134
|
+
if (!recoveredReplacementProxy) await this.startProxy()
|
|
135
|
+
if (this.ownerRetired) return {}
|
|
136
|
+
if (exposeControl) await this.exposeControl()
|
|
137
|
+
if (this.ownerRetired) return {}
|
|
138
|
+
await this.markOwnerReady()
|
|
139
|
+
try {
|
|
140
|
+
await this.resumeGenerationTransition()
|
|
141
|
+
} catch (error) {
|
|
142
|
+
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
|
|
143
|
+
}
|
|
144
|
+
return {}
|
|
145
|
+
})
|
|
146
|
+
} finally {
|
|
147
|
+
this.generationTransitionRecovery = false
|
|
148
|
+
}
|
|
149
|
+
if (!this.ownerRetired) this.logger("release generation transition recovery settled", {phase: transition.phase, releaseId: transition.candidateReleaseId})
|
|
150
|
+
return
|
|
151
|
+
}
|
|
152
|
+
if (!recoveredReplacementProxy) await this.startProxy()
|
|
153
|
+
if (!this.ownerRetired && exposeControl) await this.exposeControl()
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** @returns {Promise<boolean>} Whether startup bound a proxy for a pending committed replacement. */
|
|
157
|
+
async finalizeRecoveredOwnerReplacement() {
|
|
158
|
+
if (!this.guardian) return false
|
|
159
|
+
let status = await this.guardian.replacementStatus()
|
|
160
|
+
|
|
161
|
+
if (status.retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
162
|
+
if (!status.retirementPending) return false
|
|
163
|
+
if (!status.committedReplacementId) throw new Error("Guardian reports listener retirement without a committed replacement")
|
|
164
|
+
if (!status.retirementReady) {
|
|
165
|
+
const settled = Promise.race([
|
|
166
|
+
this.guardian.waitForEvent("replacement-listeners-retired").then(() => undefined, (error) => error instanceof Error ? error : new Error(String(error))),
|
|
167
|
+
this.guardian.waitForEvent("replacement-retirement-failed").then((event) => new Error(stringOrUndefined(event.reason) || "Retired listener state transfer failed"), (error) => error instanceof Error ? error : new Error(String(error))),
|
|
168
|
+
this.guardian.waitForEvent("replacement-committed").then(() => undefined, (error) => error instanceof Error ? error : new Error(String(error)))
|
|
169
|
+
])
|
|
170
|
+
|
|
171
|
+
status = await this.guardian.replacementStatus()
|
|
172
|
+
if (status.retirementPending && !status.retirementReady) {
|
|
173
|
+
const settlementError = await settled
|
|
174
|
+
|
|
175
|
+
if (settlementError) throw settlementError
|
|
176
|
+
status = await this.guardian.replacementStatus()
|
|
177
|
+
}
|
|
178
|
+
if (!status.retirementPending) return false
|
|
179
|
+
if (!status.retirementReady) throw new Error("Guardian listener retirement did not publish complete state")
|
|
180
|
+
}
|
|
181
|
+
const replacementId = status.committedReplacementId
|
|
182
|
+
|
|
183
|
+
if (!replacementId) throw new Error("Guardian listener retirement lost its committed replacement")
|
|
112
184
|
await this.startProxy()
|
|
113
|
-
|
|
185
|
+
await this.guardian.finalizeOwnerReplacement(replacementId)
|
|
186
|
+
return true
|
|
114
187
|
}
|
|
115
188
|
|
|
116
189
|
/** Connects to the durable process guardian and reconstructs a matching persisted owner snapshot. */
|
|
@@ -119,13 +192,8 @@ export default class RollbridgeDaemon {
|
|
|
119
192
|
const state = await readState(this.statePath)
|
|
120
193
|
const snapshot = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
|
|
121
194
|
const recovery = snapshot?.recovery
|
|
122
|
-
const configDigest = this.ownerRecoveryConfigDigest()
|
|
123
195
|
|
|
124
196
|
if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
|
|
125
|
-
if (recovery && recovery.configDigest !== configDigest) throw new Error("Owner recovery config identity does not match the persisted owner; refusing cross-authority adoption.")
|
|
126
|
-
if (snapshot && ((this.runtime?.digest ?? null) !== (snapshot.daemonRuntime?.digest ?? null))) {
|
|
127
|
-
throw new Error("Owner recovery runtime identity does not match the persisted owner; use the exact same Rollbridge runtime.")
|
|
128
|
-
}
|
|
129
197
|
|
|
130
198
|
const guardianIdentity = recovery?.guardian || {
|
|
131
199
|
socketPath: `${this.statePath}.guardian.sock`,
|
|
@@ -133,49 +201,73 @@ export default class RollbridgeDaemon {
|
|
|
133
201
|
}
|
|
134
202
|
this.guardianIdentity = guardianIdentity
|
|
135
203
|
this.guardian = new GuardianClient(guardianIdentity)
|
|
136
|
-
if (recovery)
|
|
204
|
+
if (recovery) {
|
|
205
|
+
await this.guardian.connect()
|
|
206
|
+
await this.assertGuardianDaemonRecoveryCapability()
|
|
207
|
+
}
|
|
137
208
|
else {
|
|
138
209
|
await this.guardian.launch()
|
|
139
210
|
guardianIdentity.pid = this.guardian.pid
|
|
140
211
|
}
|
|
141
|
-
await this.guardian.claimOwner(this.config.ownerRecovery?.reconnectGraceMs ?? 30000, this.ownerAuthority())
|
|
142
212
|
this.watchOwnerReplacementEvents()
|
|
213
|
+
await this.guardian.claimOwner(this.config.ownerRecovery?.reconnectGraceMs ?? 30000, this.ownerAuthority())
|
|
143
214
|
|
|
144
215
|
if (snapshot) {
|
|
145
216
|
if (!Array.isArray(snapshot.releases) || (snapshot.activeReleaseId !== null && typeof snapshot.activeReleaseId !== "string")) {
|
|
146
217
|
throw new Error("Owner recovery state is partial or corrupt; active release metadata is required.")
|
|
147
218
|
}
|
|
148
|
-
|
|
149
|
-
let releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({})
|
|
150
|
-
let singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({})
|
|
151
|
-
let recoverySnapshot = snapshot
|
|
219
|
+
const ownerState = /** @type {PrivateOwnerState} */ (await this.guardian.ownerState())
|
|
152
220
|
|
|
153
|
-
if (
|
|
154
|
-
|
|
221
|
+
if (this.ownerRetired) return
|
|
222
|
+
if (!ownerState?.config || !ownerState.snapshot) throw new Error("Committed guardian published incomplete owner recovery state")
|
|
223
|
+
let recoverySnapshot = ownerState.snapshot
|
|
155
224
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
225
|
+
if ((recovery?.format ?? 0) >= 2) {
|
|
226
|
+
const releaseConfigs = ownerState.releaseConfigs
|
|
227
|
+
|
|
228
|
+
if (!releaseConfigs) throw new Error("Durable guardian state is missing exact release definitions; refusing partial owner recovery.")
|
|
229
|
+
const guardianTransition = ownerState.snapshot.generationTransition
|
|
230
|
+
const persistedTransition = snapshot.generationTransition
|
|
231
|
+
const sameTransition = Boolean(guardianTransition && persistedTransition &&
|
|
232
|
+
guardianTransition.candidateReleaseId === persistedTransition.candidateReleaseId &&
|
|
233
|
+
guardianTransition.candidateReleasePath === persistedTransition.candidateReleasePath &&
|
|
234
|
+
guardianTransition.candidateRevision === persistedTransition.candidateRevision &&
|
|
235
|
+
guardianTransition.configDigest === persistedTransition.configDigest &&
|
|
236
|
+
guardianTransition.previousReleaseId === persistedTransition.previousReleaseId &&
|
|
237
|
+
guardianTransition.startedAt === persistedTransition.startedAt)
|
|
238
|
+
const requiredReleaseIds = new Set([
|
|
239
|
+
...ownerState.snapshot.releaseReferences.map((reference) => reference.releaseId),
|
|
240
|
+
...snapshot.releaseReferences.map((reference) => reference.releaseId),
|
|
241
|
+
...Object.values(ownerState.serviceReleaseIds || {}),
|
|
242
|
+
...Object.values(snapshot.serviceReleaseIds || {}),
|
|
243
|
+
...Object.values(ownerState.singletonReleaseIds || {}),
|
|
244
|
+
...Object.values(snapshot.singletonReleaseIds || {}),
|
|
245
|
+
...(ownerState.snapshot.activeReleaseId ? [ownerState.snapshot.activeReleaseId] : []),
|
|
246
|
+
...(snapshot.activeReleaseId ? [snapshot.activeReleaseId] : []),
|
|
247
|
+
...(guardianTransition ? [guardianTransition.candidateReleaseId, ...(guardianTransition.previousReleaseId ? [guardianTransition.previousReleaseId] : [])] : []),
|
|
248
|
+
...(persistedTransition ? [persistedTransition.candidateReleaseId, ...(persistedTransition.previousReleaseId ? [persistedTransition.previousReleaseId] : [])] : [])
|
|
249
|
+
])
|
|
250
|
+
const publicReleasesComplete = [...requiredReleaseIds].every((releaseId) => snapshot.releases.some((release) => release.releaseId === releaseId) && releaseConfigs[releaseId])
|
|
251
|
+
|
|
252
|
+
if (sameTransition && publicReleasesComplete &&
|
|
253
|
+
typeof persistedTransition?.journalRevision === "number" &&
|
|
254
|
+
typeof guardianTransition?.journalRevision === "number" &&
|
|
255
|
+
persistedTransition.journalRevision > guardianTransition.journalRevision) {
|
|
256
|
+
recoverySnapshot = snapshot
|
|
167
257
|
}
|
|
168
258
|
}
|
|
169
|
-
await this.restoreOwnerState(recoverySnapshot, {
|
|
259
|
+
await this.restoreOwnerState(recoverySnapshot, {
|
|
260
|
+
config: ownerState.config,
|
|
261
|
+
listenerConnectionSources: ownerState.listenerConnectionSources,
|
|
262
|
+
releaseConfigs: ownerState.releaseConfigs,
|
|
263
|
+
resumeDrains: false,
|
|
264
|
+
serviceReleaseIds: recoverySnapshot === snapshot ? snapshot.serviceReleaseIds || ownerState.serviceReleaseIds : ownerState.serviceReleaseIds || snapshot.serviceReleaseIds,
|
|
265
|
+
singletonReleaseIds: recoverySnapshot === snapshot ? snapshot.singletonReleaseIds || ownerState.singletonReleaseIds : ownerState.singletonReleaseIds || snapshot.singletonReleaseIds
|
|
266
|
+
})
|
|
267
|
+
if (this.ownerRetired) return
|
|
170
268
|
this.persistenceEnabled = true
|
|
171
269
|
await this.guardian.reconcileInventory()
|
|
172
|
-
if (this.
|
|
173
|
-
try {
|
|
174
|
-
await this.resumeGenerationTransition()
|
|
175
|
-
} catch (error) {
|
|
176
|
-
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: this.generationTransition.candidateReleaseId})
|
|
177
|
-
}
|
|
178
|
-
}
|
|
270
|
+
if (this.ownerRetired) return
|
|
179
271
|
for (const release of this.releases.values()) {
|
|
180
272
|
if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
181
273
|
}
|
|
@@ -184,6 +276,7 @@ export default class RollbridgeDaemon {
|
|
|
184
276
|
this.persistenceEnabled = true
|
|
185
277
|
await this.persistState({throwOnError: true})
|
|
186
278
|
}
|
|
279
|
+
if (this.ownerRetired) return
|
|
187
280
|
this.stateCleanupEnabled = true
|
|
188
281
|
await this.publishOwnerState()
|
|
189
282
|
}
|
|
@@ -196,16 +289,46 @@ export default class RollbridgeDaemon {
|
|
|
196
289
|
/** Installs event-driven drain fencing for the next prepared owner transaction. */
|
|
197
290
|
watchOwnerReplacementEvents() {
|
|
198
291
|
if (!this.guardian) throw new Error("Owner replacement event fencing requires the durable guardian")
|
|
292
|
+
this.guardian.onEvent("owner-connection-state", (event) => this.handleIncumbentListenerEvent(event))
|
|
199
293
|
this.guardian.onEvent("replacement-prepared", () => {
|
|
200
294
|
for (const release of this.releases.values()) release.pauseDrainForOwnerHandoff()
|
|
201
295
|
})
|
|
202
296
|
this.guardian.onEvent("replacement-aborted", () => {
|
|
297
|
+
if (this.retiredListenerHandoff) {
|
|
298
|
+
void this.resumeControlLessOwnerListeners().catch((error) => {
|
|
299
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
300
|
+
this.logger("control-less owner listener handoff recovery failed", {error: this.listenerHandoffFailure.message})
|
|
301
|
+
})
|
|
302
|
+
return
|
|
303
|
+
}
|
|
203
304
|
if (this.listenerHandoff || this.ownerRetired) return
|
|
204
305
|
for (const release of this.releases.values()) {
|
|
205
306
|
release.resumeDrainAfterOwnerHandoff()
|
|
206
307
|
if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
207
308
|
}
|
|
208
309
|
})
|
|
310
|
+
this.guardian.onEvent("replacement-listener-handoff-requested", (event) => {
|
|
311
|
+
if (this.ownerRetired) return
|
|
312
|
+
const replacementId = stringOrUndefined(event.replacementId)
|
|
313
|
+
|
|
314
|
+
if (!replacementId) throw new Error("Owner listener handoff request is missing its replacement id")
|
|
315
|
+
void this.yieldControlLessOwnerListeners(replacementId).catch((error) => {
|
|
316
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
317
|
+
this.logger("control-less owner listener handoff failed", {error: this.listenerHandoffFailure.message, replacementId})
|
|
318
|
+
this.guardian?.disconnect()
|
|
319
|
+
})
|
|
320
|
+
})
|
|
321
|
+
this.guardian.onEvent("replacement-retirement-requested", (event) => {
|
|
322
|
+
if (this.ownerRetired) return
|
|
323
|
+
const replacementId = stringOrUndefined(event.replacementId)
|
|
324
|
+
|
|
325
|
+
if (!replacementId) throw new Error("Committed owner retirement request is missing its replacement id")
|
|
326
|
+
void this.completeControlLessOwnerRetirement(replacementId).catch((error) => {
|
|
327
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
328
|
+
this.logger("committed owner retirement failed", {error: this.listenerHandoffFailure.message, replacementId})
|
|
329
|
+
this.guardian?.disconnect()
|
|
330
|
+
})
|
|
331
|
+
})
|
|
209
332
|
}
|
|
210
333
|
|
|
211
334
|
/** @returns {{configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}} Exact authority fence. */
|
|
@@ -217,12 +340,14 @@ export default class RollbridgeDaemon {
|
|
|
217
340
|
* @param {OwnerRecoverySnapshot} snapshot - Validated persisted owner state.
|
|
218
341
|
* @param {object} [options] - Recovery definition options.
|
|
219
342
|
* @param {import("./config.js").RollbridgeConfig} [options.config] - Owner config for daemon-wide processes.
|
|
343
|
+
* @param {Record<string, Record<string, {http: number, websocket: number}>>} [options.listenerConnectionSources] - Exact inherited listener sources.
|
|
220
344
|
* @param {Record<string, import("./config.js").RollbridgeConfig>} [options.releaseConfigs] - Exact generation configs.
|
|
221
345
|
* @param {boolean} [options.resumeDrains] - Whether to resume draining generations immediately.
|
|
346
|
+
* @param {Record<string, string>} [options.serviceReleaseIds] - Exact release owner for each persistent service definition.
|
|
222
347
|
* @param {Record<string, string>} [options.singletonReleaseIds] - Exact release owner for each singleton registration.
|
|
223
348
|
* @param {boolean} [options.synchronizeLifecycleRoles] - Whether the caller owns guardian role mutation authority.
|
|
224
349
|
*/
|
|
225
|
-
async restoreOwnerState(snapshot, {config = this.config, releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({}), resumeDrains = true, singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({}), synchronizeLifecycleRoles = true} = {}) {
|
|
350
|
+
async restoreOwnerState(snapshot, {config = this.config, listenerConnectionSources = /** @type {Record<string, Record<string, {http: number, websocket: number}>>} */ ({}), releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({}), resumeDrains = true, serviceReleaseIds = snapshot.serviceReleaseIds || /** @type {Record<string, string>} */ ({}), singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({}), synchronizeLifecycleRoles = true} = {}) {
|
|
226
351
|
if (!Array.isArray(snapshot.releases) || (snapshot.activeReleaseId !== null && typeof snapshot.activeReleaseId !== "string")) {
|
|
227
352
|
throw new Error("Owner recovery state is partial or corrupt; active release metadata is required.")
|
|
228
353
|
}
|
|
@@ -230,13 +355,19 @@ export default class RollbridgeDaemon {
|
|
|
230
355
|
if (snapshot.activeReleaseId === null && snapshot.releases.length === 0) return
|
|
231
356
|
if (!this.bootstrap) this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
|
|
232
357
|
this.ownerTransition = snapshot.ownerTransition ? {...snapshot.ownerTransition} : undefined
|
|
358
|
+
const fallbackDefinitionReleaseId = snapshot.activeReleaseId || snapshot.releases.at(-1)?.releaseId
|
|
359
|
+
const serviceOwnerReleaseIds = new Set(Object.values(serviceReleaseIds))
|
|
233
360
|
const singletonOwnerReleaseIds = new Set(Object.values(singletonReleaseIds))
|
|
361
|
+
const transitionReleaseIds = this.generationTransitionReleaseIds()
|
|
362
|
+
|
|
363
|
+
if (snapshot.services.some((service) => !serviceReleaseIds[service.id]) && fallbackDefinitionReleaseId) serviceOwnerReleaseIds.add(fallbackDefinitionReleaseId)
|
|
234
364
|
|
|
235
365
|
for (const releaseStatus of snapshot.releases) {
|
|
236
|
-
const
|
|
366
|
+
const transitionOwner = transitionReleaseIds.has(releaseStatus.releaseId)
|
|
367
|
+
const serviceOwner = serviceOwnerReleaseIds.has(releaseStatus.releaseId)
|
|
237
368
|
const singletonOwner = singletonOwnerReleaseIds.has(releaseStatus.releaseId)
|
|
238
369
|
|
|
239
|
-
if (releaseStatus.state !== "active" && releaseStatus.state !== "draining" && !
|
|
370
|
+
if (releaseStatus.state !== "active" && releaseStatus.state !== "draining" && !transitionOwner && !serviceOwner && !singletonOwner) continue
|
|
240
371
|
const release = new ReleaseGroup({
|
|
241
372
|
config: releaseConfigs[releaseStatus.releaseId] || config,
|
|
242
373
|
logger: this.logger,
|
|
@@ -253,22 +384,34 @@ export default class RollbridgeDaemon {
|
|
|
253
384
|
this.releases.set(release.releaseId, release)
|
|
254
385
|
if (release.releaseId === snapshot.activeReleaseId) this.activeRelease = release
|
|
255
386
|
}
|
|
387
|
+
for (const [sourceId, releases] of Object.entries(listenerConnectionSources)) {
|
|
388
|
+
for (const [releaseId, connections] of Object.entries(releases)) {
|
|
389
|
+
this.setListenerConnectionSource(sourceId, releaseId, connections)
|
|
390
|
+
}
|
|
391
|
+
}
|
|
256
392
|
|
|
257
393
|
if (snapshot.activeReleaseId !== null && !this.activeRelease) throw new Error(`Owner recovery state does not contain active release ${snapshot.activeReleaseId}.`)
|
|
258
394
|
const committedBootstrapRelease = this.committedBootstrapRelease()
|
|
259
395
|
const definitionRelease = this.activeRelease || committedBootstrapRelease || [...this.releases.values()].at(-1)
|
|
260
396
|
if (!definitionRelease) throw new Error("Owner recovery state has no release definition for owned processes.")
|
|
261
|
-
if (!this.activeRelease && !committedBootstrapRelease && snapshot.singletons.
|
|
397
|
+
if (!this.activeRelease && !committedBootstrapRelease && snapshot.singletons.some((singleton) => !singletonReleaseIds[singleton.id])) {
|
|
398
|
+
throw new Error("Owner recovery state has release-owned singletons without an exact release identity.")
|
|
399
|
+
}
|
|
262
400
|
for (const serviceStatus of snapshot.services) {
|
|
263
|
-
const
|
|
401
|
+
const serviceReleaseId = serviceReleaseIds[serviceStatus.id] || fallbackDefinitionReleaseId
|
|
402
|
+
const serviceRelease = serviceReleaseId ? this.releases.get(serviceReleaseId) : undefined
|
|
264
403
|
|
|
265
|
-
if (!
|
|
266
|
-
const
|
|
404
|
+
if (!serviceRelease) throw new Error(`Owner recovery state contains service ${serviceStatus.id} for an unknown release.`)
|
|
405
|
+
const processConfig = serviceRelease.config.processes.find((candidate) => candidate.id === serviceStatus.id && candidate.policy === "service" && candidate.deployStrategy !== "handoff")
|
|
406
|
+
|
|
407
|
+
if (!processConfig) throw new Error(`Owner recovery state contains unknown service ${serviceStatus.id} for release ${serviceReleaseId}.`)
|
|
408
|
+
const service = serviceRelease.buildProcess(processConfig, {guardianKey: `service:${serviceStatus.id}`, shouldRestart: () => !this.stopping})
|
|
267
409
|
|
|
268
410
|
await this.recoverGuardianProcess(service)
|
|
269
411
|
this.services.set(serviceStatus.id, service)
|
|
270
|
-
|
|
271
|
-
|
|
412
|
+
this.serviceReleaseIds.set(serviceStatus.id, serviceRelease.releaseId)
|
|
413
|
+
if (serviceRelease.ports[serviceStatus.id] !== undefined) {
|
|
414
|
+
const port = serviceRelease.ports[serviceStatus.id]
|
|
272
415
|
|
|
273
416
|
if (this.portReservations.has(port)) throw new Error(`Persisted daemon service ${serviceStatus.id} port ${port} is already reserved by a live generation`)
|
|
274
417
|
this.portReservations.add(port)
|
|
@@ -298,13 +441,34 @@ export default class RollbridgeDaemon {
|
|
|
298
441
|
/** Publishes full normalized definitions only over the authenticated guardian channel. */
|
|
299
442
|
async publishOwnerState() {
|
|
300
443
|
if (!this.guardian) return
|
|
301
|
-
await this.guardian.publishOwnerState(
|
|
444
|
+
await this.guardian.publishOwnerState(this.transferableOwnerState())
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/** Confirms guardian-owned startup only after every requested listener is available. */
|
|
448
|
+
async markOwnerReady() {
|
|
449
|
+
if (!this.guardian) return
|
|
450
|
+
await this.guardian.ownerReady()
|
|
451
|
+
this.ownerReady = true
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/** @returns {Record<string, JsonValue>} Complete private guardian owner state. */
|
|
455
|
+
transferableOwnerState() {
|
|
456
|
+
return {
|
|
302
457
|
authority: this.ownerAuthority(),
|
|
303
458
|
config: this.config,
|
|
459
|
+
recovery: this.recoveryCommand ? {
|
|
460
|
+
command: this.recoveryCommand,
|
|
461
|
+
reconnectGraceMs: this.config.ownerRecovery?.reconnectGraceMs ?? 30000,
|
|
462
|
+
startupTimeoutMs: this.recoveryCommand.startupTimeoutMs
|
|
463
|
+
} : undefined,
|
|
464
|
+
listenerConnectionSources: this.serializedListenerConnectionSources(),
|
|
465
|
+
listenerSourceId: this.listenerSourceId,
|
|
304
466
|
releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
|
|
467
|
+
retiredListenerHandoff: 2,
|
|
468
|
+
serviceReleaseIds: Object.fromEntries(this.serviceReleaseIds),
|
|
305
469
|
singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds),
|
|
306
470
|
snapshot: this.status()
|
|
307
|
-
}
|
|
471
|
+
}
|
|
308
472
|
}
|
|
309
473
|
|
|
310
474
|
/**
|
|
@@ -339,12 +503,17 @@ export default class RollbridgeDaemon {
|
|
|
339
503
|
preparedStatus = await this.guardian.replacementStatus()
|
|
340
504
|
transfer = /** @type {PrivateOwnerState} */ (prepared.ownerState)
|
|
341
505
|
if (!transfer?.config || !transfer.snapshot) throw new Error("Committed owner published incomplete replacement state")
|
|
506
|
+
const unresolvedTransition = transfer.snapshot.generationTransition
|
|
507
|
+
|
|
508
|
+
if (unresolvedTransition && unresolvedTransition.phase !== "committed" && unresolvedTransition.configDigest !== this.ownerRecoveryConfigDigest()) {
|
|
509
|
+
throw new Error(`Owner replacement cannot change config authority while unresolved generation transition ${unresolvedTransition.candidateReleaseId} remains at ${unresolvedTransition.phase}`)
|
|
510
|
+
}
|
|
342
511
|
const registeredProcesses = new Map((await this.guardian.inventory()).map(({key, provenance}) => [key, provenance]))
|
|
343
512
|
|
|
344
513
|
reservedProcessKey = legacyBridge ? undefined : reconstructableOwnerSnapshotProcessKeys(transfer.snapshot, transfer.singletonReleaseIds)
|
|
345
514
|
.find((key) => registeredProcesses.has(key))
|
|
346
515
|
if (reservedProcessKey) this.guardian.reserveProcessRecovery(reservedProcessKey, /** @type {string} */ (registeredProcesses.get(reservedProcessKey)))
|
|
347
|
-
await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, releaseConfigs: transfer.releaseConfigs, resumeDrains: false, singletonReleaseIds: transfer.singletonReleaseIds, synchronizeLifecycleRoles: false})
|
|
516
|
+
await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, listenerConnectionSources: transfer.listenerConnectionSources, releaseConfigs: transfer.releaseConfigs, resumeDrains: false, serviceReleaseIds: transfer.serviceReleaseIds, singletonReleaseIds: transfer.singletonReleaseIds, synchronizeLifecycleRoles: false})
|
|
348
517
|
} catch (error) {
|
|
349
518
|
legacyBridge?.incumbentControl?.close()
|
|
350
519
|
if (legacyBridge) {
|
|
@@ -358,6 +527,16 @@ export default class RollbridgeDaemon {
|
|
|
358
527
|
}
|
|
359
528
|
throw error
|
|
360
529
|
}
|
|
530
|
+
if (!legacyBridge) {
|
|
531
|
+
try {
|
|
532
|
+
await this.assertGuardianDaemonRecoveryCapability()
|
|
533
|
+
} catch (error) {
|
|
534
|
+
// The immediately preceding guardian protocol notifies the incumbent of
|
|
535
|
+
// an abort only when the prepared candidate disconnects.
|
|
536
|
+
this.guardian.disconnect()
|
|
537
|
+
throw error
|
|
538
|
+
}
|
|
539
|
+
}
|
|
361
540
|
for (const release of this.releases.values()) release.preserveConfigOnRetirement = true
|
|
362
541
|
this.logger("owner replacement candidate prepared", {activeReleaseId: this.activeRelease?.releaseId ?? null, replacementId: prepared.replacementId})
|
|
363
542
|
|
|
@@ -412,40 +591,61 @@ export default class RollbridgeDaemon {
|
|
|
412
591
|
listenersYielded = true
|
|
413
592
|
}
|
|
414
593
|
}
|
|
415
|
-
if (sharedFixedProxy) await this.startProxy()
|
|
594
|
+
if (sharedFixedProxy && !retiredIncumbentControl) await this.startProxy()
|
|
416
595
|
if (!retiredIncumbentControl) {
|
|
417
596
|
await fs.rename(stagingControlPath, this.config.control.path)
|
|
418
597
|
finalControlPublished = true
|
|
419
598
|
this.boundControlPath = this.config.control.path
|
|
420
599
|
}
|
|
421
|
-
|
|
422
|
-
() =>
|
|
600
|
+
const retirementFailure = this.guardian.waitForEvent("replacement-retirement-failed").then(
|
|
601
|
+
(event) => new Error(stringOrUndefined(event.reason) || "Retired listener state transfer failed"),
|
|
423
602
|
(error) => error instanceof Error ? error : new Error(String(error))
|
|
424
603
|
)
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
604
|
+
committed = Promise.race([
|
|
605
|
+
this.guardian.waitForEvent("replacement-committed").then(
|
|
606
|
+
() => undefined,
|
|
607
|
+
(error) => error instanceof Error ? error : new Error(String(error))
|
|
608
|
+
),
|
|
609
|
+
retirementFailure
|
|
610
|
+
])
|
|
611
|
+
const listenerRetirement = retiredIncumbentControl
|
|
612
|
+
? Promise.race([
|
|
613
|
+
this.guardian.waitForEvent("replacement-listeners-retired").then(() => undefined, (error) => error instanceof Error ? error : new Error(String(error))),
|
|
614
|
+
retirementFailure
|
|
615
|
+
])
|
|
616
|
+
: undefined
|
|
617
|
+
const staged = await this.guardian.stageOwnerReplacement(prepared.replacementId, this.transferableOwnerState())
|
|
432
618
|
|
|
433
619
|
if (staged.committed && reservedProcessKey) {
|
|
434
620
|
committedAuthority = true
|
|
435
621
|
await this.guardian.recoverReservedProcess(reservedProcessKey)
|
|
436
622
|
}
|
|
623
|
+
if (staged.committed && retiredIncumbentControl && sharedFixedProxy) await this.startProxy()
|
|
437
624
|
|
|
438
625
|
if (!staged.committed) {
|
|
439
626
|
if (retiredIncumbentControl) {
|
|
440
627
|
const processKey = reservedProcessKey
|
|
441
628
|
|
|
442
629
|
if (!processKey || !this.guardian.processes.has(processKey)) throw new Error("Retired owner replacement requires an exact reserved process from committed owner state")
|
|
630
|
+
if (transfer.retiredListenerHandoff !== 2) {
|
|
631
|
+
throw new Error("Cannot safely complete atomic owner replacement through the older retained guardian while the incumbent control socket is absent; incumbent owner and connections were preserved")
|
|
632
|
+
}
|
|
443
633
|
try {
|
|
634
|
+
await this.guardian.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
|
|
635
|
+
const listenerRetirementError = await listenerRetirement
|
|
636
|
+
|
|
637
|
+
if (listenerRetirementError) throw listenerRetirementError
|
|
638
|
+
if (sharedFixedProxy) await this.startProxy()
|
|
444
639
|
await this.guardian.commitRetiredOwnerReplacement(prepared.replacementId, processKey)
|
|
445
640
|
committedAuthority = true
|
|
446
641
|
await this.guardian.recoverReservedProcess(processKey)
|
|
642
|
+
await this.guardian.finalizeOwnerReplacement(prepared.replacementId)
|
|
447
643
|
} catch (error) {
|
|
448
|
-
if (!(error instanceof Error) ||
|
|
644
|
+
if (!(error instanceof Error) || ![
|
|
645
|
+
"Guardian commit-retired-owner-replacement requires the committed owner",
|
|
646
|
+
"Guardian prepare-retired-owner-listener-handoff requires the committed owner",
|
|
647
|
+
"Unknown guardian command: prepare-retired-owner-listener-handoff"
|
|
648
|
+
].includes(error.message)) throw error
|
|
449
649
|
throw new Error(
|
|
450
650
|
"Cannot safely complete atomic owner replacement through the older retained guardian while the incumbent control socket is absent; incumbent owner and connections were preserved",
|
|
451
651
|
{cause: error}
|
|
@@ -455,10 +655,12 @@ export default class RollbridgeDaemon {
|
|
|
455
655
|
try {
|
|
456
656
|
if (!incumbentControl) throw new Error("Owner replacement incumbent control session is unavailable")
|
|
457
657
|
await incumbentControl.request({command: "commit-owner-replacement", replacementId: prepared.replacementId})
|
|
658
|
+
committedAuthority = true
|
|
458
659
|
} catch (error) {
|
|
459
660
|
const status = await this.guardian.replacementStatus()
|
|
460
661
|
|
|
461
662
|
if (status.committedReplacementId !== prepared.replacementId || !status.ownerClaimed) throw error
|
|
663
|
+
committedAuthority = true
|
|
462
664
|
this.logger("owner replacement commit response lost; guardian commit confirmed", {replacementId: prepared.replacementId})
|
|
463
665
|
}
|
|
464
666
|
}
|
|
@@ -535,33 +737,109 @@ export default class RollbridgeDaemon {
|
|
|
535
737
|
}
|
|
536
738
|
}
|
|
537
739
|
|
|
740
|
+
/** Requires the persistent guardian protocol which can restart an accepted daemon. */
|
|
741
|
+
async assertGuardianDaemonRecoveryCapability() {
|
|
742
|
+
if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
|
|
743
|
+
let capabilities
|
|
744
|
+
|
|
745
|
+
try {
|
|
746
|
+
capabilities = await this.guardian.capabilities()
|
|
747
|
+
} catch (error) {
|
|
748
|
+
if (!(error instanceof Error) || !["Guardian capabilities requires a process key", "Unknown guardian command: capabilities"].includes(error.message)) throw error
|
|
749
|
+
throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
|
|
750
|
+
}
|
|
751
|
+
if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
|
|
752
|
+
}
|
|
753
|
+
|
|
538
754
|
/**
|
|
539
755
|
* Applies authenticated live-connection state from the retired listener owner.
|
|
540
756
|
* @param {Record<string, JsonValue>} event - Incumbent control event.
|
|
541
|
-
* @param {{close: () => void}} session - Exact incumbent session.
|
|
757
|
+
* @param {{close: () => void}} [session] - Exact incumbent control session.
|
|
542
758
|
*/
|
|
543
759
|
handleIncumbentListenerEvent(event, session) {
|
|
544
760
|
if (event.event !== "owner-connection-state") return
|
|
545
761
|
const releaseId = stringOrUndefined(event.releaseId)
|
|
762
|
+
let sourceId = stringOrUndefined(event.sourceId)
|
|
546
763
|
const connections = event.connections
|
|
547
764
|
|
|
548
|
-
if (!
|
|
765
|
+
if (!sourceId && session) {
|
|
766
|
+
sourceId = this.incumbentListenerSourceIds.get(session)
|
|
767
|
+
if (!sourceId) {
|
|
768
|
+
sourceId = crypto.randomUUID()
|
|
769
|
+
this.incumbentListenerSourceIds.set(session, sourceId)
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
if (!sourceId || !releaseId || !connections || typeof connections !== "object" || Array.isArray(connections)) {
|
|
549
773
|
throw new Error("Incumbent listener sent invalid connection state")
|
|
550
774
|
}
|
|
551
|
-
const
|
|
775
|
+
const normalized = {
|
|
552
776
|
http: requiredNonNegativeInteger(connections.http, "connections.http"),
|
|
553
777
|
websocket: requiredNonNegativeInteger(connections.websocket, "connections.websocket")
|
|
554
778
|
}
|
|
555
|
-
const release = this.releases.get(releaseId)
|
|
556
779
|
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
780
|
+
this.setListenerConnectionSource(sourceId, releaseId, normalized)
|
|
781
|
+
const handoffSocket = this.listenerHandoff?.completionSocket
|
|
782
|
+
|
|
783
|
+
if (handoffSocket && !handoffSocket.destroyed) {
|
|
784
|
+
handoffSocket.write(`${JSON.stringify({connections: normalized, event: "owner-connection-state", releaseId, sourceId})}\n`)
|
|
785
|
+
}
|
|
786
|
+
if (this.retiredReplacementId) {
|
|
787
|
+
void this.publishRetiredConnectionState(this.retiredReplacementId, sourceId, releaseId, normalized).catch((error) => {
|
|
788
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
789
|
+
this.logger("retired owner connection-state relay failed", {error: this.listenerHandoffFailure.message, releaseId, replacementId: this.retiredReplacementId, sourceId})
|
|
790
|
+
this.guardian?.disconnect()
|
|
791
|
+
})
|
|
792
|
+
}
|
|
793
|
+
if (session && this.incumbentListenerControl === session && ![...this.releases.values()].some((candidate) => candidate.hasTransferredConnections())) {
|
|
560
794
|
this.incumbentListenerControl = undefined
|
|
561
795
|
session.close()
|
|
562
796
|
}
|
|
563
797
|
}
|
|
564
798
|
|
|
799
|
+
/**
|
|
800
|
+
* Reconciles one physical listener source and applies its aggregate to the release.
|
|
801
|
+
* @param {string} sourceId - Stable listener source.
|
|
802
|
+
* @param {string} releaseId - Retained release.
|
|
803
|
+
* @param {{http: number, websocket: number}} connections - Exact source counts.
|
|
804
|
+
*/
|
|
805
|
+
setListenerConnectionSource(sourceId, releaseId, connections) {
|
|
806
|
+
const release = this.releases.get(releaseId)
|
|
807
|
+
|
|
808
|
+
if (!release && (connections.http > 0 || connections.websocket > 0)) throw new Error(`Incumbent listener reported unknown release ${releaseId}`)
|
|
809
|
+
const sourceReleases = this.listenerConnectionSources.get(sourceId) || new Map()
|
|
810
|
+
|
|
811
|
+
if (connections.http + connections.websocket === 0) sourceReleases.delete(releaseId)
|
|
812
|
+
else sourceReleases.set(releaseId, connections)
|
|
813
|
+
if (sourceReleases.size === 0) this.listenerConnectionSources.delete(sourceId)
|
|
814
|
+
else this.listenerConnectionSources.set(sourceId, sourceReleases)
|
|
815
|
+
if (!release) return
|
|
816
|
+
let http = 0
|
|
817
|
+
let websocket = 0
|
|
818
|
+
|
|
819
|
+
for (const candidateReleases of this.listenerConnectionSources.values()) {
|
|
820
|
+
const counts = candidateReleases.get(releaseId)
|
|
821
|
+
|
|
822
|
+
if (!counts) continue
|
|
823
|
+
http += counts.http
|
|
824
|
+
websocket += counts.websocket
|
|
825
|
+
}
|
|
826
|
+
release.setTransferredConnections({http, websocket})
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
/** @returns {Record<string, Record<string, {http: number, websocket: number}>>} Exact live listener sources. */
|
|
830
|
+
serializedListenerConnectionSources() {
|
|
831
|
+
const sources = new Map([...this.listenerConnectionSources].map(([sourceId, releases]) => [sourceId, new Map(releases)]))
|
|
832
|
+
const local = new Map()
|
|
833
|
+
|
|
834
|
+
for (const release of this.releases.values()) {
|
|
835
|
+
const connections = release.localConnections()
|
|
836
|
+
|
|
837
|
+
if (connections.http + connections.websocket > 0) local.set(release.releaseId, connections)
|
|
838
|
+
}
|
|
839
|
+
if (local.size > 0) sources.set(this.listenerSourceId, local)
|
|
840
|
+
return Object.fromEntries([...sources].map(([sourceId, releases]) => [sourceId, Object.fromEntries(releases)]))
|
|
841
|
+
}
|
|
842
|
+
|
|
565
843
|
/**
|
|
566
844
|
* Authenticates and prepares the one-time disruptive bridge for an exact legacy guardian protocol.
|
|
567
845
|
* @param {{persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
|
|
@@ -619,12 +897,14 @@ export default class RollbridgeDaemon {
|
|
|
619
897
|
|
|
620
898
|
assertLegacyIncumbentStatus(incumbentStatus, persisted)
|
|
621
899
|
}
|
|
900
|
+
const definitionReleaseId = persisted.activeReleaseId || persisted.releases.at(-1)?.releaseId
|
|
622
901
|
const ownerState = legacyPrepared
|
|
623
902
|
? legacyPrepared.ownerState
|
|
624
903
|
: {
|
|
625
904
|
authority: persistedAuthority,
|
|
626
905
|
config: this.config,
|
|
627
906
|
releaseConfigs: Object.fromEntries(persisted.releases.map((release) => [release.releaseId, this.config])),
|
|
907
|
+
serviceReleaseIds: definitionReleaseId ? Object.fromEntries(persisted.services.map((service) => [service.id, definitionReleaseId])) : {},
|
|
628
908
|
singletonReleaseIds: Object.fromEntries(persisted.singletons.map((singleton) => [singleton.id, persisted.activeReleaseId]).filter((entry) => entry[1] !== null)),
|
|
629
909
|
snapshot: persisted
|
|
630
910
|
}
|
|
@@ -715,6 +995,7 @@ export default class RollbridgeDaemon {
|
|
|
715
995
|
authority: this.ownerAuthority(),
|
|
716
996
|
config: this.config,
|
|
717
997
|
releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
|
|
998
|
+
serviceReleaseIds: Object.fromEntries(this.serviceReleaseIds),
|
|
718
999
|
singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds),
|
|
719
1000
|
snapshot: this.status()
|
|
720
1001
|
}
|
|
@@ -736,14 +1017,18 @@ export default class RollbridgeDaemon {
|
|
|
736
1017
|
incumbentPid: this.legacyIncumbentPid,
|
|
737
1018
|
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
738
1019
|
})
|
|
739
|
-
await this.guardian.beginLegacyOwnerClaim(
|
|
1020
|
+
await this.guardian.beginLegacyOwnerClaim(
|
|
1021
|
+
bridge.prepared.replacementId,
|
|
1022
|
+
bridge.recoverySnapshot.recovery.reconnectGraceMs,
|
|
1023
|
+
this.statePath,
|
|
1024
|
+
bridge.recoverySnapshot
|
|
1025
|
+
)
|
|
740
1026
|
process.kill(this.legacyIncumbentPid, "SIGKILL")
|
|
741
1027
|
bridge.boundaryCrossed = true
|
|
742
1028
|
await bridge.incumbentControl?.closed()
|
|
743
1029
|
if (legacyCommitted) await legacyCommitted
|
|
744
1030
|
bridge.legacyGuardian.disconnect()
|
|
745
1031
|
await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
|
|
746
|
-
await writeState(this.statePath, bridge.recoverySnapshot)
|
|
747
1032
|
this.ownerTransition = /** @type {OwnerTransition} */ ({
|
|
748
1033
|
disruptive: true,
|
|
749
1034
|
mode: "legacy-first-upgrade",
|
|
@@ -775,10 +1060,16 @@ export default class RollbridgeDaemon {
|
|
|
775
1060
|
|
|
776
1061
|
/** @returns {Promise<void>} Exposes control commands and begins periodic state persistence. */
|
|
777
1062
|
async exposeControl() {
|
|
1063
|
+
if (this.ownerRetired) return
|
|
778
1064
|
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
779
1065
|
|
|
780
1066
|
await this.startControlServer()
|
|
781
1067
|
|
|
1068
|
+
if (this.ownerRetired) {
|
|
1069
|
+
await this.closeServer(this.controlServer)
|
|
1070
|
+
await this.removeControlSocket()
|
|
1071
|
+
return
|
|
1072
|
+
}
|
|
782
1073
|
if (this.stopping) {
|
|
783
1074
|
await this.closeServer(this.controlServer)
|
|
784
1075
|
await fs.rm(this.config.control.path, {force: true})
|
|
@@ -792,6 +1083,7 @@ export default class RollbridgeDaemon {
|
|
|
792
1083
|
* @returns {Promise<void>} Starts the stable local proxy.
|
|
793
1084
|
*/
|
|
794
1085
|
async startProxy() {
|
|
1086
|
+
if (this.ownerRetired) return
|
|
795
1087
|
const server = http.createServer((request, response) => this.proxyHttp(request, response))
|
|
796
1088
|
|
|
797
1089
|
server.on("upgrade", (request, socket, head) => this.proxyWebSocket(request, socket, head))
|
|
@@ -806,6 +1098,7 @@ export default class RollbridgeDaemon {
|
|
|
806
1098
|
resolve(undefined)
|
|
807
1099
|
})
|
|
808
1100
|
})
|
|
1101
|
+
if (this.ownerRetired) this.proxyClosePromise = this.closeServer(server)
|
|
809
1102
|
}
|
|
810
1103
|
|
|
811
1104
|
/**
|
|
@@ -814,10 +1107,12 @@ export default class RollbridgeDaemon {
|
|
|
814
1107
|
* @returns {Promise<void>} Starts the control socket.
|
|
815
1108
|
*/
|
|
816
1109
|
async startControlServer(socketPath = this.config.control.path, applyMetadata = true) {
|
|
1110
|
+
if (this.ownerRetired) return
|
|
817
1111
|
const server = net.createServer((socket) => this.handleControlSocket(socket))
|
|
818
1112
|
|
|
819
1113
|
this.controlServer = server
|
|
820
1114
|
await this.prepareControlSocketPath(socketPath)
|
|
1115
|
+
if (this.ownerRetired) return
|
|
821
1116
|
|
|
822
1117
|
await new Promise((resolve, reject) => {
|
|
823
1118
|
server.once("error", reject)
|
|
@@ -828,7 +1123,14 @@ export default class RollbridgeDaemon {
|
|
|
828
1123
|
resolve(undefined)
|
|
829
1124
|
})
|
|
830
1125
|
})
|
|
1126
|
+
const socketIdentity = await fs.lstat(socketPath)
|
|
831
1127
|
|
|
1128
|
+
this.boundControlIdentity = {dev: socketIdentity.dev, ino: socketIdentity.ino}
|
|
1129
|
+
if (this.ownerRetired) {
|
|
1130
|
+
await this.closeServer(server)
|
|
1131
|
+
await this.removeControlSocket()
|
|
1132
|
+
return
|
|
1133
|
+
}
|
|
832
1134
|
if (applyMetadata) await this.applyControlSocketMetadata(socketPath)
|
|
833
1135
|
}
|
|
834
1136
|
|
|
@@ -1035,6 +1337,15 @@ export default class RollbridgeDaemon {
|
|
|
1035
1337
|
}))
|
|
1036
1338
|
}
|
|
1037
1339
|
|
|
1340
|
+
if (commandName === "recover-generation-transition") {
|
|
1341
|
+
return await this.executeOwnerMutation("recover generation transition", async () => await this.recoverGenerationTransition({
|
|
1342
|
+
previousReleaseId: requiredString(data.previousReleaseId, "previousReleaseId"),
|
|
1343
|
+
releaseId: requiredString(data.releaseId, "releaseId"),
|
|
1344
|
+
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1345
|
+
revision: requiredString(data.revision, "revision")
|
|
1346
|
+
}))
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1038
1349
|
if (commandName === "status") {
|
|
1039
1350
|
return this.status()
|
|
1040
1351
|
}
|
|
@@ -1087,9 +1398,12 @@ export default class RollbridgeDaemon {
|
|
|
1087
1398
|
await this.guardian.commitOwnerReplacement(replacementId)
|
|
1088
1399
|
committed = true
|
|
1089
1400
|
await this.retireCommittedOwner(controlSocket)
|
|
1401
|
+
await this.beginRetiredListenerPublication(replacementId, this.listenerHandoff?.retired)
|
|
1090
1402
|
await this.guardian.finalizeOwnerReplacement(replacementId)
|
|
1091
|
-
|
|
1403
|
+
await this.completeRetiredListenerHandoff(replacementId)
|
|
1404
|
+
} catch (error) {
|
|
1092
1405
|
if (committed) this.guardian.disconnect()
|
|
1406
|
+
throw error
|
|
1093
1407
|
}
|
|
1094
1408
|
return {message: "owner replacement committed"}
|
|
1095
1409
|
}
|
|
@@ -1165,10 +1479,12 @@ export default class RollbridgeDaemon {
|
|
|
1165
1479
|
if (!this.guardian) throw new Error("Owner listener handoff requires the durable process guardian")
|
|
1166
1480
|
if (this.listenerHandoff) throw new Error("Owner listeners are already yielded to a replacement candidate")
|
|
1167
1481
|
await this.guardian.validateOwnerReplacement(replacementId)
|
|
1168
|
-
|
|
1482
|
+
if (!completionSocket) throw new Error("Owner listener handoff requires its authenticated control session")
|
|
1483
|
+
const retired = this.guardian.waitForEvent("replacement-retired")
|
|
1484
|
+
|
|
1485
|
+
this.listenerHandoff = {completionSocket, control, proxy, replacementId, retired}
|
|
1169
1486
|
this.listenerHandoffFailure = undefined
|
|
1170
1487
|
|
|
1171
|
-
if (!completionSocket) throw new Error("Owner listener handoff requires its authenticated control session")
|
|
1172
1488
|
for (const release of this.releases.values()) release.pauseDrainForOwnerHandoff()
|
|
1173
1489
|
if (proxy) this.proxyClosePromise = this.closeServer(this.proxyServer)
|
|
1174
1490
|
if (control) {
|
|
@@ -1176,22 +1492,27 @@ export default class RollbridgeDaemon {
|
|
|
1176
1492
|
for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
|
|
1177
1493
|
await this.removeControlSocket()
|
|
1178
1494
|
}
|
|
1495
|
+
/** @type {(sourceId: string, releaseId: string, connections: {http: number, websocket: number}) => void} */
|
|
1496
|
+
const publishConnections = (sourceId, releaseId, connections) => {
|
|
1497
|
+
if (completionSocket.destroyed) return
|
|
1498
|
+
completionSocket.write(`${JSON.stringify({connections, event: "owner-connection-state", releaseId, sourceId})}\n`)
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
for (const [sourceId, releases] of this.listenerConnectionSources) {
|
|
1502
|
+
for (const [releaseId, connections] of releases) publishConnections(sourceId, releaseId, connections)
|
|
1503
|
+
}
|
|
1179
1504
|
for (const release of this.releases.values()) {
|
|
1180
|
-
const
|
|
1181
|
-
if (completionSocket.destroyed) return
|
|
1182
|
-
completionSocket.write(`${JSON.stringify({
|
|
1183
|
-
connections: release.status().connections,
|
|
1184
|
-
event: "owner-connection-state",
|
|
1185
|
-
releaseId: release.releaseId
|
|
1186
|
-
})}\n`)
|
|
1187
|
-
}
|
|
1505
|
+
const connections = release.localConnections()
|
|
1188
1506
|
|
|
1189
|
-
|
|
1190
|
-
|
|
1507
|
+
if (release.state === "active" || release.state === "draining" || connections.http + connections.websocket > 0) {
|
|
1508
|
+
publishConnections(this.listenerSourceId, release.releaseId, connections)
|
|
1509
|
+
}
|
|
1510
|
+
if (connections.http + connections.websocket > 0) {
|
|
1511
|
+
release.once("drained", () => publishConnections(this.listenerSourceId, release.releaseId, release.localConnections()))
|
|
1512
|
+
}
|
|
1191
1513
|
}
|
|
1192
1514
|
|
|
1193
1515
|
const aborted = this.guardian.waitForEvent("replacement-aborted").then(async () => await this.resumeYieldedListeners())
|
|
1194
|
-
const retired = this.guardian.waitForEvent("replacement-retired")
|
|
1195
1516
|
|
|
1196
1517
|
void Promise.race([aborted, retired]).catch((error) => {
|
|
1197
1518
|
if (this.ownerRetired) return
|
|
@@ -1211,9 +1532,11 @@ export default class RollbridgeDaemon {
|
|
|
1211
1532
|
}
|
|
1212
1533
|
if (handoff.proxy) {
|
|
1213
1534
|
await this.startProxy()
|
|
1535
|
+
if (this.ownerRetired) return
|
|
1214
1536
|
}
|
|
1215
1537
|
if (handoff.control) {
|
|
1216
1538
|
await this.startControlServer()
|
|
1539
|
+
if (this.ownerRetired) return
|
|
1217
1540
|
}
|
|
1218
1541
|
this.listenerHandoff = undefined
|
|
1219
1542
|
this.listenerHandoffFailure = undefined
|
|
@@ -1282,56 +1605,27 @@ export default class RollbridgeDaemon {
|
|
|
1282
1605
|
|
|
1283
1606
|
const previousRelease = this.activeRelease
|
|
1284
1607
|
|
|
1285
|
-
if (nextConfig.processes.some((processConfig) => processConfig.lifecycle.activateCommand !== undefined)) {
|
|
1286
|
-
this.config = nextConfig
|
|
1287
|
-
this.releases.set(release.releaseId, release)
|
|
1288
|
-
const now = new Date().toISOString()
|
|
1289
|
-
|
|
1290
|
-
this.generationTransition = /** @type {GenerationTransition} */ ({
|
|
1291
|
-
candidateReleaseId: release.releaseId,
|
|
1292
|
-
candidateReleasePath: release.releasePath,
|
|
1293
|
-
candidateRevision: release.revision,
|
|
1294
|
-
configDigest: ownerConfigDigest(nextConfig),
|
|
1295
|
-
phase: "candidate_ready",
|
|
1296
|
-
previousReleaseId: previousRelease?.releaseId ?? null,
|
|
1297
|
-
startedAt: now,
|
|
1298
|
-
updatedAt: now
|
|
1299
|
-
})
|
|
1300
|
-
await this.checkpointGenerationTransition()
|
|
1301
|
-
return await this.resumeGenerationTransition()
|
|
1302
|
-
}
|
|
1303
|
-
|
|
1304
1608
|
this.config = nextConfig
|
|
1305
1609
|
this.releases.set(release.releaseId, release)
|
|
1306
|
-
release.
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
await this.replaceSingletons(release)
|
|
1326
|
-
|
|
1327
|
-
await this.persistState()
|
|
1328
|
-
await this.publishOwnerState()
|
|
1329
|
-
|
|
1330
|
-
return {
|
|
1331
|
-
activeReleaseId: release.releaseId,
|
|
1332
|
-
previousReleaseId: previousRelease ? previousRelease.releaseId : null,
|
|
1333
|
-
...(retirementFailure && previousRelease ? {retirement: {error: retirementFailure, releaseId: previousRelease.releaseId, status: "quiescence_failed"}} : {})
|
|
1334
|
-
}
|
|
1610
|
+
for (const serviceId of startedServices) this.serviceReleaseIds.set(serviceId, release.releaseId)
|
|
1611
|
+
|
|
1612
|
+
const activationLifecycle = nextConfig.processes.some((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
|
|
1613
|
+
const now = new Date().toISOString()
|
|
1614
|
+
|
|
1615
|
+
this.generationTransition = /** @type {GenerationTransition} */ ({
|
|
1616
|
+
activationLifecycle,
|
|
1617
|
+
candidateReleaseId: release.releaseId,
|
|
1618
|
+
candidateReleasePath: release.releasePath,
|
|
1619
|
+
candidateRevision: release.revision,
|
|
1620
|
+
configDigest: ownerConfigDigest(nextConfig),
|
|
1621
|
+
journalRevision: 0,
|
|
1622
|
+
phase: activationLifecycle ? "candidate_ready" : "activating_candidate",
|
|
1623
|
+
previousReleaseId: previousRelease?.releaseId ?? null,
|
|
1624
|
+
startedAt: now,
|
|
1625
|
+
updatedAt: now
|
|
1626
|
+
})
|
|
1627
|
+
await this.checkpointGenerationTransition()
|
|
1628
|
+
return await this.resumeGenerationTransition()
|
|
1335
1629
|
}
|
|
1336
1630
|
|
|
1337
1631
|
/**
|
|
@@ -1342,11 +1636,16 @@ export default class RollbridgeDaemon {
|
|
|
1342
1636
|
const transition = this.generationTransition
|
|
1343
1637
|
|
|
1344
1638
|
if (!transition) throw new Error("No release generation transition to resume")
|
|
1639
|
+
if (transition.phase === "restoring_previous" || transition.phase === "retiring_failed_candidate") {
|
|
1640
|
+
return await this.compensatePreCommitActivationFailure()
|
|
1641
|
+
}
|
|
1345
1642
|
const release = this.releases.get(transition.candidateReleaseId)
|
|
1346
1643
|
const previousRelease = transition.previousReleaseId ? this.releases.get(transition.previousReleaseId) : undefined
|
|
1347
1644
|
|
|
1348
1645
|
if (!release) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
|
|
1349
|
-
if (transition.previousReleaseId && !previousRelease) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
1646
|
+
if (transition.previousReleaseId && !previousRelease && transition.phase !== "restoring_committed") throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
1647
|
+
const activationLifecycle = transition.activationLifecycle ?? release.config.processes.some((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
|
|
1648
|
+
let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
|
|
1350
1649
|
|
|
1351
1650
|
if (transition.phase === "candidate_ready") {
|
|
1352
1651
|
if (previousRelease) await this.updateGenerationTransition("retiring_previous")
|
|
@@ -1381,7 +1680,23 @@ export default class RollbridgeDaemon {
|
|
|
1381
1680
|
|
|
1382
1681
|
await this.failGenerationTransition(failure)
|
|
1383
1682
|
this.logger("release generation activation failed", {error: failure, releaseId: release.releaseId})
|
|
1384
|
-
throw error
|
|
1683
|
+
if (!previousRelease || !activationLifecycle) throw error
|
|
1684
|
+
|
|
1685
|
+
let compensation
|
|
1686
|
+
|
|
1687
|
+
try {
|
|
1688
|
+
compensation = await this.compensatePreCommitActivationFailure()
|
|
1689
|
+
} catch (compensationError) {
|
|
1690
|
+
throw new AggregateError(
|
|
1691
|
+
[error, compensationError],
|
|
1692
|
+
`${failure}; pre-commit compensation failed: ${compensationError instanceof Error ? compensationError.message : String(compensationError)}`,
|
|
1693
|
+
{cause: compensationError}
|
|
1694
|
+
)
|
|
1695
|
+
}
|
|
1696
|
+
const compensatedError = /** @type {Error & {compensation?: Record<string, JsonValue>}} */ (new Error(`${failure}; compensation restored incumbent ${compensation.previousReleaseId} as authoritative and retired failed candidate ${compensation.candidateReleaseId}`, {cause: error}))
|
|
1697
|
+
|
|
1698
|
+
compensatedError.compensation = compensation
|
|
1699
|
+
throw compensatedError
|
|
1385
1700
|
}
|
|
1386
1701
|
|
|
1387
1702
|
// Activation acknowledgement and the logical proxy commit deliberately share one
|
|
@@ -1391,6 +1706,7 @@ export default class RollbridgeDaemon {
|
|
|
1391
1706
|
this.activeRelease = release
|
|
1392
1707
|
transition.phase = "committed_pending"
|
|
1393
1708
|
transition.error = undefined
|
|
1709
|
+
transition.journalRevision = (transition.journalRevision ?? 0) + 1
|
|
1394
1710
|
transition.updatedAt = new Date().toISOString()
|
|
1395
1711
|
this.logger("traffic switched", {previousReleaseId: previousRelease?.releaseId ?? null, releaseId: release.releaseId})
|
|
1396
1712
|
}
|
|
@@ -1406,8 +1722,18 @@ export default class RollbridgeDaemon {
|
|
|
1406
1722
|
}
|
|
1407
1723
|
|
|
1408
1724
|
if (transition.phase === "committed_pending") {
|
|
1725
|
+
await this.refreshServiceDefinitions(release)
|
|
1726
|
+
if (!activationLifecycle && previousRelease?.state === "active") {
|
|
1727
|
+
try {
|
|
1728
|
+
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1729
|
+
|
|
1730
|
+
await previousRelease.beginRetirement(retirementConfig)
|
|
1731
|
+
} catch (error) {
|
|
1732
|
+
retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
|
|
1733
|
+
this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1409
1736
|
await this.checkpointGenerationTransition()
|
|
1410
|
-
this.refreshServiceDefinitions(release)
|
|
1411
1737
|
if (previousRelease) {
|
|
1412
1738
|
const retirementConfig = previousRelease.config
|
|
1413
1739
|
|
|
@@ -1421,11 +1747,166 @@ export default class RollbridgeDaemon {
|
|
|
1421
1747
|
}
|
|
1422
1748
|
transition.phase = "committed"
|
|
1423
1749
|
transition.error = undefined
|
|
1750
|
+
transition.journalRevision = (transition.journalRevision ?? 0) + 1
|
|
1424
1751
|
transition.updatedAt = new Date().toISOString()
|
|
1752
|
+
this.pruneStoppedReleases()
|
|
1425
1753
|
await this.checkpointGenerationTransition()
|
|
1426
1754
|
}
|
|
1427
1755
|
|
|
1428
|
-
return {
|
|
1756
|
+
return {
|
|
1757
|
+
activeReleaseId: release.releaseId,
|
|
1758
|
+
previousReleaseId: previousRelease?.releaseId ?? null,
|
|
1759
|
+
...(retirementFailure && previousRelease ? {retirement: {error: retirementFailure, releaseId: previousRelease.releaseId, status: "quiescence_failed"}} : {})
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
/**
|
|
1764
|
+
* Restores the authoritative proxy target and retires only the failed candidate.
|
|
1765
|
+
* Every external effect is preceded by a durable phase checkpoint so recovery can
|
|
1766
|
+
* safely replay the paired lifecycle hooks, which are already idempotent contracts.
|
|
1767
|
+
* @returns {Promise<Record<string, JsonValue>>} Structured recovery result.
|
|
1768
|
+
*/
|
|
1769
|
+
async compensatePreCommitActivationFailure() {
|
|
1770
|
+
const transition = this.generationTransition
|
|
1771
|
+
|
|
1772
|
+
if (!transition) throw new Error("No release generation transition to compensate")
|
|
1773
|
+
if (transition.phase !== "activating_candidate" && transition.phase !== "restoring_previous" && transition.phase !== "retiring_failed_candidate") {
|
|
1774
|
+
throw new Error(`Generation transition ${transition.candidateReleaseId} at ${transition.phase} is not safe for pre-commit compensation`)
|
|
1775
|
+
}
|
|
1776
|
+
if (!transition.previousReleaseId) throw new Error("Pre-commit compensation requires a retained previous release")
|
|
1777
|
+
const candidate = this.releases.get(transition.candidateReleaseId)
|
|
1778
|
+
const previous = this.releases.get(transition.previousReleaseId)
|
|
1779
|
+
|
|
1780
|
+
if (!candidate) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
|
|
1781
|
+
if (!previous) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
1782
|
+
if (this.activeRelease !== previous) throw new Error(`Previous release ${previous.releaseId} is not the authoritative proxy target`)
|
|
1783
|
+
|
|
1784
|
+
if (transition.phase === "activating_candidate") {
|
|
1785
|
+
if (!transition.error && !transition.activationError) throw new Error("Candidate activation has no recorded failure to compensate")
|
|
1786
|
+
transition.activationError = transition.activationError || transition.error
|
|
1787
|
+
transition.compensationError = undefined
|
|
1788
|
+
await this.updateGenerationTransition("retiring_failed_candidate")
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
if (transition.phase === "retiring_failed_candidate") {
|
|
1792
|
+
try {
|
|
1793
|
+
await candidate.beginRetirement(candidate.config, {retry: true})
|
|
1794
|
+
const candidateServiceIds = [...this.serviceReleaseIds.entries()]
|
|
1795
|
+
.filter(([, releaseId]) => releaseId === candidate.releaseId)
|
|
1796
|
+
.map(([serviceId]) => serviceId)
|
|
1797
|
+
|
|
1798
|
+
await this.stopStartedServices(candidateServiceIds)
|
|
1799
|
+
} catch (error) {
|
|
1800
|
+
const failure = error instanceof Error ? error.message : String(error)
|
|
1801
|
+
|
|
1802
|
+
transition.compensationError = failure
|
|
1803
|
+
await this.failGenerationTransition(`Failed candidate ${candidate.releaseId} retirement failed: ${failure}`)
|
|
1804
|
+
this.logger("release generation compensation candidate retirement failed", {
|
|
1805
|
+
activationError: transition.activationError,
|
|
1806
|
+
error: failure,
|
|
1807
|
+
releaseId: candidate.releaseId
|
|
1808
|
+
})
|
|
1809
|
+
throw new Error(`failed candidate ${candidate.releaseId} retirement failed: ${failure}`, {cause: error})
|
|
1810
|
+
}
|
|
1811
|
+
transition.compensationError = undefined
|
|
1812
|
+
await this.updateGenerationTransition("restoring_previous")
|
|
1813
|
+
this.logger("release generation compensation candidate retired", {releaseId: candidate.releaseId})
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
if (transition.phase === "restoring_previous") {
|
|
1817
|
+
if (candidate.state !== "draining") throw new Error(`Failed candidate ${candidate.releaseId} is not retired before incumbent restoration`)
|
|
1818
|
+
try {
|
|
1819
|
+
await previous.reactivateGeneration()
|
|
1820
|
+
} catch (error) {
|
|
1821
|
+
const failure = error instanceof Error ? error.message : String(error)
|
|
1822
|
+
|
|
1823
|
+
transition.compensationError = failure
|
|
1824
|
+
await this.failGenerationTransition(`Incumbent ${previous.releaseId} restoration failed: ${failure}`)
|
|
1825
|
+
this.logger("release generation compensation restoration failed", {
|
|
1826
|
+
activationError: transition.activationError,
|
|
1827
|
+
error: failure,
|
|
1828
|
+
releaseId: previous.releaseId
|
|
1829
|
+
})
|
|
1830
|
+
throw new Error(`incumbent ${previous.releaseId} restoration failed: ${failure}`, {cause: error})
|
|
1831
|
+
}
|
|
1832
|
+
this.activeRelease = previous
|
|
1833
|
+
transition.compensationError = undefined
|
|
1834
|
+
this.logger("release generation compensation incumbent restored", {releaseId: previous.releaseId})
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
const result = /** @type {Record<string, JsonValue>} */ ({
|
|
1838
|
+
activeReleaseId: previous.releaseId,
|
|
1839
|
+
activationError: transition.activationError,
|
|
1840
|
+
candidateReleaseId: candidate.releaseId,
|
|
1841
|
+
failedCandidateStatus: candidate.state,
|
|
1842
|
+
previousReleaseId: previous.releaseId,
|
|
1843
|
+
recoveryStatus: "recovered"
|
|
1844
|
+
})
|
|
1845
|
+
|
|
1846
|
+
this.config = previous.config
|
|
1847
|
+
this.generationTransition = undefined
|
|
1848
|
+
try {
|
|
1849
|
+
await this.checkpointGenerationTransition()
|
|
1850
|
+
} catch (error) {
|
|
1851
|
+
const failure = error instanceof Error ? error.message : String(error)
|
|
1852
|
+
|
|
1853
|
+
this.generationTransition = transition
|
|
1854
|
+
transition.compensationError = failure
|
|
1855
|
+
transition.error = `Compensation checkpoint clear failed: ${failure}`
|
|
1856
|
+
transition.journalRevision = (transition.journalRevision ?? 0) + 1
|
|
1857
|
+
transition.updatedAt = new Date().toISOString()
|
|
1858
|
+
await this.publishOwnerState().catch((publishError) => {
|
|
1859
|
+
this.logger("release generation compensation fence republish failed", {
|
|
1860
|
+
error: publishError instanceof Error ? publishError.message : String(publishError),
|
|
1861
|
+
releaseId: candidate.releaseId
|
|
1862
|
+
})
|
|
1863
|
+
})
|
|
1864
|
+
this.logger("release generation compensation checkpoint failed", {error: failure, releaseId: candidate.releaseId})
|
|
1865
|
+
throw new Error(`compensation checkpoint clear failed: ${failure}`, {cause: error})
|
|
1866
|
+
}
|
|
1867
|
+
void this.drainAndPrune(candidate, candidate.config)
|
|
1868
|
+
this.logger("release generation compensation completed", result)
|
|
1869
|
+
return result
|
|
1870
|
+
}
|
|
1871
|
+
|
|
1872
|
+
/**
|
|
1873
|
+
* Authenticated control recovery for one exact failed pre-commit transition.
|
|
1874
|
+
* @param {{previousReleaseId: string, releaseId: string, releasePath: string, revision: string}} identity - Exact transition fence.
|
|
1875
|
+
* @returns {Promise<Record<string, JsonValue>>} Structured recovery result.
|
|
1876
|
+
*/
|
|
1877
|
+
async recoverGenerationTransition(identity) {
|
|
1878
|
+
const transition = this.generationTransition
|
|
1879
|
+
|
|
1880
|
+
if (!transition) {
|
|
1881
|
+
if (this.activeRelease?.releaseId !== identity.previousReleaseId || this.activeRelease.releaseId === identity.releaseId) {
|
|
1882
|
+
throw new Error("No matching recovered generation transition; the expected incumbent is not authoritative")
|
|
1883
|
+
}
|
|
1884
|
+
return {
|
|
1885
|
+
activeReleaseId: this.activeRelease.releaseId,
|
|
1886
|
+
candidateReleaseId: identity.releaseId,
|
|
1887
|
+
previousReleaseId: identity.previousReleaseId,
|
|
1888
|
+
recoveryStatus: "already_recovered"
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
if (transition.previousReleaseId !== identity.previousReleaseId) {
|
|
1892
|
+
throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
|
|
1893
|
+
}
|
|
1894
|
+
if (transition.phase !== "activating_candidate" && transition.phase !== "restoring_previous" && transition.phase !== "retiring_failed_candidate") {
|
|
1895
|
+
throw new Error(`Generation transition ${transition.candidateReleaseId} at ${transition.phase} is not a safe failed pre-commit transition`)
|
|
1896
|
+
}
|
|
1897
|
+
const nextConfig = this.configPath ? await loadConfig(this.configPath) : this.config
|
|
1898
|
+
|
|
1899
|
+
this.assertReloadCompatible(nextConfig)
|
|
1900
|
+
this.assertExactGenerationTransition(transition, {
|
|
1901
|
+
config: nextConfig,
|
|
1902
|
+
releaseId: identity.releaseId,
|
|
1903
|
+
releasePath: identity.releasePath,
|
|
1904
|
+
revision: identity.revision
|
|
1905
|
+
})
|
|
1906
|
+
if (this.activeRelease?.releaseId !== transition.previousReleaseId) {
|
|
1907
|
+
throw new Error(`Previous release ${transition.previousReleaseId} is not the authoritative proxy target`)
|
|
1908
|
+
}
|
|
1909
|
+
return await this.compensatePreCommitActivationFailure()
|
|
1429
1910
|
}
|
|
1430
1911
|
|
|
1431
1912
|
/**
|
|
@@ -1474,6 +1955,11 @@ export default class RollbridgeDaemon {
|
|
|
1474
1955
|
})
|
|
1475
1956
|
|
|
1476
1957
|
if (stillRetiring) throw new Error(`Committed generation daemon process ${stillRetiring.id} is still retiring; exact bootstrap recovery will retry after it stops`)
|
|
1958
|
+
for (const serviceId of this.services.keys()) {
|
|
1959
|
+
const serviceReleaseId = this.serviceReleaseIds.get(serviceId)
|
|
1960
|
+
|
|
1961
|
+
if (serviceReleaseId !== release.releaseId) throw new Error(`Committed generation service ${serviceId} belongs to retained release ${serviceReleaseId}`)
|
|
1962
|
+
}
|
|
1477
1963
|
for (const [singletonId, singletonReleaseId] of this.singletonReleaseIds) {
|
|
1478
1964
|
if (singletonReleaseId !== release.releaseId) throw new Error(`Committed generation singleton ${singletonId} belongs to retained release ${singletonReleaseId}`)
|
|
1479
1965
|
}
|
|
@@ -1501,10 +1987,14 @@ export default class RollbridgeDaemon {
|
|
|
1501
1987
|
|
|
1502
1988
|
try {
|
|
1503
1989
|
for (const processInstance of this.services.values()) {
|
|
1990
|
+
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
1504
1991
|
await processInstance.start("deploy")
|
|
1505
1992
|
}
|
|
1993
|
+
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
1506
1994
|
await release.restartCommittedGeneration()
|
|
1995
|
+
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
1507
1996
|
await release.activateGeneration()
|
|
1997
|
+
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
1508
1998
|
} catch (error) {
|
|
1509
1999
|
await Promise.allSettled([
|
|
1510
2000
|
release.abortCommittedGenerationRestart(),
|
|
@@ -1519,6 +2009,7 @@ export default class RollbridgeDaemon {
|
|
|
1519
2009
|
if (!this.generationTransition) throw new Error("No release generation transition to update")
|
|
1520
2010
|
this.generationTransition.phase = phase
|
|
1521
2011
|
this.generationTransition.error = undefined
|
|
2012
|
+
this.generationTransition.journalRevision = (this.generationTransition.journalRevision ?? 0) + 1
|
|
1522
2013
|
this.generationTransition.updatedAt = new Date().toISOString()
|
|
1523
2014
|
await this.checkpointGenerationTransition()
|
|
1524
2015
|
}
|
|
@@ -1527,6 +2018,7 @@ export default class RollbridgeDaemon {
|
|
|
1527
2018
|
async failGenerationTransition(error) {
|
|
1528
2019
|
if (!this.generationTransition) throw new Error("No release generation transition to fail")
|
|
1529
2020
|
this.generationTransition.error = error
|
|
2021
|
+
this.generationTransition.journalRevision = (this.generationTransition.journalRevision ?? 0) + 1
|
|
1530
2022
|
this.generationTransition.updatedAt = new Date().toISOString()
|
|
1531
2023
|
await this.checkpointGenerationTransition()
|
|
1532
2024
|
}
|
|
@@ -1559,19 +2051,143 @@ export default class RollbridgeDaemon {
|
|
|
1559
2051
|
* @param {net.Socket | undefined} completionSocket - Commit response connection.
|
|
1560
2052
|
*/
|
|
1561
2053
|
async retireCommittedOwner(completionSocket) {
|
|
2054
|
+
if (!this.committedRetirementPromise) this.committedRetirementPromise = this.performCommittedOwnerRetirement(completionSocket)
|
|
2055
|
+
await this.committedRetirementPromise
|
|
2056
|
+
}
|
|
2057
|
+
|
|
2058
|
+
/** @param {net.Socket | undefined} completionSocket - Commit response connection. */
|
|
2059
|
+
async performCommittedOwnerRetirement(completionSocket) {
|
|
1562
2060
|
this.ownerRetired = true
|
|
2061
|
+
this.controlCommandsReady = false
|
|
1563
2062
|
if (this.persistTimer) clearInterval(this.persistTimer)
|
|
1564
2063
|
this.persistTimer = undefined
|
|
1565
2064
|
this.persistenceEnabled = false
|
|
1566
2065
|
if (this.pendingWrite) await this.pendingWrite
|
|
1567
2066
|
this.stateCleanupEnabled = false
|
|
1568
2067
|
this.controlClosePromise = this.closeServer(this.controlServer)
|
|
2068
|
+
this.proxyClosePromise = this.closeServer(this.proxyServer)
|
|
1569
2069
|
for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
|
|
1570
2070
|
await this.removeControlSocket()
|
|
1571
|
-
void this.closeServer(this.proxyServer)
|
|
1572
2071
|
this.logger("owner authority transferred", {activeReleaseId: this.activeRelease?.releaseId ?? null})
|
|
1573
2072
|
}
|
|
1574
2073
|
|
|
2074
|
+
/** @param {string} replacementId - Prepared same-authority transaction. */
|
|
2075
|
+
async yieldControlLessOwnerListeners(replacementId) {
|
|
2076
|
+
if (!this.guardian) throw new Error("Control-less owner listener handoff requires the durable process guardian")
|
|
2077
|
+
this.proxyClosePromise = this.closeServer(this.proxyServer)
|
|
2078
|
+
await this.beginRetiredListenerPublication(replacementId)
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
/**
|
|
2082
|
+
* @param {string} replacementId - Transaction receiving durable source updates.
|
|
2083
|
+
* @param {Promise<Record<string, JsonValue>>} [retired] - Existing direct-listener retirement event.
|
|
2084
|
+
*/
|
|
2085
|
+
async beginRetiredListenerPublication(replacementId, retired) {
|
|
2086
|
+
if (!this.guardian) throw new Error("Retired listener publication requires the durable process guardian")
|
|
2087
|
+
if (this.retiredListenerHandoff) throw new Error("Another retired listener handoff is already pending")
|
|
2088
|
+
const guardian = this.guardian
|
|
2089
|
+
const releases = [...this.releases.values()]
|
|
2090
|
+
|
|
2091
|
+
this.retiredListenerHandoff = {
|
|
2092
|
+
releases,
|
|
2093
|
+
replacementId,
|
|
2094
|
+
replacementRetired: false,
|
|
2095
|
+
retired: retired || guardian.waitForEvent("replacement-retired")
|
|
2096
|
+
}
|
|
2097
|
+
this.retiredReplacementId = replacementId
|
|
2098
|
+
for (const release of releases) {
|
|
2099
|
+
if (release.status().connectionCount > 0) {
|
|
2100
|
+
release.once("drained", () => {
|
|
2101
|
+
if (this.retiredReplacementId !== replacementId) return
|
|
2102
|
+
const publication = this.publishRetiredConnectionState(replacementId, this.listenerSourceId, release.releaseId, release.localConnections(), true)
|
|
2103
|
+
|
|
2104
|
+
void publication.then(async () => {
|
|
2105
|
+
await this.retiredConnectionPublication
|
|
2106
|
+
this.disconnectRetiredOwnerIfDrained()
|
|
2107
|
+
}).catch((error) => {
|
|
2108
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
2109
|
+
this.logger("retired owner connection-state publication failed", {error: this.listenerHandoffFailure.message, releaseId: release.releaseId, replacementId})
|
|
2110
|
+
guardian.disconnect()
|
|
2111
|
+
})
|
|
2112
|
+
})
|
|
2113
|
+
}
|
|
2114
|
+
}
|
|
2115
|
+
for (const [sourceId, sourceReleases] of this.listenerConnectionSources) {
|
|
2116
|
+
for (const [releaseId, connections] of sourceReleases) {
|
|
2117
|
+
await this.publishRetiredConnectionState(replacementId, sourceId, releaseId, connections)
|
|
2118
|
+
}
|
|
2119
|
+
}
|
|
2120
|
+
for (const release of releases) {
|
|
2121
|
+
const connections = release.localConnections()
|
|
2122
|
+
|
|
2123
|
+
if (release.state === "active" || release.state === "draining" || connections.http + connections.websocket > 0) {
|
|
2124
|
+
await this.publishRetiredConnectionState(replacementId, this.listenerSourceId, release.releaseId, connections, true)
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
await guardian.completeOwnerListenerRetirement(replacementId)
|
|
2128
|
+
}
|
|
2129
|
+
|
|
2130
|
+
/** Resumes the incumbent listener after a prepared candidate aborts before commit. */
|
|
2131
|
+
async resumeControlLessOwnerListeners() {
|
|
2132
|
+
const handoff = this.retiredListenerHandoff
|
|
2133
|
+
|
|
2134
|
+
if (!handoff || this.ownerRetired) return
|
|
2135
|
+
void handoff.retired.catch(() => undefined)
|
|
2136
|
+
this.retiredListenerHandoff = undefined
|
|
2137
|
+
this.retiredReplacementId = undefined
|
|
2138
|
+
for (const release of handoff.releases) {
|
|
2139
|
+
release.resumeDrainAfterOwnerHandoff()
|
|
2140
|
+
if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
2141
|
+
}
|
|
2142
|
+
await this.startProxy()
|
|
2143
|
+
this.listenerHandoffFailure = undefined
|
|
2144
|
+
this.logger("control-less owner listeners resumed", {replacementId: handoff.replacementId})
|
|
2145
|
+
}
|
|
2146
|
+
|
|
2147
|
+
/** @param {string} replacementId - Committed same-authority transaction. */
|
|
2148
|
+
async completeControlLessOwnerRetirement(replacementId) {
|
|
2149
|
+
const handoff = this.retiredListenerHandoff
|
|
2150
|
+
|
|
2151
|
+
if (!handoff || handoff.replacementId !== replacementId) throw new Error("Committed owner retirement does not match the prepared listener handoff")
|
|
2152
|
+
await this.retireCommittedOwner(undefined)
|
|
2153
|
+
await this.completeRetiredListenerHandoff(replacementId)
|
|
2154
|
+
}
|
|
2155
|
+
|
|
2156
|
+
/** @param {string} replacementId - Finalized retired-listener transaction. */
|
|
2157
|
+
async completeRetiredListenerHandoff(replacementId) {
|
|
2158
|
+
const handoff = this.retiredListenerHandoff
|
|
2159
|
+
|
|
2160
|
+
if (!handoff || handoff.replacementId !== replacementId) throw new Error("Retired listener completion does not match its prepared handoff")
|
|
2161
|
+
await handoff.retired
|
|
2162
|
+
handoff.replacementRetired = true
|
|
2163
|
+
this.disconnectRetiredOwnerIfDrained()
|
|
2164
|
+
}
|
|
2165
|
+
|
|
2166
|
+
/** Disconnects the retired daemon only after every retained listener source drains. */
|
|
2167
|
+
disconnectRetiredOwnerIfDrained() {
|
|
2168
|
+
const handoff = this.retiredListenerHandoff
|
|
2169
|
+
|
|
2170
|
+
if (handoff?.replacementRetired && handoff.releases.every((release) => release.status().connectionCount === 0)) this.guardian?.disconnect()
|
|
2171
|
+
}
|
|
2172
|
+
|
|
2173
|
+
/**
|
|
2174
|
+
* Serializes one retired-listener source update over the authenticated guardian channel.
|
|
2175
|
+
* @param {string} replacementId - Committed replacement transaction.
|
|
2176
|
+
* @param {string} sourceId - Stable physical listener source.
|
|
2177
|
+
* @param {string} releaseId - Retained release.
|
|
2178
|
+
* @param {{http: number, websocket: number}} connections - Exact source counts.
|
|
2179
|
+
* @param {boolean} [localSource] - Whether this daemon physically owns the source.
|
|
2180
|
+
*/
|
|
2181
|
+
async publishRetiredConnectionState(replacementId, sourceId, releaseId, connections, localSource = false) {
|
|
2182
|
+
if (!this.guardian) throw new Error("Retired listener state publication requires the durable process guardian")
|
|
2183
|
+
const guardian = this.guardian
|
|
2184
|
+
|
|
2185
|
+
this.retiredConnectionPublication = this.retiredConnectionPublication.then(async () => {
|
|
2186
|
+
await guardian.publishOwnerConnectionState(replacementId, sourceId, releaseId, connections, localSource)
|
|
2187
|
+
})
|
|
2188
|
+
await this.retiredConnectionPublication
|
|
2189
|
+
}
|
|
2190
|
+
|
|
1575
2191
|
/**
|
|
1576
2192
|
* Rejects config changes that require rebinding daemon-owned resources or changing process topology.
|
|
1577
2193
|
* @param {import("./config.js").RollbridgeConfig} nextConfig - Freshly loaded config.
|
|
@@ -1585,6 +2201,10 @@ export default class RollbridgeDaemon {
|
|
|
1585
2201
|
if (!isDeepStrictEqual(nextConfig.control, this.config.control)) restartRequired.push("control")
|
|
1586
2202
|
if (nextConfig.statePath !== this.config.statePath) restartRequired.push("statePath")
|
|
1587
2203
|
if (!isDeepStrictEqual(nextConfig.ownerRecovery, this.config.ownerRecovery)) restartRequired.push("ownerRecovery")
|
|
2204
|
+
const activationProcessIds = this.config.processes.filter((processConfig) => processConfig.lifecycle.activateCommand !== undefined).map((processConfig) => processConfig.id)
|
|
2205
|
+
const nextActivationProcessIds = nextConfig.processes.filter((processConfig) => processConfig.lifecycle.activateCommand !== undefined).map((processConfig) => processConfig.id)
|
|
2206
|
+
|
|
2207
|
+
if (!isDeepStrictEqual(nextActivationProcessIds, activationProcessIds)) restartRequired.push("processes.lifecycle.activateCommand")
|
|
1588
2208
|
|
|
1589
2209
|
if (nextConfig.proxy.host !== this.config.proxy.host) restartRequired.push("proxy.host")
|
|
1590
2210
|
if (nextConfig.proxy.port !== this.config.proxy.port) restartRequired.push("proxy.port")
|
|
@@ -1687,6 +2307,7 @@ export default class RollbridgeDaemon {
|
|
|
1687
2307
|
startedServices.push(processConfig.id)
|
|
1688
2308
|
} catch (error) {
|
|
1689
2309
|
this.services.delete(processConfig.id)
|
|
2310
|
+
this.serviceReleaseIds.delete(processConfig.id)
|
|
1690
2311
|
delete this.servicePorts[processConfig.id]
|
|
1691
2312
|
throw error
|
|
1692
2313
|
}
|
|
@@ -1708,6 +2329,7 @@ export default class RollbridgeDaemon {
|
|
|
1708
2329
|
|
|
1709
2330
|
await service.stop()
|
|
1710
2331
|
this.services.delete(serviceId)
|
|
2332
|
+
this.serviceReleaseIds.delete(serviceId)
|
|
1711
2333
|
const port = this.servicePorts[serviceId]
|
|
1712
2334
|
|
|
1713
2335
|
if (port !== undefined) this.portReservations.delete(port)
|
|
@@ -1718,9 +2340,9 @@ export default class RollbridgeDaemon {
|
|
|
1718
2340
|
/**
|
|
1719
2341
|
* Updates daemon-wide service restart templates after a successful deploy.
|
|
1720
2342
|
* @param {ReleaseGroup} release - Active release.
|
|
1721
|
-
* @returns {void}
|
|
2343
|
+
* @returns {Promise<void>} Resolves once every persistent service definition is committed.
|
|
1722
2344
|
*/
|
|
1723
|
-
refreshServiceDefinitions(release) {
|
|
2345
|
+
async refreshServiceDefinitions(release) {
|
|
1724
2346
|
for (const processConfig of this.config.processes) {
|
|
1725
2347
|
if (processConfig.policy !== "service") continue
|
|
1726
2348
|
|
|
@@ -1729,8 +2351,16 @@ export default class RollbridgeDaemon {
|
|
|
1729
2351
|
if (!service) continue
|
|
1730
2352
|
|
|
1731
2353
|
const nextDefinition = release.buildProcess(processConfig, {shouldRestart: () => !this.stopping})
|
|
2354
|
+
const previousReleaseId = this.serviceReleaseIds.get(processConfig.id)
|
|
1732
2355
|
|
|
1733
|
-
|
|
2356
|
+
this.serviceReleaseIds.set(processConfig.id, release.releaseId)
|
|
2357
|
+
try {
|
|
2358
|
+
await service.updateDefinition(nextDefinition, this.guardian ? this.transferableOwnerState() : undefined)
|
|
2359
|
+
} catch (error) {
|
|
2360
|
+
if (previousReleaseId) this.serviceReleaseIds.set(processConfig.id, previousReleaseId)
|
|
2361
|
+
else this.serviceReleaseIds.delete(processConfig.id)
|
|
2362
|
+
throw error
|
|
2363
|
+
}
|
|
1734
2364
|
}
|
|
1735
2365
|
}
|
|
1736
2366
|
|
|
@@ -1822,6 +2452,7 @@ export default class RollbridgeDaemon {
|
|
|
1822
2452
|
*/
|
|
1823
2453
|
runningInstances(processConfig) {
|
|
1824
2454
|
if (processConfig.policy === "service") {
|
|
2455
|
+
if (processConfig.deployStrategy === "handoff") return this.activeRelease?.getProcesses(processConfig.id) || []
|
|
1825
2456
|
const service = this.services.get(processConfig.id)
|
|
1826
2457
|
|
|
1827
2458
|
return service ? [{id: processConfig.id, process: service}] : []
|
|
@@ -1870,6 +2501,15 @@ export default class RollbridgeDaemon {
|
|
|
1870
2501
|
}
|
|
1871
2502
|
}
|
|
1872
2503
|
|
|
2504
|
+
/** @returns {Set<string>} Release definitions owned by an unresolved durable transition. */
|
|
2505
|
+
generationTransitionReleaseIds() {
|
|
2506
|
+
const transition = this.generationTransition
|
|
2507
|
+
|
|
2508
|
+
return new Set(transition && transition.phase !== "committed"
|
|
2509
|
+
? [transition.candidateReleaseId, ...(transition.previousReleaseId ? [transition.previousReleaseId] : [])]
|
|
2510
|
+
: [])
|
|
2511
|
+
}
|
|
2512
|
+
|
|
1873
2513
|
/**
|
|
1874
2514
|
* Drains and stops a retired release in the background, then prunes stopped releases.
|
|
1875
2515
|
* @param {ReleaseGroup} release - Release to drain and stop.
|
|
@@ -1892,9 +2532,11 @@ export default class RollbridgeDaemon {
|
|
|
1892
2532
|
|
|
1893
2533
|
/** @returns {void} Removes stopped releases beyond the retention policy. */
|
|
1894
2534
|
pruneStoppedReleases() {
|
|
2535
|
+
const serviceOwnerReleaseIds = new Set(this.serviceReleaseIds.values())
|
|
1895
2536
|
const singletonOwnerReleaseIds = new Set(this.singletonReleaseIds.values())
|
|
2537
|
+
const transitionReleaseIds = this.generationTransitionReleaseIds()
|
|
1896
2538
|
const statuses = [...this.releases.values()]
|
|
1897
|
-
.filter((release) => !singletonOwnerReleaseIds.has(release.releaseId))
|
|
2539
|
+
.filter((release) => !serviceOwnerReleaseIds.has(release.releaseId) && !singletonOwnerReleaseIds.has(release.releaseId) && !transitionReleaseIds.has(release.releaseId))
|
|
1898
2540
|
.map((release) => release.status())
|
|
1899
2541
|
|
|
1900
2542
|
for (const releaseId of releasesToPrune(statuses, this.config.releaseRetention, Date.now())) {
|
|
@@ -1930,6 +2572,7 @@ export default class RollbridgeDaemon {
|
|
|
1930
2572
|
...status,
|
|
1931
2573
|
events,
|
|
1932
2574
|
persistedAt: new Date().toISOString(),
|
|
2575
|
+
serviceReleaseIds: Object.fromEntries(this.serviceReleaseIds),
|
|
1933
2576
|
...(this.hasActivationLifecycle() ? {singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds)} : {}),
|
|
1934
2577
|
...(this.guardianIdentity ? {recovery: {
|
|
1935
2578
|
configDigest: this.ownerRecoveryConfigDigest(),
|
|
@@ -1943,8 +2586,8 @@ export default class RollbridgeDaemon {
|
|
|
1943
2586
|
// clearing the file — otherwise a write started before shutdown could recreate it afterward.
|
|
1944
2587
|
this.pendingWrite = Promise.resolve(this.pendingWrite)
|
|
1945
2588
|
.catch(() => {})
|
|
1946
|
-
.then(() => writeState(statePath, snapshot))
|
|
1947
2589
|
.then(() => this.publishOwnerState())
|
|
2590
|
+
.then(() => writeState(statePath, snapshot))
|
|
1948
2591
|
.catch((error) => {
|
|
1949
2592
|
this.logger("state persist failed", {error: error instanceof Error ? error.message : String(error)})
|
|
1950
2593
|
if (throwOnError) throw error
|
|
@@ -1990,6 +2633,7 @@ export default class RollbridgeDaemon {
|
|
|
1990
2633
|
* @returns {Promise<void>} Resolves when owned resources are stopped (and, by default, control connections close).
|
|
1991
2634
|
*/
|
|
1992
2635
|
async shutdown({completionSocket, waitForControlConnections = true} = {}) {
|
|
2636
|
+
this.assertNoGenerationTransitionRecovery("shut down")
|
|
1993
2637
|
if (!this.shutdownPromise) this.shutdownPromise = this.performShutdown(completionSocket)
|
|
1994
2638
|
|
|
1995
2639
|
await this.shutdownPromise
|
|
@@ -2003,11 +2647,17 @@ export default class RollbridgeDaemon {
|
|
|
2003
2647
|
* @returns {Promise<void>} Resolves once a replacement can exclusively bind listeners.
|
|
2004
2648
|
*/
|
|
2005
2649
|
async retireOwner({attestation, completionSocket}) {
|
|
2650
|
+
this.assertNoGenerationTransitionRecovery("retire owner")
|
|
2006
2651
|
if (this.retirementPromise) return await this.retirementPromise
|
|
2007
2652
|
this.retirementPromise = this.performOwnerRetirement(attestation, completionSocket)
|
|
2008
2653
|
return await this.retirementPromise
|
|
2009
2654
|
}
|
|
2010
2655
|
|
|
2656
|
+
/** @param {string} operation - Terminal owner operation. */
|
|
2657
|
+
assertNoGenerationTransitionRecovery(operation) {
|
|
2658
|
+
if (this.generationTransitionRecovery) throw new Error(`Cannot ${operation} while generation transition recovery is in progress`)
|
|
2659
|
+
}
|
|
2660
|
+
|
|
2011
2661
|
/**
|
|
2012
2662
|
* @param {string} attestation - Replacement boot attestation.
|
|
2013
2663
|
* @param {net.Socket | undefined} completionSocket - Requesting handoff connection.
|
|
@@ -2117,9 +2767,23 @@ export default class RollbridgeDaemon {
|
|
|
2117
2767
|
/** @returns {Promise<void>} Removes the configured control socket path. */
|
|
2118
2768
|
async removeControlSocket() {
|
|
2119
2769
|
if (!this.controlSocketOwned) return
|
|
2770
|
+
const socketPath = this.boundControlPath || this.config.control.path
|
|
2771
|
+
const expectedIdentity = this.boundControlIdentity
|
|
2772
|
+
let ownsCurrentPath = true
|
|
2120
2773
|
|
|
2121
|
-
|
|
2774
|
+
if (expectedIdentity) {
|
|
2775
|
+
try {
|
|
2776
|
+
const currentIdentity = await fs.lstat(socketPath)
|
|
2777
|
+
|
|
2778
|
+
ownsCurrentPath = currentIdentity.dev === expectedIdentity.dev && currentIdentity.ino === expectedIdentity.ino
|
|
2779
|
+
} catch (error) {
|
|
2780
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
2781
|
+
ownsCurrentPath = false
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2784
|
+
if (ownsCurrentPath) await fs.rm(socketPath, {force: true})
|
|
2122
2785
|
this.controlSocketOwned = false
|
|
2786
|
+
this.boundControlIdentity = undefined
|
|
2123
2787
|
this.boundControlPath = undefined
|
|
2124
2788
|
}
|
|
2125
2789
|
|
|
@@ -2145,7 +2809,9 @@ export default class RollbridgeDaemon {
|
|
|
2145
2809
|
// a cleared orphan must not reappear if the OS later recycles its pid for an unrelated process.
|
|
2146
2810
|
this.orphans = this.orphans.filter((orphan) => isProcessAlive(orphan.pid))
|
|
2147
2811
|
|
|
2812
|
+
const serviceOwnerReleaseIds = new Set(this.serviceReleaseIds.values())
|
|
2148
2813
|
const singletonOwnerReleaseIds = new Set(this.singletonReleaseIds.values())
|
|
2814
|
+
const transitionReleaseIds = this.generationTransitionReleaseIds()
|
|
2149
2815
|
|
|
2150
2816
|
return {
|
|
2151
2817
|
activeReleaseId: this.activeRelease ? this.activeRelease.releaseId : null,
|
|
@@ -2155,7 +2821,7 @@ export default class RollbridgeDaemon {
|
|
|
2155
2821
|
daemonPid: process.pid,
|
|
2156
2822
|
daemonRuntime: this.runtime ? {...this.runtime} : undefined,
|
|
2157
2823
|
generationTransition: this.generationTransition ? {...this.generationTransition} : undefined,
|
|
2158
|
-
ownerRecovery: this.guardian ? {configDigest: this.ownerRecoveryConfigDigest()} : undefined,
|
|
2824
|
+
ownerRecovery: this.guardian ? {configDigest: this.ownerRecoveryConfigDigest(), ready: this.ownerReady} : undefined,
|
|
2159
2825
|
ownerTransition: this.ownerTransition ? {...this.ownerTransition} : undefined,
|
|
2160
2826
|
orphans: [...this.orphans],
|
|
2161
2827
|
proxy: {
|
|
@@ -2164,13 +2830,15 @@ export default class RollbridgeDaemon {
|
|
|
2164
2830
|
upstreamHost: this.config.proxy.upstreamHost
|
|
2165
2831
|
},
|
|
2166
2832
|
releaseReferences: [...this.releases.values()]
|
|
2167
|
-
.filter((release) => release.state === "active" || release.state === "draining" ||
|
|
2833
|
+
.filter((release) => release.state === "active" || release.state === "draining" || serviceOwnerReleaseIds.has(release.releaseId) || singletonOwnerReleaseIds.has(release.releaseId) || transitionReleaseIds.has(release.releaseId))
|
|
2168
2834
|
.map((release) => ({releaseId: release.releaseId, releasePath: release.releasePath})),
|
|
2169
2835
|
releases: [...this.releases.values()].map((release) => release.status()),
|
|
2170
|
-
services: [...this.services.entries()]
|
|
2171
|
-
id
|
|
2172
|
-
|
|
2173
|
-
|
|
2836
|
+
services: [...this.services.entries()]
|
|
2837
|
+
.filter(([id]) => this.serviceReleaseIds.has(id))
|
|
2838
|
+
.map(([id, processInstance]) => ({
|
|
2839
|
+
id,
|
|
2840
|
+
process: processInstance.status()
|
|
2841
|
+
})),
|
|
2174
2842
|
singletons: [...this.singletons.entries()].map(([id, processInstance]) => ({
|
|
2175
2843
|
id,
|
|
2176
2844
|
process: processInstance.status()
|
|
@@ -2298,8 +2966,10 @@ async function verifyLegacyDaemonProcess(pid, configPath, socketPath) {
|
|
|
2298
2966
|
const args = await processArguments(pid, "legacy daemon")
|
|
2299
2967
|
const daemonIndex = args.indexOf("daemon")
|
|
2300
2968
|
const configIndex = args.indexOf("--config")
|
|
2969
|
+
const configuredPath = args[configIndex + 1]
|
|
2970
|
+
const configuredAbsolutePath = configuredPath ? path.resolve(await fs.readlink(`/proc/${pid}/cwd`), configuredPath) : undefined
|
|
2301
2971
|
|
|
2302
|
-
if (daemonIndex < 0 || configIndex < 0 ||
|
|
2972
|
+
if (daemonIndex < 0 || configIndex < 0 || configuredAbsolutePath !== path.resolve(configPath)) {
|
|
2303
2973
|
throw new Error(`Daemon PID ${pid} does not match the exact retained daemon config command`)
|
|
2304
2974
|
}
|
|
2305
2975
|
await verifyProcessUser(pid, "legacy daemon")
|