rollbridge 0.1.38 → 0.1.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +12 -1
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +16 -2
- package/changelog.d/20260830055159-guardian-daemon-restart.md +2 -0
- package/docs/cli.md +33 -9
- package/docs/config.md +37 -13
- package/docs/logging.md +4 -3
- package/docs/troubleshooting.md +7 -5
- package/package.json +1 -1
- package/src/cli.js +100 -26
- package/src/config.js +14 -6
- package/src/daemon.js +741 -153
- package/src/guardian-client.js +68 -16
- package/src/managed-process.js +55 -8
- package/src/process-guardian.js +731 -43
- package/src/release-group.js +77 -6
- package/test/config-validation.test.js +4 -0
- package/test/fixtures/guardian-recovery-owner.js +86 -0
- package/test/fixtures/pre-split3-process-guardian.js +14 -0
- package/test/guardian-client.test.js +1339 -62
- package/test/managed-process.test.js +136 -7
- package/test/owner-recovery.test.js +814 -52
- package/test/owner-replacement.test.js +526 -23
- package/test/release-group.test.js +19 -2
- package/test/release-runtime-retention.test.js +121 -4
- package/test/rollbridge.test.js +21 -0
- package/test/support/process.js +41 -0
package/src/daemon.js
CHANGED
|
@@ -4,6 +4,7 @@ import fs from "node:fs/promises"
|
|
|
4
4
|
import http from "node:http"
|
|
5
5
|
import net from "node:net"
|
|
6
6
|
import crypto from "node:crypto"
|
|
7
|
+
import path from "node:path"
|
|
7
8
|
import {isDeepStrictEqual} from "node:util"
|
|
8
9
|
import httpProxy from "http-proxy"
|
|
9
10
|
import {loadConfig} from "./config.js"
|
|
@@ -23,12 +24,12 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
23
24
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
24
25
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
25
26
|
* @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
26
|
-
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed"} GenerationTransitionPhase
|
|
27
|
-
* @typedef {{candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, configDigest: string, error?: string, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
28
|
-
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
27
|
+
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed" | "restoring_committed"} GenerationTransitionPhase
|
|
28
|
+
* @typedef {{activationLifecycle?: boolean, candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, configDigest: string, error?: string, journalRevision?: number, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
29
|
+
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string, ready: boolean} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
29
30
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
30
|
-
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
31
|
-
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
31
|
+
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
32
|
+
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
32
33
|
* @typedef {{boundaryCrossed: boolean, incumbentControl?: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, legacyGuardian: GuardianClient, legacyInventory: {key: string, provenance: string}[], legacyPrepared?: {ownerState: JsonValue, replacementId: string}, legacySnapshot: OwnerRecoverySnapshot, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
33
34
|
*/
|
|
34
35
|
|
|
@@ -40,12 +41,14 @@ export default class RollbridgeDaemon {
|
|
|
40
41
|
* @param {string} [args.configPath] - Config file path to reload before deploys.
|
|
41
42
|
* @param {(message: string, data?: Record<string, JsonValue>) => void} [args.logger] - Logger.
|
|
42
43
|
* @param {number} [args.legacyIncumbentPid] - Exact incumbent PID supplied by ensure-daemon.
|
|
44
|
+
* @param {{args: string[], cwd: string, env: Record<string, string>, executable: string, logPath?: string, pidPath?: string, startupTimeoutMs: number}} [args.recoveryCommand] - Exact command for guardian-owned daemon recovery.
|
|
43
45
|
* @param {import("./daemon-runtime.js").DaemonRuntimeIdentity} [args.runtime] - Immutable daemon runtime identity.
|
|
44
46
|
*/
|
|
45
|
-
constructor({bootstrap, config, configPath, legacyIncumbentPid, logger, runtime}) {
|
|
47
|
+
constructor({bootstrap, config, configPath, legacyIncumbentPid, logger, recoveryCommand, runtime}) {
|
|
46
48
|
this.bootstrap = bootstrap ? {...bootstrap} : undefined
|
|
47
49
|
this.config = config
|
|
48
50
|
this.configPath = configPath
|
|
51
|
+
this.recoveryCommand = recoveryCommand
|
|
49
52
|
this.runtime = runtime
|
|
50
53
|
this.legacyIncumbentPid = legacyIncumbentPid
|
|
51
54
|
this.eventLog = new EventLog(EVENT_HISTORY_LIMIT)
|
|
@@ -62,6 +65,7 @@ export default class RollbridgeDaemon {
|
|
|
62
65
|
|
|
63
66
|
this.releases = /** @type {Map<string, ReleaseGroup>} */ (new Map())
|
|
64
67
|
this.services = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
|
|
68
|
+
this.serviceReleaseIds = /** @type {Map<string, string>} */ (new Map())
|
|
65
69
|
this.servicePorts = /** @type {Record<string, number>} */ ({})
|
|
66
70
|
this.portReservations = /** @type {Set<number>} */ (new Set())
|
|
67
71
|
this.singletons = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
|
|
@@ -72,6 +76,7 @@ export default class RollbridgeDaemon {
|
|
|
72
76
|
this.proxyServer = /** @type {http.Server | undefined} */ (undefined)
|
|
73
77
|
this.controlServer = /** @type {net.Server | undefined} */ (undefined)
|
|
74
78
|
this.controlSocketOwned = false
|
|
79
|
+
this.boundControlIdentity = /** @type {{dev: number, ino: number} | undefined} */ (undefined)
|
|
75
80
|
this.boundControlPath = /** @type {string | undefined} */ (undefined)
|
|
76
81
|
this.controlSockets = /** @type {Set<net.Socket>} */ (new Set())
|
|
77
82
|
this.proxyPort = /** @type {number | undefined} */ (undefined)
|
|
@@ -83,17 +88,26 @@ export default class RollbridgeDaemon {
|
|
|
83
88
|
this.stateCleanupEnabled = false
|
|
84
89
|
this.shutdownPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
85
90
|
this.retirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
91
|
+
this.committedRetirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
86
92
|
this.ownerRetired = false
|
|
87
93
|
this.controlClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
88
94
|
this.proxyClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
89
|
-
this.listenerHandoff = /** @type {{control: boolean, proxy: boolean, replacementId: string} | undefined} */ (undefined)
|
|
95
|
+
this.listenerHandoff = /** @type {{completionSocket: net.Socket, control: boolean, proxy: boolean, replacementId: string, retired: Promise<Record<string, JsonValue>>} | undefined} */ (undefined)
|
|
90
96
|
this.listenerHandoffFailure = /** @type {Error | undefined} */ (undefined)
|
|
91
97
|
this.incumbentListenerControl = /** @type {Awaited<ReturnType<typeof openControlSession>> | undefined} */ (undefined)
|
|
98
|
+
this.incumbentListenerSourceIds = /** @type {WeakMap<object, string>} */ (new WeakMap())
|
|
99
|
+
this.listenerConnectionSources = /** @type {Map<string, Map<string, {http: number, websocket: number}>>} */ (new Map())
|
|
100
|
+
this.listenerSourceId = crypto.randomUUID()
|
|
101
|
+
this.retiredConnectionPublication = Promise.resolve()
|
|
102
|
+
this.retiredReplacementId = /** @type {string | undefined} */ (undefined)
|
|
103
|
+
this.retiredListenerHandoff = /** @type {{releases: ReleaseGroup[], replacementId: string, replacementRetired: boolean, retired: Promise<Record<string, JsonValue>>} | undefined} */ (undefined)
|
|
92
104
|
this.ownerTransition = /** @type {OwnerTransition | undefined} */ (undefined)
|
|
93
105
|
this.controlCommandsReady = true
|
|
94
106
|
this.startingReleases = /** @type {Set<ReleaseGroup>} */ (new Set())
|
|
95
107
|
this.guardian = /** @type {GuardianClient | undefined} */ (undefined)
|
|
96
108
|
this.guardianIdentity = /** @type {{pid?: number, socketPath: string, token: string} | undefined} */ (undefined)
|
|
109
|
+
this.ownerReady = false
|
|
110
|
+
this.generationTransitionRecovery = false
|
|
97
111
|
// Still-alive managed processes left by a previous daemon (from statePath), captured at
|
|
98
112
|
// startup and surfaced in status(). The daemon cannot re-manage them, only report them.
|
|
99
113
|
this.orphans = /** @type {{id: string, pid: number, releaseId: string | null}[]} */ ([])
|
|
@@ -109,8 +123,67 @@ export default class RollbridgeDaemon {
|
|
|
109
123
|
async start({exposeControl = true, reportOrphans = true} = {}) {
|
|
110
124
|
if (this.config.ownerRecovery) await this.initializeOwnerRecovery()
|
|
111
125
|
else if (reportOrphans) await this.reportOrphans()
|
|
126
|
+
if (this.ownerRetired) return
|
|
127
|
+
const transition = this.generationTransition
|
|
128
|
+
const recoveredReplacementProxy = this.guardian ? await this.finalizeRecoveredOwnerReplacement() : false
|
|
129
|
+
|
|
130
|
+
if (this.guardian && transition && transition.phase !== "committed" && !transition.error) {
|
|
131
|
+
this.generationTransitionRecovery = true
|
|
132
|
+
try {
|
|
133
|
+
await this.executeOwnerMutation("recover generation transition", async () => {
|
|
134
|
+
if (!recoveredReplacementProxy) await this.startProxy()
|
|
135
|
+
if (this.ownerRetired) return {}
|
|
136
|
+
if (exposeControl) await this.exposeControl()
|
|
137
|
+
if (this.ownerRetired) return {}
|
|
138
|
+
await this.markOwnerReady()
|
|
139
|
+
try {
|
|
140
|
+
await this.resumeGenerationTransition()
|
|
141
|
+
} catch (error) {
|
|
142
|
+
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
|
|
143
|
+
}
|
|
144
|
+
return {}
|
|
145
|
+
})
|
|
146
|
+
} finally {
|
|
147
|
+
this.generationTransitionRecovery = false
|
|
148
|
+
}
|
|
149
|
+
if (!this.ownerRetired) this.logger("release generation transition recovery settled", {phase: transition.phase, releaseId: transition.candidateReleaseId})
|
|
150
|
+
return
|
|
151
|
+
}
|
|
152
|
+
if (!recoveredReplacementProxy) await this.startProxy()
|
|
153
|
+
if (!this.ownerRetired && exposeControl) await this.exposeControl()
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** @returns {Promise<boolean>} Whether startup bound a proxy for a pending committed replacement. */
|
|
157
|
+
async finalizeRecoveredOwnerReplacement() {
|
|
158
|
+
if (!this.guardian) return false
|
|
159
|
+
let status = await this.guardian.replacementStatus()
|
|
160
|
+
|
|
161
|
+
if (status.retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
162
|
+
if (!status.retirementPending) return false
|
|
163
|
+
if (!status.committedReplacementId) throw new Error("Guardian reports listener retirement without a committed replacement")
|
|
164
|
+
if (!status.retirementReady) {
|
|
165
|
+
const settled = Promise.race([
|
|
166
|
+
this.guardian.waitForEvent("replacement-listeners-retired").then(() => undefined, (error) => error instanceof Error ? error : new Error(String(error))),
|
|
167
|
+
this.guardian.waitForEvent("replacement-retirement-failed").then((event) => new Error(stringOrUndefined(event.reason) || "Retired listener state transfer failed"), (error) => error instanceof Error ? error : new Error(String(error))),
|
|
168
|
+
this.guardian.waitForEvent("replacement-committed").then(() => undefined, (error) => error instanceof Error ? error : new Error(String(error)))
|
|
169
|
+
])
|
|
170
|
+
|
|
171
|
+
status = await this.guardian.replacementStatus()
|
|
172
|
+
if (status.retirementPending && !status.retirementReady) {
|
|
173
|
+
const settlementError = await settled
|
|
174
|
+
|
|
175
|
+
if (settlementError) throw settlementError
|
|
176
|
+
status = await this.guardian.replacementStatus()
|
|
177
|
+
}
|
|
178
|
+
if (!status.retirementPending) return false
|
|
179
|
+
if (!status.retirementReady) throw new Error("Guardian listener retirement did not publish complete state")
|
|
180
|
+
}
|
|
181
|
+
const replacementId = status.committedReplacementId
|
|
182
|
+
|
|
183
|
+
if (!replacementId) throw new Error("Guardian listener retirement lost its committed replacement")
|
|
112
184
|
await this.startProxy()
|
|
113
|
-
|
|
185
|
+
await this.guardian.finalizeOwnerReplacement(replacementId)
|
|
186
|
+
return true
|
|
114
187
|
}
|
|
115
188
|
|
|
116
189
|
/** Connects to the durable process guardian and reconstructs a matching persisted owner snapshot. */
|
|
@@ -119,13 +192,8 @@ export default class RollbridgeDaemon {
|
|
|
119
192
|
const state = await readState(this.statePath)
|
|
120
193
|
const snapshot = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
|
|
121
194
|
const recovery = snapshot?.recovery
|
|
122
|
-
const configDigest = this.ownerRecoveryConfigDigest()
|
|
123
195
|
|
|
124
196
|
if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
|
|
125
|
-
if (recovery && recovery.configDigest !== configDigest) throw new Error("Owner recovery config identity does not match the persisted owner; refusing cross-authority adoption.")
|
|
126
|
-
if (snapshot && ((this.runtime?.digest ?? null) !== (snapshot.daemonRuntime?.digest ?? null))) {
|
|
127
|
-
throw new Error("Owner recovery runtime identity does not match the persisted owner; use the exact same Rollbridge runtime.")
|
|
128
|
-
}
|
|
129
197
|
|
|
130
198
|
const guardianIdentity = recovery?.guardian || {
|
|
131
199
|
socketPath: `${this.statePath}.guardian.sock`,
|
|
@@ -133,49 +201,73 @@ export default class RollbridgeDaemon {
|
|
|
133
201
|
}
|
|
134
202
|
this.guardianIdentity = guardianIdentity
|
|
135
203
|
this.guardian = new GuardianClient(guardianIdentity)
|
|
136
|
-
if (recovery)
|
|
204
|
+
if (recovery) {
|
|
205
|
+
await this.guardian.connect()
|
|
206
|
+
await this.assertGuardianDaemonRecoveryCapability()
|
|
207
|
+
}
|
|
137
208
|
else {
|
|
138
209
|
await this.guardian.launch()
|
|
139
210
|
guardianIdentity.pid = this.guardian.pid
|
|
140
211
|
}
|
|
141
|
-
await this.guardian.claimOwner(this.config.ownerRecovery?.reconnectGraceMs ?? 30000, this.ownerAuthority())
|
|
142
212
|
this.watchOwnerReplacementEvents()
|
|
213
|
+
await this.guardian.claimOwner(this.config.ownerRecovery?.reconnectGraceMs ?? 30000, this.ownerAuthority())
|
|
143
214
|
|
|
144
215
|
if (snapshot) {
|
|
145
216
|
if (!Array.isArray(snapshot.releases) || (snapshot.activeReleaseId !== null && typeof snapshot.activeReleaseId !== "string")) {
|
|
146
217
|
throw new Error("Owner recovery state is partial or corrupt; active release metadata is required.")
|
|
147
218
|
}
|
|
148
|
-
|
|
149
|
-
let releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({})
|
|
150
|
-
let singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({})
|
|
151
|
-
let recoverySnapshot = snapshot
|
|
219
|
+
const ownerState = /** @type {PrivateOwnerState} */ (await this.guardian.ownerState())
|
|
152
220
|
|
|
153
|
-
if (
|
|
154
|
-
|
|
221
|
+
if (this.ownerRetired) return
|
|
222
|
+
if (!ownerState?.config || !ownerState.snapshot) throw new Error("Committed guardian published incomplete owner recovery state")
|
|
223
|
+
let recoverySnapshot = ownerState.snapshot
|
|
155
224
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
225
|
+
if ((recovery?.format ?? 0) >= 2) {
|
|
226
|
+
const releaseConfigs = ownerState.releaseConfigs
|
|
227
|
+
|
|
228
|
+
if (!releaseConfigs) throw new Error("Durable guardian state is missing exact release definitions; refusing partial owner recovery.")
|
|
229
|
+
const guardianTransition = ownerState.snapshot.generationTransition
|
|
230
|
+
const persistedTransition = snapshot.generationTransition
|
|
231
|
+
const sameTransition = Boolean(guardianTransition && persistedTransition &&
|
|
232
|
+
guardianTransition.candidateReleaseId === persistedTransition.candidateReleaseId &&
|
|
233
|
+
guardianTransition.candidateReleasePath === persistedTransition.candidateReleasePath &&
|
|
234
|
+
guardianTransition.candidateRevision === persistedTransition.candidateRevision &&
|
|
235
|
+
guardianTransition.configDigest === persistedTransition.configDigest &&
|
|
236
|
+
guardianTransition.previousReleaseId === persistedTransition.previousReleaseId &&
|
|
237
|
+
guardianTransition.startedAt === persistedTransition.startedAt)
|
|
238
|
+
const requiredReleaseIds = new Set([
|
|
239
|
+
...ownerState.snapshot.releaseReferences.map((reference) => reference.releaseId),
|
|
240
|
+
...snapshot.releaseReferences.map((reference) => reference.releaseId),
|
|
241
|
+
...Object.values(ownerState.serviceReleaseIds || {}),
|
|
242
|
+
...Object.values(snapshot.serviceReleaseIds || {}),
|
|
243
|
+
...Object.values(ownerState.singletonReleaseIds || {}),
|
|
244
|
+
...Object.values(snapshot.singletonReleaseIds || {}),
|
|
245
|
+
...(ownerState.snapshot.activeReleaseId ? [ownerState.snapshot.activeReleaseId] : []),
|
|
246
|
+
...(snapshot.activeReleaseId ? [snapshot.activeReleaseId] : []),
|
|
247
|
+
...(guardianTransition ? [guardianTransition.candidateReleaseId, ...(guardianTransition.previousReleaseId ? [guardianTransition.previousReleaseId] : [])] : []),
|
|
248
|
+
...(persistedTransition ? [persistedTransition.candidateReleaseId, ...(persistedTransition.previousReleaseId ? [persistedTransition.previousReleaseId] : [])] : [])
|
|
249
|
+
])
|
|
250
|
+
const publicReleasesComplete = [...requiredReleaseIds].every((releaseId) => snapshot.releases.some((release) => release.releaseId === releaseId) && releaseConfigs[releaseId])
|
|
251
|
+
|
|
252
|
+
if (sameTransition && publicReleasesComplete &&
|
|
253
|
+
typeof persistedTransition?.journalRevision === "number" &&
|
|
254
|
+
typeof guardianTransition?.journalRevision === "number" &&
|
|
255
|
+
persistedTransition.journalRevision > guardianTransition.journalRevision) {
|
|
256
|
+
recoverySnapshot = snapshot
|
|
167
257
|
}
|
|
168
258
|
}
|
|
169
|
-
await this.restoreOwnerState(recoverySnapshot, {
|
|
259
|
+
await this.restoreOwnerState(recoverySnapshot, {
|
|
260
|
+
config: ownerState.config,
|
|
261
|
+
listenerConnectionSources: ownerState.listenerConnectionSources,
|
|
262
|
+
releaseConfigs: ownerState.releaseConfigs,
|
|
263
|
+
resumeDrains: false,
|
|
264
|
+
serviceReleaseIds: recoverySnapshot === snapshot ? snapshot.serviceReleaseIds || ownerState.serviceReleaseIds : ownerState.serviceReleaseIds || snapshot.serviceReleaseIds,
|
|
265
|
+
singletonReleaseIds: recoverySnapshot === snapshot ? snapshot.singletonReleaseIds || ownerState.singletonReleaseIds : ownerState.singletonReleaseIds || snapshot.singletonReleaseIds
|
|
266
|
+
})
|
|
267
|
+
if (this.ownerRetired) return
|
|
170
268
|
this.persistenceEnabled = true
|
|
171
269
|
await this.guardian.reconcileInventory()
|
|
172
|
-
if (this.
|
|
173
|
-
try {
|
|
174
|
-
await this.resumeGenerationTransition()
|
|
175
|
-
} catch (error) {
|
|
176
|
-
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: this.generationTransition.candidateReleaseId})
|
|
177
|
-
}
|
|
178
|
-
}
|
|
270
|
+
if (this.ownerRetired) return
|
|
179
271
|
for (const release of this.releases.values()) {
|
|
180
272
|
if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
181
273
|
}
|
|
@@ -184,6 +276,7 @@ export default class RollbridgeDaemon {
|
|
|
184
276
|
this.persistenceEnabled = true
|
|
185
277
|
await this.persistState({throwOnError: true})
|
|
186
278
|
}
|
|
279
|
+
if (this.ownerRetired) return
|
|
187
280
|
this.stateCleanupEnabled = true
|
|
188
281
|
await this.publishOwnerState()
|
|
189
282
|
}
|
|
@@ -196,16 +289,46 @@ export default class RollbridgeDaemon {
|
|
|
196
289
|
/** Installs event-driven drain fencing for the next prepared owner transaction. */
|
|
197
290
|
watchOwnerReplacementEvents() {
|
|
198
291
|
if (!this.guardian) throw new Error("Owner replacement event fencing requires the durable guardian")
|
|
292
|
+
this.guardian.onEvent("owner-connection-state", (event) => this.handleIncumbentListenerEvent(event))
|
|
199
293
|
this.guardian.onEvent("replacement-prepared", () => {
|
|
200
294
|
for (const release of this.releases.values()) release.pauseDrainForOwnerHandoff()
|
|
201
295
|
})
|
|
202
296
|
this.guardian.onEvent("replacement-aborted", () => {
|
|
297
|
+
if (this.retiredListenerHandoff) {
|
|
298
|
+
void this.resumeControlLessOwnerListeners().catch((error) => {
|
|
299
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
300
|
+
this.logger("control-less owner listener handoff recovery failed", {error: this.listenerHandoffFailure.message})
|
|
301
|
+
})
|
|
302
|
+
return
|
|
303
|
+
}
|
|
203
304
|
if (this.listenerHandoff || this.ownerRetired) return
|
|
204
305
|
for (const release of this.releases.values()) {
|
|
205
306
|
release.resumeDrainAfterOwnerHandoff()
|
|
206
307
|
if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
207
308
|
}
|
|
208
309
|
})
|
|
310
|
+
this.guardian.onEvent("replacement-listener-handoff-requested", (event) => {
|
|
311
|
+
if (this.ownerRetired) return
|
|
312
|
+
const replacementId = stringOrUndefined(event.replacementId)
|
|
313
|
+
|
|
314
|
+
if (!replacementId) throw new Error("Owner listener handoff request is missing its replacement id")
|
|
315
|
+
void this.yieldControlLessOwnerListeners(replacementId).catch((error) => {
|
|
316
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
317
|
+
this.logger("control-less owner listener handoff failed", {error: this.listenerHandoffFailure.message, replacementId})
|
|
318
|
+
this.guardian?.disconnect()
|
|
319
|
+
})
|
|
320
|
+
})
|
|
321
|
+
this.guardian.onEvent("replacement-retirement-requested", (event) => {
|
|
322
|
+
if (this.ownerRetired) return
|
|
323
|
+
const replacementId = stringOrUndefined(event.replacementId)
|
|
324
|
+
|
|
325
|
+
if (!replacementId) throw new Error("Committed owner retirement request is missing its replacement id")
|
|
326
|
+
void this.completeControlLessOwnerRetirement(replacementId).catch((error) => {
|
|
327
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
328
|
+
this.logger("committed owner retirement failed", {error: this.listenerHandoffFailure.message, replacementId})
|
|
329
|
+
this.guardian?.disconnect()
|
|
330
|
+
})
|
|
331
|
+
})
|
|
209
332
|
}
|
|
210
333
|
|
|
211
334
|
/** @returns {{configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}} Exact authority fence. */
|
|
@@ -217,26 +340,34 @@ export default class RollbridgeDaemon {
|
|
|
217
340
|
* @param {OwnerRecoverySnapshot} snapshot - Validated persisted owner state.
|
|
218
341
|
* @param {object} [options] - Recovery definition options.
|
|
219
342
|
* @param {import("./config.js").RollbridgeConfig} [options.config] - Owner config for daemon-wide processes.
|
|
343
|
+
* @param {Record<string, Record<string, {http: number, websocket: number}>>} [options.listenerConnectionSources] - Exact inherited listener sources.
|
|
220
344
|
* @param {Record<string, import("./config.js").RollbridgeConfig>} [options.releaseConfigs] - Exact generation configs.
|
|
221
345
|
* @param {boolean} [options.resumeDrains] - Whether to resume draining generations immediately.
|
|
346
|
+
* @param {Record<string, string>} [options.serviceReleaseIds] - Exact release owner for each persistent service definition.
|
|
222
347
|
* @param {Record<string, string>} [options.singletonReleaseIds] - Exact release owner for each singleton registration.
|
|
223
348
|
* @param {boolean} [options.synchronizeLifecycleRoles] - Whether the caller owns guardian role mutation authority.
|
|
224
349
|
*/
|
|
225
|
-
async restoreOwnerState(snapshot, {config = this.config, releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({}), resumeDrains = true, singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({}), synchronizeLifecycleRoles = true} = {}) {
|
|
350
|
+
async restoreOwnerState(snapshot, {config = this.config, listenerConnectionSources = /** @type {Record<string, Record<string, {http: number, websocket: number}>>} */ ({}), releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({}), resumeDrains = true, serviceReleaseIds = snapshot.serviceReleaseIds || /** @type {Record<string, string>} */ ({}), singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({}), synchronizeLifecycleRoles = true} = {}) {
|
|
226
351
|
if (!Array.isArray(snapshot.releases) || (snapshot.activeReleaseId !== null && typeof snapshot.activeReleaseId !== "string")) {
|
|
227
352
|
throw new Error("Owner recovery state is partial or corrupt; active release metadata is required.")
|
|
228
353
|
}
|
|
229
354
|
this.generationTransition = snapshot.generationTransition ? {...snapshot.generationTransition} : undefined
|
|
230
355
|
if (snapshot.activeReleaseId === null && snapshot.releases.length === 0) return
|
|
231
|
-
this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
|
|
356
|
+
if (!this.bootstrap) this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
|
|
232
357
|
this.ownerTransition = snapshot.ownerTransition ? {...snapshot.ownerTransition} : undefined
|
|
358
|
+
const fallbackDefinitionReleaseId = snapshot.activeReleaseId || snapshot.releases.at(-1)?.releaseId
|
|
359
|
+
const serviceOwnerReleaseIds = new Set(Object.values(serviceReleaseIds))
|
|
233
360
|
const singletonOwnerReleaseIds = new Set(Object.values(singletonReleaseIds))
|
|
361
|
+
const transitionReleaseIds = this.generationTransitionReleaseIds()
|
|
362
|
+
|
|
363
|
+
if (snapshot.services.some((service) => !serviceReleaseIds[service.id]) && fallbackDefinitionReleaseId) serviceOwnerReleaseIds.add(fallbackDefinitionReleaseId)
|
|
234
364
|
|
|
235
365
|
for (const releaseStatus of snapshot.releases) {
|
|
236
|
-
const
|
|
366
|
+
const transitionOwner = transitionReleaseIds.has(releaseStatus.releaseId)
|
|
367
|
+
const serviceOwner = serviceOwnerReleaseIds.has(releaseStatus.releaseId)
|
|
237
368
|
const singletonOwner = singletonOwnerReleaseIds.has(releaseStatus.releaseId)
|
|
238
369
|
|
|
239
|
-
if (releaseStatus.state !== "active" && releaseStatus.state !== "draining" && !
|
|
370
|
+
if (releaseStatus.state !== "active" && releaseStatus.state !== "draining" && !transitionOwner && !serviceOwner && !singletonOwner) continue
|
|
240
371
|
const release = new ReleaseGroup({
|
|
241
372
|
config: releaseConfigs[releaseStatus.releaseId] || config,
|
|
242
373
|
logger: this.logger,
|
|
@@ -253,21 +384,34 @@ export default class RollbridgeDaemon {
|
|
|
253
384
|
this.releases.set(release.releaseId, release)
|
|
254
385
|
if (release.releaseId === snapshot.activeReleaseId) this.activeRelease = release
|
|
255
386
|
}
|
|
387
|
+
for (const [sourceId, releases] of Object.entries(listenerConnectionSources)) {
|
|
388
|
+
for (const [releaseId, connections] of Object.entries(releases)) {
|
|
389
|
+
this.setListenerConnectionSource(sourceId, releaseId, connections)
|
|
390
|
+
}
|
|
391
|
+
}
|
|
256
392
|
|
|
257
393
|
if (snapshot.activeReleaseId !== null && !this.activeRelease) throw new Error(`Owner recovery state does not contain active release ${snapshot.activeReleaseId}.`)
|
|
258
|
-
const
|
|
394
|
+
const committedBootstrapRelease = this.committedBootstrapRelease()
|
|
395
|
+
const definitionRelease = this.activeRelease || committedBootstrapRelease || [...this.releases.values()].at(-1)
|
|
259
396
|
if (!definitionRelease) throw new Error("Owner recovery state has no release definition for owned processes.")
|
|
260
|
-
if (!this.activeRelease && snapshot.singletons.
|
|
397
|
+
if (!this.activeRelease && !committedBootstrapRelease && snapshot.singletons.some((singleton) => !singletonReleaseIds[singleton.id])) {
|
|
398
|
+
throw new Error("Owner recovery state has release-owned singletons without an exact release identity.")
|
|
399
|
+
}
|
|
261
400
|
for (const serviceStatus of snapshot.services) {
|
|
262
|
-
const
|
|
401
|
+
const serviceReleaseId = serviceReleaseIds[serviceStatus.id] || fallbackDefinitionReleaseId
|
|
402
|
+
const serviceRelease = serviceReleaseId ? this.releases.get(serviceReleaseId) : undefined
|
|
403
|
+
|
|
404
|
+
if (!serviceRelease) throw new Error(`Owner recovery state contains service ${serviceStatus.id} for an unknown release.`)
|
|
405
|
+
const processConfig = serviceRelease.config.processes.find((candidate) => candidate.id === serviceStatus.id && candidate.policy === "service" && candidate.deployStrategy !== "handoff")
|
|
263
406
|
|
|
264
|
-
if (!processConfig) throw new Error(`Owner recovery state contains unknown service ${serviceStatus.id}.`)
|
|
265
|
-
const service =
|
|
407
|
+
if (!processConfig) throw new Error(`Owner recovery state contains unknown service ${serviceStatus.id} for release ${serviceReleaseId}.`)
|
|
408
|
+
const service = serviceRelease.buildProcess(processConfig, {guardianKey: `service:${serviceStatus.id}`, shouldRestart: () => !this.stopping})
|
|
266
409
|
|
|
267
410
|
await this.recoverGuardianProcess(service)
|
|
268
411
|
this.services.set(serviceStatus.id, service)
|
|
269
|
-
|
|
270
|
-
|
|
412
|
+
this.serviceReleaseIds.set(serviceStatus.id, serviceRelease.releaseId)
|
|
413
|
+
if (serviceRelease.ports[serviceStatus.id] !== undefined) {
|
|
414
|
+
const port = serviceRelease.ports[serviceStatus.id]
|
|
271
415
|
|
|
272
416
|
if (this.portReservations.has(port)) throw new Error(`Persisted daemon service ${serviceStatus.id} port ${port} is already reserved by a live generation`)
|
|
273
417
|
this.portReservations.add(port)
|
|
@@ -297,13 +441,34 @@ export default class RollbridgeDaemon {
|
|
|
297
441
|
/** Publishes full normalized definitions only over the authenticated guardian channel. */
|
|
298
442
|
async publishOwnerState() {
|
|
299
443
|
if (!this.guardian) return
|
|
300
|
-
await this.guardian.publishOwnerState(
|
|
444
|
+
await this.guardian.publishOwnerState(this.transferableOwnerState())
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/** Confirms guardian-owned startup only after every requested listener is available. */
|
|
448
|
+
async markOwnerReady() {
|
|
449
|
+
if (!this.guardian) return
|
|
450
|
+
await this.guardian.ownerReady()
|
|
451
|
+
this.ownerReady = true
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/** @returns {Record<string, JsonValue>} Complete private guardian owner state. */
|
|
455
|
+
transferableOwnerState() {
|
|
456
|
+
return {
|
|
301
457
|
authority: this.ownerAuthority(),
|
|
302
458
|
config: this.config,
|
|
459
|
+
recovery: this.recoveryCommand ? {
|
|
460
|
+
command: this.recoveryCommand,
|
|
461
|
+
reconnectGraceMs: this.config.ownerRecovery?.reconnectGraceMs ?? 30000,
|
|
462
|
+
startupTimeoutMs: this.recoveryCommand.startupTimeoutMs
|
|
463
|
+
} : undefined,
|
|
464
|
+
listenerConnectionSources: this.serializedListenerConnectionSources(),
|
|
465
|
+
listenerSourceId: this.listenerSourceId,
|
|
303
466
|
releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
|
|
467
|
+
retiredListenerHandoff: 2,
|
|
468
|
+
serviceReleaseIds: Object.fromEntries(this.serviceReleaseIds),
|
|
304
469
|
singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds),
|
|
305
470
|
snapshot: this.status()
|
|
306
|
-
}
|
|
471
|
+
}
|
|
307
472
|
}
|
|
308
473
|
|
|
309
474
|
/**
|
|
@@ -338,12 +503,17 @@ export default class RollbridgeDaemon {
|
|
|
338
503
|
preparedStatus = await this.guardian.replacementStatus()
|
|
339
504
|
transfer = /** @type {PrivateOwnerState} */ (prepared.ownerState)
|
|
340
505
|
if (!transfer?.config || !transfer.snapshot) throw new Error("Committed owner published incomplete replacement state")
|
|
506
|
+
const unresolvedTransition = transfer.snapshot.generationTransition
|
|
507
|
+
|
|
508
|
+
if (unresolvedTransition && unresolvedTransition.phase !== "committed" && unresolvedTransition.configDigest !== this.ownerRecoveryConfigDigest()) {
|
|
509
|
+
throw new Error(`Owner replacement cannot change config authority while unresolved generation transition ${unresolvedTransition.candidateReleaseId} remains at ${unresolvedTransition.phase}`)
|
|
510
|
+
}
|
|
341
511
|
const registeredProcesses = new Map((await this.guardian.inventory()).map(({key, provenance}) => [key, provenance]))
|
|
342
512
|
|
|
343
513
|
reservedProcessKey = legacyBridge ? undefined : reconstructableOwnerSnapshotProcessKeys(transfer.snapshot, transfer.singletonReleaseIds)
|
|
344
514
|
.find((key) => registeredProcesses.has(key))
|
|
345
515
|
if (reservedProcessKey) this.guardian.reserveProcessRecovery(reservedProcessKey, /** @type {string} */ (registeredProcesses.get(reservedProcessKey)))
|
|
346
|
-
await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, releaseConfigs: transfer.releaseConfigs, resumeDrains: false, singletonReleaseIds: transfer.singletonReleaseIds, synchronizeLifecycleRoles: false})
|
|
516
|
+
await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, listenerConnectionSources: transfer.listenerConnectionSources, releaseConfigs: transfer.releaseConfigs, resumeDrains: false, serviceReleaseIds: transfer.serviceReleaseIds, singletonReleaseIds: transfer.singletonReleaseIds, synchronizeLifecycleRoles: false})
|
|
347
517
|
} catch (error) {
|
|
348
518
|
legacyBridge?.incumbentControl?.close()
|
|
349
519
|
if (legacyBridge) {
|
|
@@ -357,6 +527,16 @@ export default class RollbridgeDaemon {
|
|
|
357
527
|
}
|
|
358
528
|
throw error
|
|
359
529
|
}
|
|
530
|
+
if (!legacyBridge) {
|
|
531
|
+
try {
|
|
532
|
+
await this.assertGuardianDaemonRecoveryCapability()
|
|
533
|
+
} catch (error) {
|
|
534
|
+
// The immediately preceding guardian protocol notifies the incumbent of
|
|
535
|
+
// an abort only when the prepared candidate disconnects.
|
|
536
|
+
this.guardian.disconnect()
|
|
537
|
+
throw error
|
|
538
|
+
}
|
|
539
|
+
}
|
|
360
540
|
for (const release of this.releases.values()) release.preserveConfigOnRetirement = true
|
|
361
541
|
this.logger("owner replacement candidate prepared", {activeReleaseId: this.activeRelease?.releaseId ?? null, replacementId: prepared.replacementId})
|
|
362
542
|
|
|
@@ -411,40 +591,61 @@ export default class RollbridgeDaemon {
|
|
|
411
591
|
listenersYielded = true
|
|
412
592
|
}
|
|
413
593
|
}
|
|
414
|
-
if (sharedFixedProxy) await this.startProxy()
|
|
594
|
+
if (sharedFixedProxy && !retiredIncumbentControl) await this.startProxy()
|
|
415
595
|
if (!retiredIncumbentControl) {
|
|
416
596
|
await fs.rename(stagingControlPath, this.config.control.path)
|
|
417
597
|
finalControlPublished = true
|
|
418
598
|
this.boundControlPath = this.config.control.path
|
|
419
599
|
}
|
|
420
|
-
|
|
421
|
-
() =>
|
|
600
|
+
const retirementFailure = this.guardian.waitForEvent("replacement-retirement-failed").then(
|
|
601
|
+
(event) => new Error(stringOrUndefined(event.reason) || "Retired listener state transfer failed"),
|
|
422
602
|
(error) => error instanceof Error ? error : new Error(String(error))
|
|
423
603
|
)
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
604
|
+
committed = Promise.race([
|
|
605
|
+
this.guardian.waitForEvent("replacement-committed").then(
|
|
606
|
+
() => undefined,
|
|
607
|
+
(error) => error instanceof Error ? error : new Error(String(error))
|
|
608
|
+
),
|
|
609
|
+
retirementFailure
|
|
610
|
+
])
|
|
611
|
+
const listenerRetirement = retiredIncumbentControl
|
|
612
|
+
? Promise.race([
|
|
613
|
+
this.guardian.waitForEvent("replacement-listeners-retired").then(() => undefined, (error) => error instanceof Error ? error : new Error(String(error))),
|
|
614
|
+
retirementFailure
|
|
615
|
+
])
|
|
616
|
+
: undefined
|
|
617
|
+
const staged = await this.guardian.stageOwnerReplacement(prepared.replacementId, this.transferableOwnerState())
|
|
431
618
|
|
|
432
619
|
if (staged.committed && reservedProcessKey) {
|
|
433
620
|
committedAuthority = true
|
|
434
621
|
await this.guardian.recoverReservedProcess(reservedProcessKey)
|
|
435
622
|
}
|
|
623
|
+
if (staged.committed && retiredIncumbentControl && sharedFixedProxy) await this.startProxy()
|
|
436
624
|
|
|
437
625
|
if (!staged.committed) {
|
|
438
626
|
if (retiredIncumbentControl) {
|
|
439
627
|
const processKey = reservedProcessKey
|
|
440
628
|
|
|
441
629
|
if (!processKey || !this.guardian.processes.has(processKey)) throw new Error("Retired owner replacement requires an exact reserved process from committed owner state")
|
|
630
|
+
if (transfer.retiredListenerHandoff !== 2) {
|
|
631
|
+
throw new Error("Cannot safely complete atomic owner replacement through the older retained guardian while the incumbent control socket is absent; incumbent owner and connections were preserved")
|
|
632
|
+
}
|
|
442
633
|
try {
|
|
634
|
+
await this.guardian.prepareRetiredOwnerListenerHandoff(prepared.replacementId, processKey)
|
|
635
|
+
const listenerRetirementError = await listenerRetirement
|
|
636
|
+
|
|
637
|
+
if (listenerRetirementError) throw listenerRetirementError
|
|
638
|
+
if (sharedFixedProxy) await this.startProxy()
|
|
443
639
|
await this.guardian.commitRetiredOwnerReplacement(prepared.replacementId, processKey)
|
|
444
640
|
committedAuthority = true
|
|
445
641
|
await this.guardian.recoverReservedProcess(processKey)
|
|
642
|
+
await this.guardian.finalizeOwnerReplacement(prepared.replacementId)
|
|
446
643
|
} catch (error) {
|
|
447
|
-
if (!(error instanceof Error) ||
|
|
644
|
+
if (!(error instanceof Error) || ![
|
|
645
|
+
"Guardian commit-retired-owner-replacement requires the committed owner",
|
|
646
|
+
"Guardian prepare-retired-owner-listener-handoff requires the committed owner",
|
|
647
|
+
"Unknown guardian command: prepare-retired-owner-listener-handoff"
|
|
648
|
+
].includes(error.message)) throw error
|
|
448
649
|
throw new Error(
|
|
449
650
|
"Cannot safely complete atomic owner replacement through the older retained guardian while the incumbent control socket is absent; incumbent owner and connections were preserved",
|
|
450
651
|
{cause: error}
|
|
@@ -454,10 +655,12 @@ export default class RollbridgeDaemon {
|
|
|
454
655
|
try {
|
|
455
656
|
if (!incumbentControl) throw new Error("Owner replacement incumbent control session is unavailable")
|
|
456
657
|
await incumbentControl.request({command: "commit-owner-replacement", replacementId: prepared.replacementId})
|
|
658
|
+
committedAuthority = true
|
|
457
659
|
} catch (error) {
|
|
458
660
|
const status = await this.guardian.replacementStatus()
|
|
459
661
|
|
|
460
662
|
if (status.committedReplacementId !== prepared.replacementId || !status.ownerClaimed) throw error
|
|
663
|
+
committedAuthority = true
|
|
461
664
|
this.logger("owner replacement commit response lost; guardian commit confirmed", {replacementId: prepared.replacementId})
|
|
462
665
|
}
|
|
463
666
|
}
|
|
@@ -534,33 +737,109 @@ export default class RollbridgeDaemon {
|
|
|
534
737
|
}
|
|
535
738
|
}
|
|
536
739
|
|
|
740
|
+
/** Requires the persistent guardian protocol which can restart an accepted daemon. */
|
|
741
|
+
async assertGuardianDaemonRecoveryCapability() {
|
|
742
|
+
if (!this.guardian) throw new Error("Owner recovery capability check requires the durable process guardian")
|
|
743
|
+
let capabilities
|
|
744
|
+
|
|
745
|
+
try {
|
|
746
|
+
capabilities = await this.guardian.capabilities()
|
|
747
|
+
} catch (error) {
|
|
748
|
+
if (!(error instanceof Error) || !["Guardian capabilities requires a process key", "Unknown guardian command: capabilities"].includes(error.message)) throw error
|
|
749
|
+
throw new Error("The persistent Rollbridge guardian predates daemon recovery. Perform one explicit clean Rollbridge shutdown and restart before retrying this package upgrade.", {cause: error})
|
|
750
|
+
}
|
|
751
|
+
if (capabilities.daemonRecovery !== 1) throw new Error("The persistent Rollbridge guardian does not support the required daemon recovery protocol")
|
|
752
|
+
}
|
|
753
|
+
|
|
537
754
|
/**
|
|
538
755
|
* Applies authenticated live-connection state from the retired listener owner.
|
|
539
756
|
* @param {Record<string, JsonValue>} event - Incumbent control event.
|
|
540
|
-
* @param {{close: () => void}} session - Exact incumbent session.
|
|
757
|
+
* @param {{close: () => void}} [session] - Exact incumbent control session.
|
|
541
758
|
*/
|
|
542
759
|
handleIncumbentListenerEvent(event, session) {
|
|
543
760
|
if (event.event !== "owner-connection-state") return
|
|
544
761
|
const releaseId = stringOrUndefined(event.releaseId)
|
|
762
|
+
let sourceId = stringOrUndefined(event.sourceId)
|
|
545
763
|
const connections = event.connections
|
|
546
764
|
|
|
547
|
-
if (!
|
|
765
|
+
if (!sourceId && session) {
|
|
766
|
+
sourceId = this.incumbentListenerSourceIds.get(session)
|
|
767
|
+
if (!sourceId) {
|
|
768
|
+
sourceId = crypto.randomUUID()
|
|
769
|
+
this.incumbentListenerSourceIds.set(session, sourceId)
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
if (!sourceId || !releaseId || !connections || typeof connections !== "object" || Array.isArray(connections)) {
|
|
548
773
|
throw new Error("Incumbent listener sent invalid connection state")
|
|
549
774
|
}
|
|
550
|
-
const
|
|
775
|
+
const normalized = {
|
|
551
776
|
http: requiredNonNegativeInteger(connections.http, "connections.http"),
|
|
552
777
|
websocket: requiredNonNegativeInteger(connections.websocket, "connections.websocket")
|
|
553
778
|
}
|
|
554
|
-
const release = this.releases.get(releaseId)
|
|
555
779
|
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
780
|
+
this.setListenerConnectionSource(sourceId, releaseId, normalized)
|
|
781
|
+
const handoffSocket = this.listenerHandoff?.completionSocket
|
|
782
|
+
|
|
783
|
+
if (handoffSocket && !handoffSocket.destroyed) {
|
|
784
|
+
handoffSocket.write(`${JSON.stringify({connections: normalized, event: "owner-connection-state", releaseId, sourceId})}\n`)
|
|
785
|
+
}
|
|
786
|
+
if (this.retiredReplacementId) {
|
|
787
|
+
void this.publishRetiredConnectionState(this.retiredReplacementId, sourceId, releaseId, normalized).catch((error) => {
|
|
788
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
789
|
+
this.logger("retired owner connection-state relay failed", {error: this.listenerHandoffFailure.message, releaseId, replacementId: this.retiredReplacementId, sourceId})
|
|
790
|
+
this.guardian?.disconnect()
|
|
791
|
+
})
|
|
792
|
+
}
|
|
793
|
+
if (session && this.incumbentListenerControl === session && ![...this.releases.values()].some((candidate) => candidate.hasTransferredConnections())) {
|
|
559
794
|
this.incumbentListenerControl = undefined
|
|
560
795
|
session.close()
|
|
561
796
|
}
|
|
562
797
|
}
|
|
563
798
|
|
|
799
|
+
/**
|
|
800
|
+
* Reconciles one physical listener source and applies its aggregate to the release.
|
|
801
|
+
* @param {string} sourceId - Stable listener source.
|
|
802
|
+
* @param {string} releaseId - Retained release.
|
|
803
|
+
* @param {{http: number, websocket: number}} connections - Exact source counts.
|
|
804
|
+
*/
|
|
805
|
+
setListenerConnectionSource(sourceId, releaseId, connections) {
|
|
806
|
+
const release = this.releases.get(releaseId)
|
|
807
|
+
|
|
808
|
+
if (!release && (connections.http > 0 || connections.websocket > 0)) throw new Error(`Incumbent listener reported unknown release ${releaseId}`)
|
|
809
|
+
const sourceReleases = this.listenerConnectionSources.get(sourceId) || new Map()
|
|
810
|
+
|
|
811
|
+
if (connections.http + connections.websocket === 0) sourceReleases.delete(releaseId)
|
|
812
|
+
else sourceReleases.set(releaseId, connections)
|
|
813
|
+
if (sourceReleases.size === 0) this.listenerConnectionSources.delete(sourceId)
|
|
814
|
+
else this.listenerConnectionSources.set(sourceId, sourceReleases)
|
|
815
|
+
if (!release) return
|
|
816
|
+
let http = 0
|
|
817
|
+
let websocket = 0
|
|
818
|
+
|
|
819
|
+
for (const candidateReleases of this.listenerConnectionSources.values()) {
|
|
820
|
+
const counts = candidateReleases.get(releaseId)
|
|
821
|
+
|
|
822
|
+
if (!counts) continue
|
|
823
|
+
http += counts.http
|
|
824
|
+
websocket += counts.websocket
|
|
825
|
+
}
|
|
826
|
+
release.setTransferredConnections({http, websocket})
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
/** @returns {Record<string, Record<string, {http: number, websocket: number}>>} Exact live listener sources. */
|
|
830
|
+
serializedListenerConnectionSources() {
|
|
831
|
+
const sources = new Map([...this.listenerConnectionSources].map(([sourceId, releases]) => [sourceId, new Map(releases)]))
|
|
832
|
+
const local = new Map()
|
|
833
|
+
|
|
834
|
+
for (const release of this.releases.values()) {
|
|
835
|
+
const connections = release.localConnections()
|
|
836
|
+
|
|
837
|
+
if (connections.http + connections.websocket > 0) local.set(release.releaseId, connections)
|
|
838
|
+
}
|
|
839
|
+
if (local.size > 0) sources.set(this.listenerSourceId, local)
|
|
840
|
+
return Object.fromEntries([...sources].map(([sourceId, releases]) => [sourceId, Object.fromEntries(releases)]))
|
|
841
|
+
}
|
|
842
|
+
|
|
564
843
|
/**
|
|
565
844
|
* Authenticates and prepares the one-time disruptive bridge for an exact legacy guardian protocol.
|
|
566
845
|
* @param {{persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
|
|
@@ -618,12 +897,14 @@ export default class RollbridgeDaemon {
|
|
|
618
897
|
|
|
619
898
|
assertLegacyIncumbentStatus(incumbentStatus, persisted)
|
|
620
899
|
}
|
|
900
|
+
const definitionReleaseId = persisted.activeReleaseId || persisted.releases.at(-1)?.releaseId
|
|
621
901
|
const ownerState = legacyPrepared
|
|
622
902
|
? legacyPrepared.ownerState
|
|
623
903
|
: {
|
|
624
904
|
authority: persistedAuthority,
|
|
625
905
|
config: this.config,
|
|
626
906
|
releaseConfigs: Object.fromEntries(persisted.releases.map((release) => [release.releaseId, this.config])),
|
|
907
|
+
serviceReleaseIds: definitionReleaseId ? Object.fromEntries(persisted.services.map((service) => [service.id, definitionReleaseId])) : {},
|
|
627
908
|
singletonReleaseIds: Object.fromEntries(persisted.singletons.map((singleton) => [singleton.id, persisted.activeReleaseId]).filter((entry) => entry[1] !== null)),
|
|
628
909
|
snapshot: persisted
|
|
629
910
|
}
|
|
@@ -714,6 +995,7 @@ export default class RollbridgeDaemon {
|
|
|
714
995
|
authority: this.ownerAuthority(),
|
|
715
996
|
config: this.config,
|
|
716
997
|
releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
|
|
998
|
+
serviceReleaseIds: Object.fromEntries(this.serviceReleaseIds),
|
|
717
999
|
singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds),
|
|
718
1000
|
snapshot: this.status()
|
|
719
1001
|
}
|
|
@@ -735,14 +1017,18 @@ export default class RollbridgeDaemon {
|
|
|
735
1017
|
incumbentPid: this.legacyIncumbentPid,
|
|
736
1018
|
reason: "retained guardian and daemon lacked atomic replacement protocol"
|
|
737
1019
|
})
|
|
738
|
-
await this.guardian.beginLegacyOwnerClaim(
|
|
1020
|
+
await this.guardian.beginLegacyOwnerClaim(
|
|
1021
|
+
bridge.prepared.replacementId,
|
|
1022
|
+
bridge.recoverySnapshot.recovery.reconnectGraceMs,
|
|
1023
|
+
this.statePath,
|
|
1024
|
+
bridge.recoverySnapshot
|
|
1025
|
+
)
|
|
739
1026
|
process.kill(this.legacyIncumbentPid, "SIGKILL")
|
|
740
1027
|
bridge.boundaryCrossed = true
|
|
741
1028
|
await bridge.incumbentControl?.closed()
|
|
742
1029
|
if (legacyCommitted) await legacyCommitted
|
|
743
1030
|
bridge.legacyGuardian.disconnect()
|
|
744
1031
|
await this.guardian.completeLegacyOwnerClaim(bridge.prepared.replacementId)
|
|
745
|
-
await writeState(this.statePath, bridge.recoverySnapshot)
|
|
746
1032
|
this.ownerTransition = /** @type {OwnerTransition} */ ({
|
|
747
1033
|
disruptive: true,
|
|
748
1034
|
mode: "legacy-first-upgrade",
|
|
@@ -774,10 +1060,16 @@ export default class RollbridgeDaemon {
|
|
|
774
1060
|
|
|
775
1061
|
/** @returns {Promise<void>} Exposes control commands and begins periodic state persistence. */
|
|
776
1062
|
async exposeControl() {
|
|
1063
|
+
if (this.ownerRetired) return
|
|
777
1064
|
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
778
1065
|
|
|
779
1066
|
await this.startControlServer()
|
|
780
1067
|
|
|
1068
|
+
if (this.ownerRetired) {
|
|
1069
|
+
await this.closeServer(this.controlServer)
|
|
1070
|
+
await this.removeControlSocket()
|
|
1071
|
+
return
|
|
1072
|
+
}
|
|
781
1073
|
if (this.stopping) {
|
|
782
1074
|
await this.closeServer(this.controlServer)
|
|
783
1075
|
await fs.rm(this.config.control.path, {force: true})
|
|
@@ -791,6 +1083,7 @@ export default class RollbridgeDaemon {
|
|
|
791
1083
|
* @returns {Promise<void>} Starts the stable local proxy.
|
|
792
1084
|
*/
|
|
793
1085
|
async startProxy() {
|
|
1086
|
+
if (this.ownerRetired) return
|
|
794
1087
|
const server = http.createServer((request, response) => this.proxyHttp(request, response))
|
|
795
1088
|
|
|
796
1089
|
server.on("upgrade", (request, socket, head) => this.proxyWebSocket(request, socket, head))
|
|
@@ -805,6 +1098,7 @@ export default class RollbridgeDaemon {
|
|
|
805
1098
|
resolve(undefined)
|
|
806
1099
|
})
|
|
807
1100
|
})
|
|
1101
|
+
if (this.ownerRetired) this.proxyClosePromise = this.closeServer(server)
|
|
808
1102
|
}
|
|
809
1103
|
|
|
810
1104
|
/**
|
|
@@ -813,10 +1107,12 @@ export default class RollbridgeDaemon {
|
|
|
813
1107
|
* @returns {Promise<void>} Starts the control socket.
|
|
814
1108
|
*/
|
|
815
1109
|
async startControlServer(socketPath = this.config.control.path, applyMetadata = true) {
|
|
1110
|
+
if (this.ownerRetired) return
|
|
816
1111
|
const server = net.createServer((socket) => this.handleControlSocket(socket))
|
|
817
1112
|
|
|
818
1113
|
this.controlServer = server
|
|
819
1114
|
await this.prepareControlSocketPath(socketPath)
|
|
1115
|
+
if (this.ownerRetired) return
|
|
820
1116
|
|
|
821
1117
|
await new Promise((resolve, reject) => {
|
|
822
1118
|
server.once("error", reject)
|
|
@@ -827,7 +1123,14 @@ export default class RollbridgeDaemon {
|
|
|
827
1123
|
resolve(undefined)
|
|
828
1124
|
})
|
|
829
1125
|
})
|
|
1126
|
+
const socketIdentity = await fs.lstat(socketPath)
|
|
830
1127
|
|
|
1128
|
+
this.boundControlIdentity = {dev: socketIdentity.dev, ino: socketIdentity.ino}
|
|
1129
|
+
if (this.ownerRetired) {
|
|
1130
|
+
await this.closeServer(server)
|
|
1131
|
+
await this.removeControlSocket()
|
|
1132
|
+
return
|
|
1133
|
+
}
|
|
831
1134
|
if (applyMetadata) await this.applyControlSocketMetadata(socketPath)
|
|
832
1135
|
}
|
|
833
1136
|
|
|
@@ -1086,9 +1389,12 @@ export default class RollbridgeDaemon {
|
|
|
1086
1389
|
await this.guardian.commitOwnerReplacement(replacementId)
|
|
1087
1390
|
committed = true
|
|
1088
1391
|
await this.retireCommittedOwner(controlSocket)
|
|
1392
|
+
await this.beginRetiredListenerPublication(replacementId, this.listenerHandoff?.retired)
|
|
1089
1393
|
await this.guardian.finalizeOwnerReplacement(replacementId)
|
|
1090
|
-
|
|
1394
|
+
await this.completeRetiredListenerHandoff(replacementId)
|
|
1395
|
+
} catch (error) {
|
|
1091
1396
|
if (committed) this.guardian.disconnect()
|
|
1397
|
+
throw error
|
|
1092
1398
|
}
|
|
1093
1399
|
return {message: "owner replacement committed"}
|
|
1094
1400
|
}
|
|
@@ -1164,10 +1470,12 @@ export default class RollbridgeDaemon {
|
|
|
1164
1470
|
if (!this.guardian) throw new Error("Owner listener handoff requires the durable process guardian")
|
|
1165
1471
|
if (this.listenerHandoff) throw new Error("Owner listeners are already yielded to a replacement candidate")
|
|
1166
1472
|
await this.guardian.validateOwnerReplacement(replacementId)
|
|
1167
|
-
|
|
1473
|
+
if (!completionSocket) throw new Error("Owner listener handoff requires its authenticated control session")
|
|
1474
|
+
const retired = this.guardian.waitForEvent("replacement-retired")
|
|
1475
|
+
|
|
1476
|
+
this.listenerHandoff = {completionSocket, control, proxy, replacementId, retired}
|
|
1168
1477
|
this.listenerHandoffFailure = undefined
|
|
1169
1478
|
|
|
1170
|
-
if (!completionSocket) throw new Error("Owner listener handoff requires its authenticated control session")
|
|
1171
1479
|
for (const release of this.releases.values()) release.pauseDrainForOwnerHandoff()
|
|
1172
1480
|
if (proxy) this.proxyClosePromise = this.closeServer(this.proxyServer)
|
|
1173
1481
|
if (control) {
|
|
@@ -1175,22 +1483,27 @@ export default class RollbridgeDaemon {
|
|
|
1175
1483
|
for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
|
|
1176
1484
|
await this.removeControlSocket()
|
|
1177
1485
|
}
|
|
1486
|
+
/** @type {(sourceId: string, releaseId: string, connections: {http: number, websocket: number}) => void} */
|
|
1487
|
+
const publishConnections = (sourceId, releaseId, connections) => {
|
|
1488
|
+
if (completionSocket.destroyed) return
|
|
1489
|
+
completionSocket.write(`${JSON.stringify({connections, event: "owner-connection-state", releaseId, sourceId})}\n`)
|
|
1490
|
+
}
|
|
1491
|
+
|
|
1492
|
+
for (const [sourceId, releases] of this.listenerConnectionSources) {
|
|
1493
|
+
for (const [releaseId, connections] of releases) publishConnections(sourceId, releaseId, connections)
|
|
1494
|
+
}
|
|
1178
1495
|
for (const release of this.releases.values()) {
|
|
1179
|
-
const
|
|
1180
|
-
if (completionSocket.destroyed) return
|
|
1181
|
-
completionSocket.write(`${JSON.stringify({
|
|
1182
|
-
connections: release.status().connections,
|
|
1183
|
-
event: "owner-connection-state",
|
|
1184
|
-
releaseId: release.releaseId
|
|
1185
|
-
})}\n`)
|
|
1186
|
-
}
|
|
1496
|
+
const connections = release.localConnections()
|
|
1187
1497
|
|
|
1188
|
-
|
|
1189
|
-
|
|
1498
|
+
if (release.state === "active" || release.state === "draining" || connections.http + connections.websocket > 0) {
|
|
1499
|
+
publishConnections(this.listenerSourceId, release.releaseId, connections)
|
|
1500
|
+
}
|
|
1501
|
+
if (connections.http + connections.websocket > 0) {
|
|
1502
|
+
release.once("drained", () => publishConnections(this.listenerSourceId, release.releaseId, release.localConnections()))
|
|
1503
|
+
}
|
|
1190
1504
|
}
|
|
1191
1505
|
|
|
1192
1506
|
const aborted = this.guardian.waitForEvent("replacement-aborted").then(async () => await this.resumeYieldedListeners())
|
|
1193
|
-
const retired = this.guardian.waitForEvent("replacement-retired")
|
|
1194
1507
|
|
|
1195
1508
|
void Promise.race([aborted, retired]).catch((error) => {
|
|
1196
1509
|
if (this.ownerRetired) return
|
|
@@ -1210,9 +1523,11 @@ export default class RollbridgeDaemon {
|
|
|
1210
1523
|
}
|
|
1211
1524
|
if (handoff.proxy) {
|
|
1212
1525
|
await this.startProxy()
|
|
1526
|
+
if (this.ownerRetired) return
|
|
1213
1527
|
}
|
|
1214
1528
|
if (handoff.control) {
|
|
1215
1529
|
await this.startControlServer()
|
|
1530
|
+
if (this.ownerRetired) return
|
|
1216
1531
|
}
|
|
1217
1532
|
this.listenerHandoff = undefined
|
|
1218
1533
|
this.listenerHandoffFailure = undefined
|
|
@@ -1242,6 +1557,13 @@ export default class RollbridgeDaemon {
|
|
|
1242
1557
|
this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
|
|
1243
1558
|
return {activeReleaseId: newReleaseId, previousReleaseId: transition.previousReleaseId}
|
|
1244
1559
|
}
|
|
1560
|
+
if (transition?.phase === "committed" && !this.activeRelease && this.bootstrap && this.releases.get(transition.candidateReleaseId)?.state === "draining") {
|
|
1561
|
+
this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
|
|
1562
|
+
this.assertCommittedBootstrapRecoveryReady()
|
|
1563
|
+
this.config = nextConfig
|
|
1564
|
+
await this.updateGenerationTransition("restoring_committed")
|
|
1565
|
+
return await this.resumeGenerationTransition()
|
|
1566
|
+
}
|
|
1245
1567
|
const release = new ReleaseGroup({
|
|
1246
1568
|
config: nextConfig,
|
|
1247
1569
|
logger: this.logger,
|
|
@@ -1274,56 +1596,27 @@ export default class RollbridgeDaemon {
|
|
|
1274
1596
|
|
|
1275
1597
|
const previousRelease = this.activeRelease
|
|
1276
1598
|
|
|
1277
|
-
if (nextConfig.processes.some((processConfig) => processConfig.lifecycle.activateCommand !== undefined)) {
|
|
1278
|
-
this.config = nextConfig
|
|
1279
|
-
this.releases.set(release.releaseId, release)
|
|
1280
|
-
const now = new Date().toISOString()
|
|
1281
|
-
|
|
1282
|
-
this.generationTransition = /** @type {GenerationTransition} */ ({
|
|
1283
|
-
candidateReleaseId: release.releaseId,
|
|
1284
|
-
candidateReleasePath: release.releasePath,
|
|
1285
|
-
candidateRevision: release.revision,
|
|
1286
|
-
configDigest: ownerConfigDigest(nextConfig),
|
|
1287
|
-
phase: "candidate_ready",
|
|
1288
|
-
previousReleaseId: previousRelease?.releaseId ?? null,
|
|
1289
|
-
startedAt: now,
|
|
1290
|
-
updatedAt: now
|
|
1291
|
-
})
|
|
1292
|
-
await this.checkpointGenerationTransition()
|
|
1293
|
-
return await this.resumeGenerationTransition()
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
1599
|
this.config = nextConfig
|
|
1297
1600
|
this.releases.set(release.releaseId, release)
|
|
1298
|
-
release.
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
await this.replaceSingletons(release)
|
|
1318
|
-
|
|
1319
|
-
await this.persistState()
|
|
1320
|
-
await this.publishOwnerState()
|
|
1321
|
-
|
|
1322
|
-
return {
|
|
1323
|
-
activeReleaseId: release.releaseId,
|
|
1324
|
-
previousReleaseId: previousRelease ? previousRelease.releaseId : null,
|
|
1325
|
-
...(retirementFailure && previousRelease ? {retirement: {error: retirementFailure, releaseId: previousRelease.releaseId, status: "quiescence_failed"}} : {})
|
|
1326
|
-
}
|
|
1601
|
+
for (const serviceId of startedServices) this.serviceReleaseIds.set(serviceId, release.releaseId)
|
|
1602
|
+
|
|
1603
|
+
const activationLifecycle = nextConfig.processes.some((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
|
|
1604
|
+
const now = new Date().toISOString()
|
|
1605
|
+
|
|
1606
|
+
this.generationTransition = /** @type {GenerationTransition} */ ({
|
|
1607
|
+
activationLifecycle,
|
|
1608
|
+
candidateReleaseId: release.releaseId,
|
|
1609
|
+
candidateReleasePath: release.releasePath,
|
|
1610
|
+
candidateRevision: release.revision,
|
|
1611
|
+
configDigest: ownerConfigDigest(nextConfig),
|
|
1612
|
+
journalRevision: 0,
|
|
1613
|
+
phase: activationLifecycle ? "candidate_ready" : "activating_candidate",
|
|
1614
|
+
previousReleaseId: previousRelease?.releaseId ?? null,
|
|
1615
|
+
startedAt: now,
|
|
1616
|
+
updatedAt: now
|
|
1617
|
+
})
|
|
1618
|
+
await this.checkpointGenerationTransition()
|
|
1619
|
+
return await this.resumeGenerationTransition()
|
|
1327
1620
|
}
|
|
1328
1621
|
|
|
1329
1622
|
/**
|
|
@@ -1338,7 +1631,9 @@ export default class RollbridgeDaemon {
|
|
|
1338
1631
|
const previousRelease = transition.previousReleaseId ? this.releases.get(transition.previousReleaseId) : undefined
|
|
1339
1632
|
|
|
1340
1633
|
if (!release) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
|
|
1341
|
-
if (transition.previousReleaseId && !previousRelease) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
1634
|
+
if (transition.previousReleaseId && !previousRelease && transition.phase !== "restoring_committed") throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
1635
|
+
const activationLifecycle = transition.activationLifecycle ?? release.config.processes.some((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
|
|
1636
|
+
let retirementFailure = activationLifecycle ? undefined : previousRelease?.retirementError
|
|
1342
1637
|
|
|
1343
1638
|
if (transition.phase === "candidate_ready") {
|
|
1344
1639
|
if (previousRelease) await this.updateGenerationTransition("retiring_previous")
|
|
@@ -1383,13 +1678,34 @@ export default class RollbridgeDaemon {
|
|
|
1383
1678
|
this.activeRelease = release
|
|
1384
1679
|
transition.phase = "committed_pending"
|
|
1385
1680
|
transition.error = undefined
|
|
1681
|
+
transition.journalRevision = (transition.journalRevision ?? 0) + 1
|
|
1386
1682
|
transition.updatedAt = new Date().toISOString()
|
|
1387
1683
|
this.logger("traffic switched", {previousReleaseId: previousRelease?.releaseId ?? null, releaseId: release.releaseId})
|
|
1388
1684
|
}
|
|
1389
1685
|
|
|
1686
|
+
if (transition.phase === "restoring_committed") {
|
|
1687
|
+
await this.resumeCommittedBootstrapGeneration(release)
|
|
1688
|
+
release.activate()
|
|
1689
|
+
this.activeRelease = release
|
|
1690
|
+
transition.phase = "committed_pending"
|
|
1691
|
+
transition.error = undefined
|
|
1692
|
+
transition.updatedAt = new Date().toISOString()
|
|
1693
|
+
this.logger("committed bootstrap generation restored", {releaseId: release.releaseId})
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1390
1696
|
if (transition.phase === "committed_pending") {
|
|
1697
|
+
await this.refreshServiceDefinitions(release)
|
|
1698
|
+
if (!activationLifecycle && previousRelease?.state === "active") {
|
|
1699
|
+
try {
|
|
1700
|
+
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : release.config
|
|
1701
|
+
|
|
1702
|
+
await previousRelease.beginRetirement(retirementConfig)
|
|
1703
|
+
} catch (error) {
|
|
1704
|
+
retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
|
|
1705
|
+
this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1391
1708
|
await this.checkpointGenerationTransition()
|
|
1392
|
-
this.refreshServiceDefinitions(release)
|
|
1393
1709
|
if (previousRelease) {
|
|
1394
1710
|
const retirementConfig = previousRelease.config
|
|
1395
1711
|
|
|
@@ -1403,11 +1719,17 @@ export default class RollbridgeDaemon {
|
|
|
1403
1719
|
}
|
|
1404
1720
|
transition.phase = "committed"
|
|
1405
1721
|
transition.error = undefined
|
|
1722
|
+
transition.journalRevision = (transition.journalRevision ?? 0) + 1
|
|
1406
1723
|
transition.updatedAt = new Date().toISOString()
|
|
1724
|
+
this.pruneStoppedReleases()
|
|
1407
1725
|
await this.checkpointGenerationTransition()
|
|
1408
1726
|
}
|
|
1409
1727
|
|
|
1410
|
-
return {
|
|
1728
|
+
return {
|
|
1729
|
+
activeReleaseId: release.releaseId,
|
|
1730
|
+
previousReleaseId: previousRelease?.releaseId ?? null,
|
|
1731
|
+
...(retirementFailure && previousRelease ? {retirement: {error: retirementFailure, releaseId: previousRelease.releaseId, status: "quiescence_failed"}} : {})
|
|
1732
|
+
}
|
|
1411
1733
|
}
|
|
1412
1734
|
|
|
1413
1735
|
/**
|
|
@@ -1420,11 +1742,97 @@ export default class RollbridgeDaemon {
|
|
|
1420
1742
|
}
|
|
1421
1743
|
}
|
|
1422
1744
|
|
|
1745
|
+
/**
|
|
1746
|
+
* Returns the retained candidate only when the foreground bootstrap exactly proves the
|
|
1747
|
+
* committed transition that an external owner retirement left without an active role.
|
|
1748
|
+
* @returns {ReleaseGroup | undefined} Exact committed bootstrap candidate.
|
|
1749
|
+
*/
|
|
1750
|
+
committedBootstrapRelease() {
|
|
1751
|
+
const bootstrap = this.bootstrap
|
|
1752
|
+
const transition = this.generationTransition
|
|
1753
|
+
|
|
1754
|
+
if (!bootstrap || !transition || (transition.phase !== "committed" && transition.phase !== "restoring_committed") || transition.candidateReleaseId !== bootstrap.releaseId || transition.candidateReleasePath !== bootstrap.releasePath || transition.candidateRevision !== bootstrap.revision || transition.configDigest !== ownerConfigDigest(this.config)) return undefined
|
|
1755
|
+
const release = this.releases.get(bootstrap.releaseId)
|
|
1756
|
+
|
|
1757
|
+
if (!release || release.releasePath !== bootstrap.releasePath || release.revision !== bootstrap.revision) return undefined
|
|
1758
|
+
return release
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
/**
|
|
1762
|
+
* Fails closed before journaling recovery unless external retirement has fully stopped
|
|
1763
|
+
* the exact candidate and daemon-owned processes.
|
|
1764
|
+
* @returns {void}
|
|
1765
|
+
*/
|
|
1766
|
+
assertCommittedBootstrapRecoveryReady() {
|
|
1767
|
+
const release = this.committedBootstrapRelease()
|
|
1768
|
+
|
|
1769
|
+
if (!release) throw new Error("Committed generation has no exact foreground bootstrap recovery proof")
|
|
1770
|
+
release.assertCommittedGenerationStopped()
|
|
1771
|
+
const ownedProcesses = [
|
|
1772
|
+
...this.services.values(),
|
|
1773
|
+
...this.singletons.values()
|
|
1774
|
+
]
|
|
1775
|
+
const stillRetiring = ownedProcesses.find((processInstance) => {
|
|
1776
|
+
const {pid, state} = processInstance.status()
|
|
1777
|
+
return pid !== undefined || (state !== "stopped" && state !== "failed")
|
|
1778
|
+
})
|
|
1779
|
+
|
|
1780
|
+
if (stillRetiring) throw new Error(`Committed generation daemon process ${stillRetiring.id} is still retiring; exact bootstrap recovery will retry after it stops`)
|
|
1781
|
+
for (const serviceId of this.services.keys()) {
|
|
1782
|
+
const serviceReleaseId = this.serviceReleaseIds.get(serviceId)
|
|
1783
|
+
|
|
1784
|
+
if (serviceReleaseId !== release.releaseId) throw new Error(`Committed generation service ${serviceId} belongs to retained release ${serviceReleaseId}`)
|
|
1785
|
+
}
|
|
1786
|
+
for (const [singletonId, singletonReleaseId] of this.singletonReleaseIds) {
|
|
1787
|
+
if (singletonReleaseId !== release.releaseId) throw new Error(`Committed generation singleton ${singletonId} belongs to retained release ${singletonReleaseId}`)
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
/**
|
|
1792
|
+
* Resumes a durably journaled committed-candidate restart. Running processes are
|
|
1793
|
+
* necessarily owned by this exact recovery phase; stopped processes are restarted.
|
|
1794
|
+
* Singleton completion remains in the established committed_pending phase.
|
|
1795
|
+
* @param {ReleaseGroup} release - Exact committed candidate.
|
|
1796
|
+
* @returns {Promise<void>}
|
|
1797
|
+
*/
|
|
1798
|
+
async resumeCommittedBootstrapGeneration(release) {
|
|
1799
|
+
if (this.generationTransition?.phase !== "restoring_committed" || release !== this.committedBootstrapRelease()) {
|
|
1800
|
+
throw new Error("Committed bootstrap recovery is not durably journaled for this exact candidate")
|
|
1801
|
+
}
|
|
1802
|
+
const resumableStates = new Set(["failed", "running", "stopped"])
|
|
1803
|
+
const invalidProcess = [...this.services.values(), ...this.singletons.values()].find((processInstance) => {
|
|
1804
|
+
const {pid, state} = processInstance.status()
|
|
1805
|
+
return !resumableStates.has(state) || (state === "running") !== (pid !== undefined)
|
|
1806
|
+
})
|
|
1807
|
+
|
|
1808
|
+
if (invalidProcess) throw new Error(`Committed bootstrap recovery found daemon process ${invalidProcess.id} outside its journaled restart states`)
|
|
1809
|
+
release.assertCommittedGenerationRecoverable()
|
|
1810
|
+
|
|
1811
|
+
try {
|
|
1812
|
+
for (const processInstance of this.services.values()) {
|
|
1813
|
+
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
1814
|
+
await processInstance.start("deploy")
|
|
1815
|
+
}
|
|
1816
|
+
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
1817
|
+
await release.restartCommittedGeneration()
|
|
1818
|
+
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
1819
|
+
await release.activateGeneration()
|
|
1820
|
+
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
1821
|
+
} catch (error) {
|
|
1822
|
+
await Promise.allSettled([
|
|
1823
|
+
release.abortCommittedGenerationRestart(),
|
|
1824
|
+
...[...this.services.values()].map((processInstance) => processInstance.stop())
|
|
1825
|
+
])
|
|
1826
|
+
throw error
|
|
1827
|
+
}
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1423
1830
|
/** @param {GenerationTransitionPhase} phase - Durable phase to enter. */
|
|
1424
1831
|
async updateGenerationTransition(phase) {
|
|
1425
1832
|
if (!this.generationTransition) throw new Error("No release generation transition to update")
|
|
1426
1833
|
this.generationTransition.phase = phase
|
|
1427
1834
|
this.generationTransition.error = undefined
|
|
1835
|
+
this.generationTransition.journalRevision = (this.generationTransition.journalRevision ?? 0) + 1
|
|
1428
1836
|
this.generationTransition.updatedAt = new Date().toISOString()
|
|
1429
1837
|
await this.checkpointGenerationTransition()
|
|
1430
1838
|
}
|
|
@@ -1433,6 +1841,7 @@ export default class RollbridgeDaemon {
|
|
|
1433
1841
|
async failGenerationTransition(error) {
|
|
1434
1842
|
if (!this.generationTransition) throw new Error("No release generation transition to fail")
|
|
1435
1843
|
this.generationTransition.error = error
|
|
1844
|
+
this.generationTransition.journalRevision = (this.generationTransition.journalRevision ?? 0) + 1
|
|
1436
1845
|
this.generationTransition.updatedAt = new Date().toISOString()
|
|
1437
1846
|
await this.checkpointGenerationTransition()
|
|
1438
1847
|
}
|
|
@@ -1455,6 +1864,7 @@ export default class RollbridgeDaemon {
|
|
|
1455
1864
|
shouldResumeDrain(release) {
|
|
1456
1865
|
const transition = this.generationTransition
|
|
1457
1866
|
|
|
1867
|
+
if (release === this.committedBootstrapRelease()) return false
|
|
1458
1868
|
return !transition || transition.phase === "committed_pending" || transition.phase === "committed" || transition.previousReleaseId !== release.releaseId
|
|
1459
1869
|
}
|
|
1460
1870
|
|
|
@@ -1464,19 +1874,143 @@ export default class RollbridgeDaemon {
|
|
|
1464
1874
|
* @param {net.Socket | undefined} completionSocket - Commit response connection.
|
|
1465
1875
|
*/
|
|
1466
1876
|
async retireCommittedOwner(completionSocket) {
|
|
1877
|
+
if (!this.committedRetirementPromise) this.committedRetirementPromise = this.performCommittedOwnerRetirement(completionSocket)
|
|
1878
|
+
await this.committedRetirementPromise
|
|
1879
|
+
}
|
|
1880
|
+
|
|
1881
|
+
/** @param {net.Socket | undefined} completionSocket - Commit response connection. */
|
|
1882
|
+
async performCommittedOwnerRetirement(completionSocket) {
|
|
1467
1883
|
this.ownerRetired = true
|
|
1884
|
+
this.controlCommandsReady = false
|
|
1468
1885
|
if (this.persistTimer) clearInterval(this.persistTimer)
|
|
1469
1886
|
this.persistTimer = undefined
|
|
1470
1887
|
this.persistenceEnabled = false
|
|
1471
1888
|
if (this.pendingWrite) await this.pendingWrite
|
|
1472
1889
|
this.stateCleanupEnabled = false
|
|
1473
1890
|
this.controlClosePromise = this.closeServer(this.controlServer)
|
|
1891
|
+
this.proxyClosePromise = this.closeServer(this.proxyServer)
|
|
1474
1892
|
for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
|
|
1475
1893
|
await this.removeControlSocket()
|
|
1476
|
-
void this.closeServer(this.proxyServer)
|
|
1477
1894
|
this.logger("owner authority transferred", {activeReleaseId: this.activeRelease?.releaseId ?? null})
|
|
1478
1895
|
}
|
|
1479
1896
|
|
|
1897
|
+
/** @param {string} replacementId - Prepared same-authority transaction. */
|
|
1898
|
+
async yieldControlLessOwnerListeners(replacementId) {
|
|
1899
|
+
if (!this.guardian) throw new Error("Control-less owner listener handoff requires the durable process guardian")
|
|
1900
|
+
this.proxyClosePromise = this.closeServer(this.proxyServer)
|
|
1901
|
+
await this.beginRetiredListenerPublication(replacementId)
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
/**
|
|
1905
|
+
* @param {string} replacementId - Transaction receiving durable source updates.
|
|
1906
|
+
* @param {Promise<Record<string, JsonValue>>} [retired] - Existing direct-listener retirement event.
|
|
1907
|
+
*/
|
|
1908
|
+
async beginRetiredListenerPublication(replacementId, retired) {
|
|
1909
|
+
if (!this.guardian) throw new Error("Retired listener publication requires the durable process guardian")
|
|
1910
|
+
if (this.retiredListenerHandoff) throw new Error("Another retired listener handoff is already pending")
|
|
1911
|
+
const guardian = this.guardian
|
|
1912
|
+
const releases = [...this.releases.values()]
|
|
1913
|
+
|
|
1914
|
+
this.retiredListenerHandoff = {
|
|
1915
|
+
releases,
|
|
1916
|
+
replacementId,
|
|
1917
|
+
replacementRetired: false,
|
|
1918
|
+
retired: retired || guardian.waitForEvent("replacement-retired")
|
|
1919
|
+
}
|
|
1920
|
+
this.retiredReplacementId = replacementId
|
|
1921
|
+
for (const release of releases) {
|
|
1922
|
+
if (release.status().connectionCount > 0) {
|
|
1923
|
+
release.once("drained", () => {
|
|
1924
|
+
if (this.retiredReplacementId !== replacementId) return
|
|
1925
|
+
const publication = this.publishRetiredConnectionState(replacementId, this.listenerSourceId, release.releaseId, release.localConnections(), true)
|
|
1926
|
+
|
|
1927
|
+
void publication.then(async () => {
|
|
1928
|
+
await this.retiredConnectionPublication
|
|
1929
|
+
this.disconnectRetiredOwnerIfDrained()
|
|
1930
|
+
}).catch((error) => {
|
|
1931
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
1932
|
+
this.logger("retired owner connection-state publication failed", {error: this.listenerHandoffFailure.message, releaseId: release.releaseId, replacementId})
|
|
1933
|
+
guardian.disconnect()
|
|
1934
|
+
})
|
|
1935
|
+
})
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
for (const [sourceId, sourceReleases] of this.listenerConnectionSources) {
|
|
1939
|
+
for (const [releaseId, connections] of sourceReleases) {
|
|
1940
|
+
await this.publishRetiredConnectionState(replacementId, sourceId, releaseId, connections)
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1943
|
+
for (const release of releases) {
|
|
1944
|
+
const connections = release.localConnections()
|
|
1945
|
+
|
|
1946
|
+
if (release.state === "active" || release.state === "draining" || connections.http + connections.websocket > 0) {
|
|
1947
|
+
await this.publishRetiredConnectionState(replacementId, this.listenerSourceId, release.releaseId, connections, true)
|
|
1948
|
+
}
|
|
1949
|
+
}
|
|
1950
|
+
await guardian.completeOwnerListenerRetirement(replacementId)
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1953
|
+
/** Resumes the incumbent listener after a prepared candidate aborts before commit. */
|
|
1954
|
+
async resumeControlLessOwnerListeners() {
|
|
1955
|
+
const handoff = this.retiredListenerHandoff
|
|
1956
|
+
|
|
1957
|
+
if (!handoff || this.ownerRetired) return
|
|
1958
|
+
void handoff.retired.catch(() => undefined)
|
|
1959
|
+
this.retiredListenerHandoff = undefined
|
|
1960
|
+
this.retiredReplacementId = undefined
|
|
1961
|
+
for (const release of handoff.releases) {
|
|
1962
|
+
release.resumeDrainAfterOwnerHandoff()
|
|
1963
|
+
if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
1964
|
+
}
|
|
1965
|
+
await this.startProxy()
|
|
1966
|
+
this.listenerHandoffFailure = undefined
|
|
1967
|
+
this.logger("control-less owner listeners resumed", {replacementId: handoff.replacementId})
|
|
1968
|
+
}
|
|
1969
|
+
|
|
1970
|
+
/** @param {string} replacementId - Committed same-authority transaction. */
|
|
1971
|
+
async completeControlLessOwnerRetirement(replacementId) {
|
|
1972
|
+
const handoff = this.retiredListenerHandoff
|
|
1973
|
+
|
|
1974
|
+
if (!handoff || handoff.replacementId !== replacementId) throw new Error("Committed owner retirement does not match the prepared listener handoff")
|
|
1975
|
+
await this.retireCommittedOwner(undefined)
|
|
1976
|
+
await this.completeRetiredListenerHandoff(replacementId)
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
/** @param {string} replacementId - Finalized retired-listener transaction. */
|
|
1980
|
+
async completeRetiredListenerHandoff(replacementId) {
|
|
1981
|
+
const handoff = this.retiredListenerHandoff
|
|
1982
|
+
|
|
1983
|
+
if (!handoff || handoff.replacementId !== replacementId) throw new Error("Retired listener completion does not match its prepared handoff")
|
|
1984
|
+
await handoff.retired
|
|
1985
|
+
handoff.replacementRetired = true
|
|
1986
|
+
this.disconnectRetiredOwnerIfDrained()
|
|
1987
|
+
}
|
|
1988
|
+
|
|
1989
|
+
/** Disconnects the retired daemon only after every retained listener source drains. */
|
|
1990
|
+
disconnectRetiredOwnerIfDrained() {
|
|
1991
|
+
const handoff = this.retiredListenerHandoff
|
|
1992
|
+
|
|
1993
|
+
if (handoff?.replacementRetired && handoff.releases.every((release) => release.status().connectionCount === 0)) this.guardian?.disconnect()
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
/**
|
|
1997
|
+
* Serializes one retired-listener source update over the authenticated guardian channel.
|
|
1998
|
+
* @param {string} replacementId - Committed replacement transaction.
|
|
1999
|
+
* @param {string} sourceId - Stable physical listener source.
|
|
2000
|
+
* @param {string} releaseId - Retained release.
|
|
2001
|
+
* @param {{http: number, websocket: number}} connections - Exact source counts.
|
|
2002
|
+
* @param {boolean} [localSource] - Whether this daemon physically owns the source.
|
|
2003
|
+
*/
|
|
2004
|
+
async publishRetiredConnectionState(replacementId, sourceId, releaseId, connections, localSource = false) {
|
|
2005
|
+
if (!this.guardian) throw new Error("Retired listener state publication requires the durable process guardian")
|
|
2006
|
+
const guardian = this.guardian
|
|
2007
|
+
|
|
2008
|
+
this.retiredConnectionPublication = this.retiredConnectionPublication.then(async () => {
|
|
2009
|
+
await guardian.publishOwnerConnectionState(replacementId, sourceId, releaseId, connections, localSource)
|
|
2010
|
+
})
|
|
2011
|
+
await this.retiredConnectionPublication
|
|
2012
|
+
}
|
|
2013
|
+
|
|
1480
2014
|
/**
|
|
1481
2015
|
* Rejects config changes that require rebinding daemon-owned resources or changing process topology.
|
|
1482
2016
|
* @param {import("./config.js").RollbridgeConfig} nextConfig - Freshly loaded config.
|
|
@@ -1490,6 +2024,10 @@ export default class RollbridgeDaemon {
|
|
|
1490
2024
|
if (!isDeepStrictEqual(nextConfig.control, this.config.control)) restartRequired.push("control")
|
|
1491
2025
|
if (nextConfig.statePath !== this.config.statePath) restartRequired.push("statePath")
|
|
1492
2026
|
if (!isDeepStrictEqual(nextConfig.ownerRecovery, this.config.ownerRecovery)) restartRequired.push("ownerRecovery")
|
|
2027
|
+
const activationProcessIds = this.config.processes.filter((processConfig) => processConfig.lifecycle.activateCommand !== undefined).map((processConfig) => processConfig.id)
|
|
2028
|
+
const nextActivationProcessIds = nextConfig.processes.filter((processConfig) => processConfig.lifecycle.activateCommand !== undefined).map((processConfig) => processConfig.id)
|
|
2029
|
+
|
|
2030
|
+
if (!isDeepStrictEqual(nextActivationProcessIds, activationProcessIds)) restartRequired.push("processes.lifecycle.activateCommand")
|
|
1493
2031
|
|
|
1494
2032
|
if (nextConfig.proxy.host !== this.config.proxy.host) restartRequired.push("proxy.host")
|
|
1495
2033
|
if (nextConfig.proxy.port !== this.config.proxy.port) restartRequired.push("proxy.port")
|
|
@@ -1592,6 +2130,7 @@ export default class RollbridgeDaemon {
|
|
|
1592
2130
|
startedServices.push(processConfig.id)
|
|
1593
2131
|
} catch (error) {
|
|
1594
2132
|
this.services.delete(processConfig.id)
|
|
2133
|
+
this.serviceReleaseIds.delete(processConfig.id)
|
|
1595
2134
|
delete this.servicePorts[processConfig.id]
|
|
1596
2135
|
throw error
|
|
1597
2136
|
}
|
|
@@ -1613,6 +2152,7 @@ export default class RollbridgeDaemon {
|
|
|
1613
2152
|
|
|
1614
2153
|
await service.stop()
|
|
1615
2154
|
this.services.delete(serviceId)
|
|
2155
|
+
this.serviceReleaseIds.delete(serviceId)
|
|
1616
2156
|
const port = this.servicePorts[serviceId]
|
|
1617
2157
|
|
|
1618
2158
|
if (port !== undefined) this.portReservations.delete(port)
|
|
@@ -1623,9 +2163,9 @@ export default class RollbridgeDaemon {
|
|
|
1623
2163
|
/**
|
|
1624
2164
|
* Updates daemon-wide service restart templates after a successful deploy.
|
|
1625
2165
|
* @param {ReleaseGroup} release - Active release.
|
|
1626
|
-
* @returns {void}
|
|
2166
|
+
* @returns {Promise<void>} Resolves once every persistent service definition is committed.
|
|
1627
2167
|
*/
|
|
1628
|
-
refreshServiceDefinitions(release) {
|
|
2168
|
+
async refreshServiceDefinitions(release) {
|
|
1629
2169
|
for (const processConfig of this.config.processes) {
|
|
1630
2170
|
if (processConfig.policy !== "service") continue
|
|
1631
2171
|
|
|
@@ -1634,8 +2174,16 @@ export default class RollbridgeDaemon {
|
|
|
1634
2174
|
if (!service) continue
|
|
1635
2175
|
|
|
1636
2176
|
const nextDefinition = release.buildProcess(processConfig, {shouldRestart: () => !this.stopping})
|
|
2177
|
+
const previousReleaseId = this.serviceReleaseIds.get(processConfig.id)
|
|
1637
2178
|
|
|
1638
|
-
|
|
2179
|
+
this.serviceReleaseIds.set(processConfig.id, release.releaseId)
|
|
2180
|
+
try {
|
|
2181
|
+
await service.updateDefinition(nextDefinition, this.guardian ? this.transferableOwnerState() : undefined)
|
|
2182
|
+
} catch (error) {
|
|
2183
|
+
if (previousReleaseId) this.serviceReleaseIds.set(processConfig.id, previousReleaseId)
|
|
2184
|
+
else this.serviceReleaseIds.delete(processConfig.id)
|
|
2185
|
+
throw error
|
|
2186
|
+
}
|
|
1639
2187
|
}
|
|
1640
2188
|
}
|
|
1641
2189
|
|
|
@@ -1727,6 +2275,7 @@ export default class RollbridgeDaemon {
|
|
|
1727
2275
|
*/
|
|
1728
2276
|
runningInstances(processConfig) {
|
|
1729
2277
|
if (processConfig.policy === "service") {
|
|
2278
|
+
if (processConfig.deployStrategy === "handoff") return this.activeRelease?.getProcesses(processConfig.id) || []
|
|
1730
2279
|
const service = this.services.get(processConfig.id)
|
|
1731
2280
|
|
|
1732
2281
|
return service ? [{id: processConfig.id, process: service}] : []
|
|
@@ -1775,6 +2324,15 @@ export default class RollbridgeDaemon {
|
|
|
1775
2324
|
}
|
|
1776
2325
|
}
|
|
1777
2326
|
|
|
2327
|
+
/** @returns {Set<string>} Release definitions owned by an unresolved durable transition. */
|
|
2328
|
+
generationTransitionReleaseIds() {
|
|
2329
|
+
const transition = this.generationTransition
|
|
2330
|
+
|
|
2331
|
+
return new Set(transition && transition.phase !== "committed"
|
|
2332
|
+
? [transition.candidateReleaseId, ...(transition.previousReleaseId ? [transition.previousReleaseId] : [])]
|
|
2333
|
+
: [])
|
|
2334
|
+
}
|
|
2335
|
+
|
|
1778
2336
|
/**
|
|
1779
2337
|
* Drains and stops a retired release in the background, then prunes stopped releases.
|
|
1780
2338
|
* @param {ReleaseGroup} release - Release to drain and stop.
|
|
@@ -1797,9 +2355,11 @@ export default class RollbridgeDaemon {
|
|
|
1797
2355
|
|
|
1798
2356
|
/** @returns {void} Removes stopped releases beyond the retention policy. */
|
|
1799
2357
|
pruneStoppedReleases() {
|
|
2358
|
+
const serviceOwnerReleaseIds = new Set(this.serviceReleaseIds.values())
|
|
1800
2359
|
const singletonOwnerReleaseIds = new Set(this.singletonReleaseIds.values())
|
|
2360
|
+
const transitionReleaseIds = this.generationTransitionReleaseIds()
|
|
1801
2361
|
const statuses = [...this.releases.values()]
|
|
1802
|
-
.filter((release) => !singletonOwnerReleaseIds.has(release.releaseId))
|
|
2362
|
+
.filter((release) => !serviceOwnerReleaseIds.has(release.releaseId) && !singletonOwnerReleaseIds.has(release.releaseId) && !transitionReleaseIds.has(release.releaseId))
|
|
1803
2363
|
.map((release) => release.status())
|
|
1804
2364
|
|
|
1805
2365
|
for (const releaseId of releasesToPrune(statuses, this.config.releaseRetention, Date.now())) {
|
|
@@ -1835,6 +2395,7 @@ export default class RollbridgeDaemon {
|
|
|
1835
2395
|
...status,
|
|
1836
2396
|
events,
|
|
1837
2397
|
persistedAt: new Date().toISOString(),
|
|
2398
|
+
serviceReleaseIds: Object.fromEntries(this.serviceReleaseIds),
|
|
1838
2399
|
...(this.hasActivationLifecycle() ? {singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds)} : {}),
|
|
1839
2400
|
...(this.guardianIdentity ? {recovery: {
|
|
1840
2401
|
configDigest: this.ownerRecoveryConfigDigest(),
|
|
@@ -1848,8 +2409,8 @@ export default class RollbridgeDaemon {
|
|
|
1848
2409
|
// clearing the file — otherwise a write started before shutdown could recreate it afterward.
|
|
1849
2410
|
this.pendingWrite = Promise.resolve(this.pendingWrite)
|
|
1850
2411
|
.catch(() => {})
|
|
1851
|
-
.then(() => writeState(statePath, snapshot))
|
|
1852
2412
|
.then(() => this.publishOwnerState())
|
|
2413
|
+
.then(() => writeState(statePath, snapshot))
|
|
1853
2414
|
.catch((error) => {
|
|
1854
2415
|
this.logger("state persist failed", {error: error instanceof Error ? error.message : String(error)})
|
|
1855
2416
|
if (throwOnError) throw error
|
|
@@ -1895,6 +2456,7 @@ export default class RollbridgeDaemon {
|
|
|
1895
2456
|
* @returns {Promise<void>} Resolves when owned resources are stopped (and, by default, control connections close).
|
|
1896
2457
|
*/
|
|
1897
2458
|
async shutdown({completionSocket, waitForControlConnections = true} = {}) {
|
|
2459
|
+
this.assertNoGenerationTransitionRecovery("shut down")
|
|
1898
2460
|
if (!this.shutdownPromise) this.shutdownPromise = this.performShutdown(completionSocket)
|
|
1899
2461
|
|
|
1900
2462
|
await this.shutdownPromise
|
|
@@ -1908,11 +2470,17 @@ export default class RollbridgeDaemon {
|
|
|
1908
2470
|
* @returns {Promise<void>} Resolves once a replacement can exclusively bind listeners.
|
|
1909
2471
|
*/
|
|
1910
2472
|
async retireOwner({attestation, completionSocket}) {
|
|
2473
|
+
this.assertNoGenerationTransitionRecovery("retire owner")
|
|
1911
2474
|
if (this.retirementPromise) return await this.retirementPromise
|
|
1912
2475
|
this.retirementPromise = this.performOwnerRetirement(attestation, completionSocket)
|
|
1913
2476
|
return await this.retirementPromise
|
|
1914
2477
|
}
|
|
1915
2478
|
|
|
2479
|
+
/** @param {string} operation - Terminal owner operation. */
|
|
2480
|
+
assertNoGenerationTransitionRecovery(operation) {
|
|
2481
|
+
if (this.generationTransitionRecovery) throw new Error(`Cannot ${operation} while generation transition recovery is in progress`)
|
|
2482
|
+
}
|
|
2483
|
+
|
|
1916
2484
|
/**
|
|
1917
2485
|
* @param {string} attestation - Replacement boot attestation.
|
|
1918
2486
|
* @param {net.Socket | undefined} completionSocket - Requesting handoff connection.
|
|
@@ -2022,9 +2590,23 @@ export default class RollbridgeDaemon {
|
|
|
2022
2590
|
/** @returns {Promise<void>} Removes the configured control socket path. */
|
|
2023
2591
|
async removeControlSocket() {
|
|
2024
2592
|
if (!this.controlSocketOwned) return
|
|
2593
|
+
const socketPath = this.boundControlPath || this.config.control.path
|
|
2594
|
+
const expectedIdentity = this.boundControlIdentity
|
|
2595
|
+
let ownsCurrentPath = true
|
|
2596
|
+
|
|
2597
|
+
if (expectedIdentity) {
|
|
2598
|
+
try {
|
|
2599
|
+
const currentIdentity = await fs.lstat(socketPath)
|
|
2025
2600
|
|
|
2026
|
-
|
|
2601
|
+
ownsCurrentPath = currentIdentity.dev === expectedIdentity.dev && currentIdentity.ino === expectedIdentity.ino
|
|
2602
|
+
} catch (error) {
|
|
2603
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
2604
|
+
ownsCurrentPath = false
|
|
2605
|
+
}
|
|
2606
|
+
}
|
|
2607
|
+
if (ownsCurrentPath) await fs.rm(socketPath, {force: true})
|
|
2027
2608
|
this.controlSocketOwned = false
|
|
2609
|
+
this.boundControlIdentity = undefined
|
|
2028
2610
|
this.boundControlPath = undefined
|
|
2029
2611
|
}
|
|
2030
2612
|
|
|
@@ -2050,7 +2632,9 @@ export default class RollbridgeDaemon {
|
|
|
2050
2632
|
// a cleared orphan must not reappear if the OS later recycles its pid for an unrelated process.
|
|
2051
2633
|
this.orphans = this.orphans.filter((orphan) => isProcessAlive(orphan.pid))
|
|
2052
2634
|
|
|
2635
|
+
const serviceOwnerReleaseIds = new Set(this.serviceReleaseIds.values())
|
|
2053
2636
|
const singletonOwnerReleaseIds = new Set(this.singletonReleaseIds.values())
|
|
2637
|
+
const transitionReleaseIds = this.generationTransitionReleaseIds()
|
|
2054
2638
|
|
|
2055
2639
|
return {
|
|
2056
2640
|
activeReleaseId: this.activeRelease ? this.activeRelease.releaseId : null,
|
|
@@ -2060,7 +2644,7 @@ export default class RollbridgeDaemon {
|
|
|
2060
2644
|
daemonPid: process.pid,
|
|
2061
2645
|
daemonRuntime: this.runtime ? {...this.runtime} : undefined,
|
|
2062
2646
|
generationTransition: this.generationTransition ? {...this.generationTransition} : undefined,
|
|
2063
|
-
ownerRecovery: this.guardian ? {configDigest: this.ownerRecoveryConfigDigest()} : undefined,
|
|
2647
|
+
ownerRecovery: this.guardian ? {configDigest: this.ownerRecoveryConfigDigest(), ready: this.ownerReady} : undefined,
|
|
2064
2648
|
ownerTransition: this.ownerTransition ? {...this.ownerTransition} : undefined,
|
|
2065
2649
|
orphans: [...this.orphans],
|
|
2066
2650
|
proxy: {
|
|
@@ -2069,13 +2653,15 @@ export default class RollbridgeDaemon {
|
|
|
2069
2653
|
upstreamHost: this.config.proxy.upstreamHost
|
|
2070
2654
|
},
|
|
2071
2655
|
releaseReferences: [...this.releases.values()]
|
|
2072
|
-
.filter((release) => release.state === "active" || release.state === "draining" ||
|
|
2656
|
+
.filter((release) => release.state === "active" || release.state === "draining" || serviceOwnerReleaseIds.has(release.releaseId) || singletonOwnerReleaseIds.has(release.releaseId) || transitionReleaseIds.has(release.releaseId))
|
|
2073
2657
|
.map((release) => ({releaseId: release.releaseId, releasePath: release.releasePath})),
|
|
2074
2658
|
releases: [...this.releases.values()].map((release) => release.status()),
|
|
2075
|
-
services: [...this.services.entries()]
|
|
2076
|
-
id
|
|
2077
|
-
|
|
2078
|
-
|
|
2659
|
+
services: [...this.services.entries()]
|
|
2660
|
+
.filter(([id]) => this.serviceReleaseIds.has(id))
|
|
2661
|
+
.map(([id, processInstance]) => ({
|
|
2662
|
+
id,
|
|
2663
|
+
process: processInstance.status()
|
|
2664
|
+
})),
|
|
2079
2665
|
singletons: [...this.singletons.entries()].map(([id, processInstance]) => ({
|
|
2080
2666
|
id,
|
|
2081
2667
|
process: processInstance.status()
|
|
@@ -2203,8 +2789,10 @@ async function verifyLegacyDaemonProcess(pid, configPath, socketPath) {
|
|
|
2203
2789
|
const args = await processArguments(pid, "legacy daemon")
|
|
2204
2790
|
const daemonIndex = args.indexOf("daemon")
|
|
2205
2791
|
const configIndex = args.indexOf("--config")
|
|
2792
|
+
const configuredPath = args[configIndex + 1]
|
|
2793
|
+
const configuredAbsolutePath = configuredPath ? path.resolve(await fs.readlink(`/proc/${pid}/cwd`), configuredPath) : undefined
|
|
2206
2794
|
|
|
2207
|
-
if (daemonIndex < 0 || configIndex < 0 ||
|
|
2795
|
+
if (daemonIndex < 0 || configIndex < 0 || configuredAbsolutePath !== path.resolve(configPath)) {
|
|
2208
2796
|
throw new Error(`Daemon PID ${pid} does not match the exact retained daemon config command`)
|
|
2209
2797
|
}
|
|
2210
2798
|
await verifyProcessUser(pid, "legacy daemon")
|