rollbridge 0.1.28 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +14 -0
- package/README.md +69 -14
- package/TODO.md +5 -2
- package/changelog.d/20260828-atomic-owner-replacement.md +22 -0
- package/changelog.d/20260828-durable-owner-recovery.md +7 -0
- package/changelog.d/20260828-same-owner-jobs-generations.md +6 -0
- package/docs/cli.md +43 -21
- package/docs/config.md +71 -18
- package/docs/logging.md +8 -3
- package/docs/tensorbuzz-runbook.md +7 -6
- package/docs/troubleshooting.md +28 -12
- package/docs/velocious.md +11 -4
- package/docs/workers.md +8 -2
- package/examples/tensorbuzz.com.js +12 -4
- package/package.json +1 -1
- package/src/cli.js +209 -36
- package/src/config.js +8 -2
- package/src/control-client.js +118 -1
- package/src/daemon.js +939 -53
- package/src/guardian-client.js +434 -0
- package/src/managed-process.js +45 -15
- package/src/process-guardian.js +601 -0
- package/src/release-group.js +190 -15
- package/src/state-store.js +1 -1
- package/test/config-validation.test.js +22 -0
- package/test/fixtures/pre-split3-daemon-runner.js +30 -0
- package/test/fixtures/pre-split3-daemon.js +1336 -0
- package/test/fixtures/pre-split3-guardian-client.js +293 -0
- package/test/fixtures/pre-split3-process-guardian.js +292 -0
- package/test/fixtures/service-app.js +32 -2
- package/test/guardian-client.test.js +304 -0
- package/test/owner-recovery.test.js +950 -0
- package/test/owner-replacement.test.js +772 -0
- package/test/release-runtime-retention.test.js +1 -1
- package/test/rollbridge.test.js +178 -5
- package/test/shutdown-completion.test.js +1 -1
- package/test/state-store.test.js +12 -0
package/src/daemon.js
CHANGED
|
@@ -3,10 +3,13 @@
|
|
|
3
3
|
import fs from "node:fs/promises"
|
|
4
4
|
import http from "node:http"
|
|
5
5
|
import net from "node:net"
|
|
6
|
+
import crypto from "node:crypto"
|
|
6
7
|
import {isDeepStrictEqual} from "node:util"
|
|
7
8
|
import httpProxy from "http-proxy"
|
|
8
9
|
import {loadConfig} from "./config.js"
|
|
10
|
+
import {openControlSession} from "./control-client.js"
|
|
9
11
|
import EventLog from "./event-log.js"
|
|
12
|
+
import GuardianClient from "./guardian-client.js"
|
|
10
13
|
import ReleaseGroup from "./release-group.js"
|
|
11
14
|
import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
|
|
12
15
|
import {resolveGroupId, resolveUserId} from "./system-ids.js"
|
|
@@ -19,7 +22,11 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
19
22
|
* @typedef {{releaseId?: string, releasePath: string, revision?: string}} DeployArgs
|
|
20
23
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
21
24
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
22
|
-
* @typedef {{
|
|
25
|
+
* @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
26
|
+
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, ownerRecovery: {configDigest: string} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
27
|
+
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
28
|
+
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata}} OwnerRecoverySnapshot
|
|
29
|
+
* @typedef {{boundaryCrossed: boolean, incumbentControl: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
23
30
|
*/
|
|
24
31
|
|
|
25
32
|
export default class RollbridgeDaemon {
|
|
@@ -29,13 +36,15 @@ export default class RollbridgeDaemon {
|
|
|
29
36
|
* @param {import("./config.js").RollbridgeConfig} args.config - Rollbridge config.
|
|
30
37
|
* @param {string} [args.configPath] - Config file path to reload before deploys.
|
|
31
38
|
* @param {(message: string, data?: Record<string, JsonValue>) => void} [args.logger] - Logger.
|
|
39
|
+
* @param {number} [args.legacyIncumbentPid] - Exact incumbent PID supplied by ensure-daemon.
|
|
32
40
|
* @param {import("./daemon-runtime.js").DaemonRuntimeIdentity} [args.runtime] - Immutable daemon runtime identity.
|
|
33
41
|
*/
|
|
34
|
-
constructor({bootstrap, config, configPath, logger, runtime}) {
|
|
42
|
+
constructor({bootstrap, config, configPath, legacyIncumbentPid, logger, runtime}) {
|
|
35
43
|
this.bootstrap = bootstrap ? {...bootstrap} : undefined
|
|
36
44
|
this.config = config
|
|
37
45
|
this.configPath = configPath
|
|
38
46
|
this.runtime = runtime
|
|
47
|
+
this.legacyIncumbentPid = legacyIncumbentPid
|
|
39
48
|
this.eventLog = new EventLog(EVENT_HISTORY_LIMIT)
|
|
40
49
|
|
|
41
50
|
const baseLogger = logger || ((message, data = {}) => console.log(JSON.stringify({at: new Date().toISOString(), data, message})))
|
|
@@ -51,12 +60,14 @@ export default class RollbridgeDaemon {
|
|
|
51
60
|
this.releases = /** @type {Map<string, ReleaseGroup>} */ (new Map())
|
|
52
61
|
this.services = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
|
|
53
62
|
this.servicePorts = /** @type {Record<string, number>} */ ({})
|
|
63
|
+
this.portReservations = /** @type {Set<number>} */ (new Set())
|
|
54
64
|
this.singletons = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
|
|
55
65
|
this.activeRelease = /** @type {ReleaseGroup | undefined} */ (undefined)
|
|
56
66
|
this.proxy = httpProxy.createProxyServer({ws: true, xfwd: true})
|
|
57
67
|
this.proxyServer = /** @type {http.Server | undefined} */ (undefined)
|
|
58
68
|
this.controlServer = /** @type {net.Server | undefined} */ (undefined)
|
|
59
69
|
this.controlSocketOwned = false
|
|
70
|
+
this.boundControlPath = /** @type {string | undefined} */ (undefined)
|
|
60
71
|
this.controlSockets = /** @type {Set<net.Socket>} */ (new Set())
|
|
61
72
|
this.proxyPort = /** @type {number | undefined} */ (undefined)
|
|
62
73
|
this.stopping = false
|
|
@@ -67,8 +78,17 @@ export default class RollbridgeDaemon {
|
|
|
67
78
|
this.stateCleanupEnabled = false
|
|
68
79
|
this.shutdownPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
69
80
|
this.retirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
81
|
+
this.ownerRetired = false
|
|
70
82
|
this.controlClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
83
|
+
this.proxyClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
84
|
+
this.listenerHandoff = /** @type {{control: boolean, proxy: boolean, replacementId: string} | undefined} */ (undefined)
|
|
85
|
+
this.listenerHandoffFailure = /** @type {Error | undefined} */ (undefined)
|
|
86
|
+
this.incumbentListenerControl = /** @type {Awaited<ReturnType<typeof openControlSession>> | undefined} */ (undefined)
|
|
87
|
+
this.ownerTransition = /** @type {OwnerTransition | undefined} */ (undefined)
|
|
88
|
+
this.controlCommandsReady = true
|
|
71
89
|
this.startingReleases = /** @type {Set<ReleaseGroup>} */ (new Set())
|
|
90
|
+
this.guardian = /** @type {GuardianClient | undefined} */ (undefined)
|
|
91
|
+
this.guardianIdentity = /** @type {{pid?: number, socketPath: string, token: string} | undefined} */ (undefined)
|
|
72
92
|
// Still-alive managed processes left by a previous daemon (from statePath), captured at
|
|
73
93
|
// startup and surfaced in status(). The daemon cannot re-manage them, only report them.
|
|
74
94
|
this.orphans = /** @type {{id: string, pid: number, releaseId: string | null}[]} */ ([])
|
|
@@ -82,11 +102,473 @@ export default class RollbridgeDaemon {
|
|
|
82
102
|
* @returns {Promise<void>} Resolves when the requested listeners are ready.
|
|
83
103
|
*/
|
|
84
104
|
async start({exposeControl = true, reportOrphans = true} = {}) {
|
|
85
|
-
if (
|
|
105
|
+
if (this.config.ownerRecovery) await this.initializeOwnerRecovery()
|
|
106
|
+
else if (reportOrphans) await this.reportOrphans()
|
|
86
107
|
await this.startProxy()
|
|
87
108
|
if (exposeControl) await this.exposeControl()
|
|
88
109
|
}
|
|
89
110
|
|
|
111
|
+
/** Connects to the durable process guardian and reconstructs a matching persisted owner snapshot. */
|
|
112
|
+
async initializeOwnerRecovery() {
|
|
113
|
+
if (!this.statePath) throw new Error("ownerRecovery requires statePath")
|
|
114
|
+
const state = await readState(this.statePath)
|
|
115
|
+
const snapshot = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
|
|
116
|
+
const recovery = snapshot?.recovery
|
|
117
|
+
const configDigest = this.ownerRecoveryConfigDigest()
|
|
118
|
+
|
|
119
|
+
if (snapshot && !recovery) throw new Error(`Owner recovery state ${this.statePath} is missing durable guardian identity; refusing to overwrite it.`)
|
|
120
|
+
if (recovery && recovery.configDigest !== configDigest) throw new Error("Owner recovery config identity does not match the persisted owner; refusing cross-authority adoption.")
|
|
121
|
+
if (snapshot && ((this.runtime?.digest ?? null) !== (snapshot.daemonRuntime?.digest ?? null))) {
|
|
122
|
+
throw new Error("Owner recovery runtime identity does not match the persisted owner; use the exact same Rollbridge runtime.")
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const guardianIdentity = recovery?.guardian || {
|
|
126
|
+
socketPath: `${this.statePath}.guardian.sock`,
|
|
127
|
+
token: crypto.randomBytes(32).toString("hex")
|
|
128
|
+
}
|
|
129
|
+
this.guardianIdentity = guardianIdentity
|
|
130
|
+
this.guardian = new GuardianClient(guardianIdentity)
|
|
131
|
+
if (recovery) await this.guardian.connect()
|
|
132
|
+
else {
|
|
133
|
+
await this.guardian.launch()
|
|
134
|
+
guardianIdentity.pid = this.guardian.pid
|
|
135
|
+
}
|
|
136
|
+
await this.guardian.claimOwner(this.config.ownerRecovery?.reconnectGraceMs ?? 30000, this.ownerAuthority())
|
|
137
|
+
this.watchOwnerReplacementEvents()
|
|
138
|
+
|
|
139
|
+
if (snapshot) {
|
|
140
|
+
await this.restoreOwnerState(snapshot, {resumeDrains: false})
|
|
141
|
+
await this.guardian.reconcileInventory()
|
|
142
|
+
for (const release of this.releases.values()) {
|
|
143
|
+
if (release.state === "draining") void this.drainAndPrune(release, release.config)
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
this.persistenceEnabled = true
|
|
148
|
+
await this.persistState({throwOnError: true})
|
|
149
|
+
}
|
|
150
|
+
this.stateCleanupEnabled = true
|
|
151
|
+
await this.publishOwnerState()
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** @returns {string} Stable identity for same-authority recovery. */
|
|
155
|
+
ownerRecoveryConfigDigest() {
|
|
156
|
+
return ownerConfigDigest(this.config)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Installs event-driven drain fencing for the next prepared owner transaction. */
|
|
160
|
+
watchOwnerReplacementEvents() {
|
|
161
|
+
if (!this.guardian) throw new Error("Owner replacement event fencing requires the durable guardian")
|
|
162
|
+
this.guardian.onEvent("replacement-prepared", () => {
|
|
163
|
+
for (const release of this.releases.values()) release.pauseDrainForOwnerHandoff()
|
|
164
|
+
})
|
|
165
|
+
this.guardian.onEvent("replacement-aborted", () => {
|
|
166
|
+
if (this.listenerHandoff || this.ownerRetired) return
|
|
167
|
+
for (const release of this.releases.values()) {
|
|
168
|
+
release.resumeDrainAfterOwnerHandoff()
|
|
169
|
+
if (release.state === "draining") void this.drainAndPrune(release, release.config)
|
|
170
|
+
}
|
|
171
|
+
})
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** @returns {{configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}} Exact authority fence. */
|
|
175
|
+
ownerAuthority() {
|
|
176
|
+
return {configDigest: this.ownerRecoveryConfigDigest(), runtime: this.runtime ? {...this.runtime} : null}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* @param {OwnerRecoverySnapshot} snapshot - Validated persisted owner state.
|
|
181
|
+
* @param {object} [options] - Recovery definition options.
|
|
182
|
+
* @param {import("./config.js").RollbridgeConfig} [options.config] - Owner config for daemon-wide processes.
|
|
183
|
+
* @param {Record<string, import("./config.js").RollbridgeConfig>} [options.releaseConfigs] - Exact generation configs.
|
|
184
|
+
* @param {boolean} [options.resumeDrains] - Whether to resume draining generations immediately.
|
|
185
|
+
*/
|
|
186
|
+
async restoreOwnerState(snapshot, {config = this.config, releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({}), resumeDrains = true} = {}) {
|
|
187
|
+
if (!Array.isArray(snapshot.releases) || (snapshot.activeReleaseId !== null && typeof snapshot.activeReleaseId !== "string")) {
|
|
188
|
+
throw new Error("Owner recovery state is partial or corrupt; active release metadata is required.")
|
|
189
|
+
}
|
|
190
|
+
if (snapshot.activeReleaseId === null && snapshot.releases.length === 0) return
|
|
191
|
+
this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
|
|
192
|
+
this.ownerTransition = snapshot.ownerTransition ? {...snapshot.ownerTransition} : undefined
|
|
193
|
+
|
|
194
|
+
for (const releaseStatus of snapshot.releases) {
|
|
195
|
+
if (releaseStatus.state !== "active" && releaseStatus.state !== "draining") continue
|
|
196
|
+
const release = new ReleaseGroup({
|
|
197
|
+
config: releaseConfigs[releaseStatus.releaseId] || config,
|
|
198
|
+
logger: this.logger,
|
|
199
|
+
portReservations: this.portReservations,
|
|
200
|
+
processFactory: (key, definition) => this.guardianProcess(key, definition),
|
|
201
|
+
releaseId: releaseStatus.releaseId,
|
|
202
|
+
releasePath: releaseStatus.releasePath,
|
|
203
|
+
revision: releaseStatus.revision,
|
|
204
|
+
servicePorts: this.servicePorts,
|
|
205
|
+
shouldStart: () => !this.stopping
|
|
206
|
+
})
|
|
207
|
+
|
|
208
|
+
await release.restore(releaseStatus)
|
|
209
|
+
this.releases.set(release.releaseId, release)
|
|
210
|
+
if (release.releaseId === snapshot.activeReleaseId) this.activeRelease = release
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
if (snapshot.activeReleaseId !== null && !this.activeRelease) throw new Error(`Owner recovery state does not contain active release ${snapshot.activeReleaseId}.`)
|
|
214
|
+
const definitionRelease = this.activeRelease || [...this.releases.values()].at(-1)
|
|
215
|
+
if (!definitionRelease) throw new Error("Owner recovery state has no release definition for owned processes.")
|
|
216
|
+
if (!this.activeRelease && snapshot.singletons.length > 0) throw new Error("Owner recovery state has release-owned singletons without an active release identity.")
|
|
217
|
+
for (const serviceStatus of snapshot.services) {
|
|
218
|
+
const processConfig = config.processes.find((candidate) => candidate.id === serviceStatus.id && candidate.policy === "service" && candidate.deployStrategy !== "handoff")
|
|
219
|
+
|
|
220
|
+
if (!processConfig) throw new Error(`Owner recovery state contains unknown service ${serviceStatus.id}.`)
|
|
221
|
+
const service = definitionRelease.buildProcess(processConfig, {guardianKey: `service:${serviceStatus.id}`, shouldRestart: () => !this.stopping})
|
|
222
|
+
|
|
223
|
+
await this.recoverGuardianProcess(service)
|
|
224
|
+
this.services.set(serviceStatus.id, service)
|
|
225
|
+
if (definitionRelease.ports[serviceStatus.id] !== undefined) {
|
|
226
|
+
const port = definitionRelease.ports[serviceStatus.id]
|
|
227
|
+
|
|
228
|
+
if (this.portReservations.has(port)) throw new Error(`Persisted daemon service ${serviceStatus.id} port ${port} is already reserved by a live generation`)
|
|
229
|
+
this.portReservations.add(port)
|
|
230
|
+
this.servicePorts[serviceStatus.id] = port
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
for (const singletonStatus of snapshot.singletons) {
|
|
234
|
+
const processConfig = config.processes.find((candidate) => candidate.id === singletonStatus.id && candidate.policy === "singleton")
|
|
235
|
+
|
|
236
|
+
if (!processConfig) throw new Error(`Owner recovery state contains unknown singleton ${singletonStatus.id}.`)
|
|
237
|
+
const singleton = definitionRelease.buildProcess(processConfig, {guardianKey: `singleton:${definitionRelease.releaseId}:${singletonStatus.id}`})
|
|
238
|
+
|
|
239
|
+
await this.recoverGuardianProcess(singleton)
|
|
240
|
+
this.singletons.set(singletonStatus.id, singleton)
|
|
241
|
+
}
|
|
242
|
+
for (const release of this.releases.values()) {
|
|
243
|
+
if (resumeDrains && release.state === "draining") void this.drainAndPrune(release, release.config)
|
|
244
|
+
}
|
|
245
|
+
this.logger("owner state recovered", {activeReleaseId: this.activeRelease?.releaseId ?? null, releases: this.releases.size})
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Publishes full normalized definitions only over the authenticated guardian channel. */
|
|
249
|
+
async publishOwnerState() {
|
|
250
|
+
if (!this.guardian) return
|
|
251
|
+
await this.guardian.publishOwnerState({
|
|
252
|
+
authority: this.ownerAuthority(),
|
|
253
|
+
config: this.config,
|
|
254
|
+
releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
|
|
255
|
+
snapshot: this.status()
|
|
256
|
+
})
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Prepares from private guardian state, starts listeners, then asks the committed owner
|
|
261
|
+
* to atomically fence itself and transfer authority.
|
|
262
|
+
*/
|
|
263
|
+
async replaceIncompatibleOwner() {
|
|
264
|
+
if (!this.statePath || !this.config.ownerRecovery) throw new Error("Atomic owner replacement requires ownerRecovery and the same statePath transaction anchor")
|
|
265
|
+
this.controlCommandsReady = false
|
|
266
|
+
const state = await readState(this.statePath)
|
|
267
|
+
const persisted = state && typeof state === "object" && !Array.isArray(state) ? /** @type {OwnerRecoverySnapshot} */ (state) : undefined
|
|
268
|
+
|
|
269
|
+
if (!persisted?.recovery?.guardian) throw new Error(`No committed durable owner transaction was found at ${this.statePath}`)
|
|
270
|
+
this.guardianIdentity = persisted.recovery.guardian
|
|
271
|
+
this.guardian = new GuardianClient(this.guardianIdentity)
|
|
272
|
+
await this.guardian.connect()
|
|
273
|
+
this.watchOwnerReplacementEvents()
|
|
274
|
+
const persistedAuthority = {
|
|
275
|
+
configDigest: persisted.recovery.configDigest,
|
|
276
|
+
runtime: persisted.daemonRuntime ? {...persisted.daemonRuntime} : null
|
|
277
|
+
}
|
|
278
|
+
let legacyBridge
|
|
279
|
+
let prepared
|
|
280
|
+
|
|
281
|
+
try {
|
|
282
|
+
prepared = await this.guardian.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
|
|
283
|
+
} catch (error) {
|
|
284
|
+
legacyBridge = await this.prepareLegacyOwnerReplacement({
|
|
285
|
+
error: error instanceof Error ? error : String(error),
|
|
286
|
+
persisted,
|
|
287
|
+
persistedAuthority
|
|
288
|
+
})
|
|
289
|
+
prepared = legacyBridge.prepared
|
|
290
|
+
}
|
|
291
|
+
const preparedStatus = await this.guardian.replacementStatus()
|
|
292
|
+
const transfer = /** @type {{config: import("./config.js").RollbridgeConfig, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, snapshot: OwnerRecoverySnapshot}} */ (prepared.ownerState)
|
|
293
|
+
|
|
294
|
+
if (!transfer?.config || !transfer.snapshot) throw new Error("Committed owner published incomplete replacement state")
|
|
295
|
+
await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, releaseConfigs: transfer.releaseConfigs, resumeDrains: false})
|
|
296
|
+
for (const release of this.releases.values()) release.preserveConfigOnRetirement = true
|
|
297
|
+
this.logger("owner replacement candidate prepared", {activeReleaseId: this.activeRelease?.releaseId ?? null, replacementId: prepared.replacementId})
|
|
298
|
+
|
|
299
|
+
let committedAuthority = false
|
|
300
|
+
let stagingControlPath
|
|
301
|
+
let finalControlPublished = false
|
|
302
|
+
let listenersYielded = false
|
|
303
|
+
let retainIncumbentControl = false
|
|
304
|
+
let incumbentControl = legacyBridge?.incumbentControl
|
|
305
|
+
|
|
306
|
+
try {
|
|
307
|
+
if (this.config.control.path !== transfer.snapshot.control.path) {
|
|
308
|
+
const finalSocket = await inspectControlSocket(this.config.control.path)
|
|
309
|
+
|
|
310
|
+
if (finalSocket.alive) throw new Error(`Owner replacement final control socket ${this.config.control.path} already answers another live process`)
|
|
311
|
+
try {
|
|
312
|
+
await fs.lstat(this.config.control.path)
|
|
313
|
+
throw new Error(`Owner replacement final control socket ${this.config.control.path} already exists; refusing to replace it`)
|
|
314
|
+
} catch (error) {
|
|
315
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
stagingControlPath = `${this.config.control.path}.replacement-${process.pid}`
|
|
320
|
+
await this.startControlServer(stagingControlPath)
|
|
321
|
+
const sharedFixedProxy = this.config.proxy.port !== 0 &&
|
|
322
|
+
transfer.snapshot.proxy.host === this.config.proxy.host && transfer.snapshot.proxy.port === this.config.proxy.port
|
|
323
|
+
|
|
324
|
+
if (!sharedFixedProxy) await this.startProxy()
|
|
325
|
+
if (legacyBridge) {
|
|
326
|
+
await this.crossLegacyDisruptiveBoundary(legacyBridge)
|
|
327
|
+
} else if (preparedStatus.ownerClaimed) {
|
|
328
|
+
incumbentControl = await openControlSession(transfer.snapshot.control.path)
|
|
329
|
+
const listenerSession = incumbentControl
|
|
330
|
+
|
|
331
|
+
incumbentControl.onEvent((event) => this.handleIncumbentListenerEvent(event, listenerSession))
|
|
332
|
+
await incumbentControl.request({
|
|
333
|
+
command: "yield-owner-listeners",
|
|
334
|
+
control: transfer.snapshot.control.path === this.config.control.path,
|
|
335
|
+
proxy: true,
|
|
336
|
+
replacementId: prepared.replacementId
|
|
337
|
+
})
|
|
338
|
+
listenersYielded = true
|
|
339
|
+
}
|
|
340
|
+
if (sharedFixedProxy) await this.startProxy()
|
|
341
|
+
await fs.rename(stagingControlPath, this.config.control.path)
|
|
342
|
+
finalControlPublished = true
|
|
343
|
+
this.boundControlPath = this.config.control.path
|
|
344
|
+
const committed = this.guardian.waitForEvent("replacement-committed")
|
|
345
|
+
const staged = await this.guardian.stageOwnerReplacement(prepared.replacementId, {
|
|
346
|
+
authority: this.ownerAuthority(),
|
|
347
|
+
config: this.config,
|
|
348
|
+
releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
|
|
349
|
+
snapshot: this.status()
|
|
350
|
+
})
|
|
351
|
+
|
|
352
|
+
if (!staged.committed) {
|
|
353
|
+
try {
|
|
354
|
+
if (!incumbentControl) throw new Error("Owner replacement incumbent control session is unavailable")
|
|
355
|
+
await incumbentControl.request({command: "commit-owner-replacement", replacementId: prepared.replacementId})
|
|
356
|
+
} catch (error) {
|
|
357
|
+
const status = await this.guardian.replacementStatus()
|
|
358
|
+
|
|
359
|
+
if (status.committedReplacementId !== prepared.replacementId || !status.ownerClaimed) throw error
|
|
360
|
+
this.logger("owner replacement commit response lost; guardian commit confirmed", {replacementId: prepared.replacementId})
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
await committed
|
|
364
|
+
committedAuthority = true
|
|
365
|
+
if (!legacyBridge && incumbentControl && [...this.releases.values()].some((release) => release.hasTransferredConnections())) {
|
|
366
|
+
this.incumbentListenerControl = incumbentControl
|
|
367
|
+
retainIncumbentControl = true
|
|
368
|
+
}
|
|
369
|
+
for (const release of this.releases.values()) {
|
|
370
|
+
if (release.state === "draining") void this.drainAndPrune(release, release.config)
|
|
371
|
+
}
|
|
372
|
+
this.startStatePersistence()
|
|
373
|
+
await this.persistState({throwOnError: true})
|
|
374
|
+
await this.publishOwnerState()
|
|
375
|
+
this.controlCommandsReady = true
|
|
376
|
+
this.logger("owner replacement committed", {replacementId: prepared.replacementId})
|
|
377
|
+
} catch (error) {
|
|
378
|
+
if (committedAuthority) {
|
|
379
|
+
this.logger("owner replacement post-commit finalization failed", {error: error instanceof Error ? error.message : String(error), replacementId: prepared.replacementId})
|
|
380
|
+
throw error
|
|
381
|
+
}
|
|
382
|
+
await this.closeServer(this.controlServer)
|
|
383
|
+
await this.closeServer(this.proxyServer)
|
|
384
|
+
if (finalControlPublished) await this.removeControlSocket()
|
|
385
|
+
else if (stagingControlPath) await fs.rm(stagingControlPath, {force: true})
|
|
386
|
+
let abortError
|
|
387
|
+
|
|
388
|
+
if (listenersYielded && incumbentControl) {
|
|
389
|
+
try {
|
|
390
|
+
await incumbentControl.request({command: "abort-owner-listener-handoff", replacementId: prepared.replacementId})
|
|
391
|
+
} catch (failure) {
|
|
392
|
+
abortError = failure instanceof Error ? failure : new Error(String(failure))
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
if (legacyBridge && !legacyBridge.boundaryCrossed) {
|
|
396
|
+
try {
|
|
397
|
+
await this.guardian.abandonLegacyUpgrade()
|
|
398
|
+
} catch (failure) {
|
|
399
|
+
abortError = failure instanceof Error ? failure : new Error(String(failure))
|
|
400
|
+
}
|
|
401
|
+
} else {
|
|
402
|
+
this.guardian.disconnect()
|
|
403
|
+
}
|
|
404
|
+
incumbentControl?.close()
|
|
405
|
+
if (legacyBridge?.boundaryCrossed) {
|
|
406
|
+
this.logger("legacy disruptive owner replacement failed after incumbent exit", {
|
|
407
|
+
error: error instanceof Error ? error.message : String(error),
|
|
408
|
+
recoveryStatePath: this.statePath,
|
|
409
|
+
replacementId: prepared.replacementId
|
|
410
|
+
})
|
|
411
|
+
}
|
|
412
|
+
if (abortError) throw new AggregateError([error, abortError], `Owner replacement failed and recovery cleanup failed: ${abortError.message}`, {cause: error})
|
|
413
|
+
throw error
|
|
414
|
+
} finally {
|
|
415
|
+
if (!retainIncumbentControl) incumbentControl?.close()
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* Applies authenticated live-connection state from the retired listener owner.
|
|
421
|
+
* @param {Record<string, JsonValue>} event - Incumbent control event.
|
|
422
|
+
* @param {{close: () => void}} session - Exact incumbent session.
|
|
423
|
+
*/
|
|
424
|
+
handleIncumbentListenerEvent(event, session) {
|
|
425
|
+
if (event.event !== "owner-connection-state") return
|
|
426
|
+
const releaseId = stringOrUndefined(event.releaseId)
|
|
427
|
+
const connections = event.connections
|
|
428
|
+
|
|
429
|
+
if (!releaseId || !connections || typeof connections !== "object" || Array.isArray(connections)) {
|
|
430
|
+
throw new Error("Incumbent listener sent invalid connection state")
|
|
431
|
+
}
|
|
432
|
+
const release = this.releases.get(releaseId)
|
|
433
|
+
|
|
434
|
+
if (!release) throw new Error(`Incumbent listener reported unknown release ${releaseId}`)
|
|
435
|
+
release.setTransferredConnections({
|
|
436
|
+
http: requiredNonNegativeInteger(connections.http, "connections.http"),
|
|
437
|
+
websocket: requiredNonNegativeInteger(connections.websocket, "connections.websocket")
|
|
438
|
+
})
|
|
439
|
+
if (this.incumbentListenerControl === session && ![...this.releases.values()].some((candidate) => candidate.hasTransferredConnections())) {
|
|
440
|
+
this.incumbentListenerControl = undefined
|
|
441
|
+
session.close()
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* Authenticates and prepares the one-time disruptive bridge for a genuine pre-split guardian.
|
|
447
|
+
* @param {{error: Error | string, persisted: OwnerRecoverySnapshot, persistedAuthority: {configDigest: string, runtime: import("./daemon-runtime.js").DaemonRuntimeIdentity | null}}} options - Legacy evidence.
|
|
448
|
+
* @returns {Promise<LegacyOwnerBridge>} Prepared bridge.
|
|
449
|
+
*/
|
|
450
|
+
async prepareLegacyOwnerReplacement({error, persisted, persistedAuthority}) {
|
|
451
|
+
const diagnostic = error instanceof Error ? error.message : String(error)
|
|
452
|
+
const legacyProcessKey = legacyGuardianKeys(persisted)[0]
|
|
453
|
+
|
|
454
|
+
if (!isLegacyGuardianPrepareDiagnostic(diagnostic)) throw error
|
|
455
|
+
if (!legacyProcessKey) throw new Error("Legacy disruptive owner replacement requires an exact guardian-owned process registration in durable state", {cause: error})
|
|
456
|
+
if (diagnostic !== "Unknown guardian command: prepare-owner-replacement") {
|
|
457
|
+
try {
|
|
458
|
+
await this.guardian?.request({
|
|
459
|
+
authority: persistedAuthority,
|
|
460
|
+
command: "prepare-owner-replacement",
|
|
461
|
+
key: legacyProcessKey,
|
|
462
|
+
nextAuthority: this.ownerAuthority()
|
|
463
|
+
})
|
|
464
|
+
throw new Error("Legacy guardian protocol probe unexpectedly accepted owner replacement")
|
|
465
|
+
} catch (probeError) {
|
|
466
|
+
if (!(probeError instanceof Error) || probeError.message !== "Unknown guardian command: prepare-owner-replacement") {
|
|
467
|
+
throw new Error("Guardian does not match the authenticated pre-split replacement protocol signature", {cause: probeError})
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
if (persisted.recovery.configDigest !== this.ownerRecoveryConfigDigest()) {
|
|
472
|
+
throw new Error("The one-time legacy guardian bridge requires the incumbent config identity unchanged; retry the config change after the bridge establishes split-3 authority")
|
|
473
|
+
}
|
|
474
|
+
if (!this.legacyIncumbentPid) throw new Error("The authenticated pre-split guardian requires an exact incumbent PID from ensure-daemon for the disruptive bridge")
|
|
475
|
+
if (!this.guardianIdentity?.pid) throw new Error("The authenticated pre-split guardian state is missing its exact guardian PID")
|
|
476
|
+
const legacyGuardian = this.guardian
|
|
477
|
+
|
|
478
|
+
if (!legacyGuardian) throw new Error("Legacy disruptive replacement is missing its authenticated guardian connection")
|
|
479
|
+
|
|
480
|
+
await verifyLegacyGuardianProcess(this.guardianIdentity.pid, this.guardianIdentity.socketPath)
|
|
481
|
+
const incumbentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, persisted.control.path)
|
|
482
|
+
const incumbentControl = await openControlSession(persisted.control.path)
|
|
483
|
+
let upgraded
|
|
484
|
+
|
|
485
|
+
try {
|
|
486
|
+
const incumbentStatus = await incumbentControl.request({command: "status"})
|
|
487
|
+
|
|
488
|
+
assertLegacyIncumbentStatus(incumbentStatus, persisted)
|
|
489
|
+
const ownerState = {
|
|
490
|
+
authority: persistedAuthority,
|
|
491
|
+
config: this.config,
|
|
492
|
+
releaseConfigs: Object.fromEntries(persisted.releases.map((release) => [release.releaseId, this.config])),
|
|
493
|
+
snapshot: persisted
|
|
494
|
+
}
|
|
495
|
+
const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
|
|
496
|
+
socketPath: `${this.statePath}.split3-guardian.sock`,
|
|
497
|
+
token: crypto.randomBytes(32).toString("hex")
|
|
498
|
+
})
|
|
499
|
+
|
|
500
|
+
await assertPathAbsent(upgradedIdentity.socketPath, "Legacy upgrade guardian socket")
|
|
501
|
+
upgraded = await legacyGuardian.upgradeLegacyGuardian({ownerState, ...upgradedIdentity})
|
|
502
|
+
upgradedIdentity.pid = upgraded.pid
|
|
503
|
+
legacyGuardian.disconnect()
|
|
504
|
+
this.guardian = upgraded
|
|
505
|
+
this.guardianIdentity = upgradedIdentity
|
|
506
|
+
const prepared = await upgraded.prepareOwnerReplacement(persistedAuthority, this.ownerAuthority())
|
|
507
|
+
const recoverySnapshot = /** @type {OwnerRecoverySnapshot} */ ({
|
|
508
|
+
...persisted,
|
|
509
|
+
recovery: {...persisted.recovery, guardian: upgradedIdentity}
|
|
510
|
+
})
|
|
511
|
+
|
|
512
|
+
this.logger("legacy owner replacement bridge prepared", {
|
|
513
|
+
guardianPid: upgradedIdentity.pid ?? null,
|
|
514
|
+
incumbentPid: this.legacyIncumbentPid,
|
|
515
|
+
replacementId: prepared.replacementId
|
|
516
|
+
})
|
|
517
|
+
return {boundaryCrossed: false, incumbentControl, incumbentStartTime, prepared, recoverySnapshot}
|
|
518
|
+
} catch (upgradeError) {
|
|
519
|
+
incumbentControl.close()
|
|
520
|
+
if (upgraded) await upgraded.abandonLegacyUpgrade()
|
|
521
|
+
throw upgradeError
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/**
|
|
526
|
+
* Crosses the explicitly disruptive legacy-only boundary after candidate reconstruction.
|
|
527
|
+
* @param {LegacyOwnerBridge} bridge - Prepared bridge.
|
|
528
|
+
*/
|
|
529
|
+
async crossLegacyDisruptiveBoundary(bridge) {
|
|
530
|
+
if (!this.legacyIncumbentPid || !this.statePath) throw new Error("Legacy disruptive boundary is missing its exact incumbent identity")
|
|
531
|
+
const currentStartTime = await verifyLegacyDaemonProcess(this.legacyIncumbentPid, this.configPath, bridge.recoverySnapshot.control.path)
|
|
532
|
+
|
|
533
|
+
if (currentStartTime !== bridge.incumbentStartTime) throw new Error("Legacy incumbent PID identity changed before the disruptive boundary")
|
|
534
|
+
this.logger("legacy owner replacement disruptive boundary", {
|
|
535
|
+
disruptive: true,
|
|
536
|
+
incumbentPid: this.legacyIncumbentPid,
|
|
537
|
+
reason: "pre-split guardian and daemon lacked atomic replacement protocol"
|
|
538
|
+
})
|
|
539
|
+
process.kill(this.legacyIncumbentPid, "SIGKILL")
|
|
540
|
+
bridge.boundaryCrossed = true
|
|
541
|
+
await bridge.incumbentControl.closed()
|
|
542
|
+
await writeState(this.statePath, bridge.recoverySnapshot)
|
|
543
|
+
this.ownerTransition = /** @type {OwnerTransition} */ ({
|
|
544
|
+
disruptive: true,
|
|
545
|
+
mode: "legacy-first-upgrade",
|
|
546
|
+
reason: "pre-split guardian and daemon lacked atomic replacement protocol"
|
|
547
|
+
})
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* @param {string} key - Guardian key.
|
|
552
|
+
* @param {Parameters<GuardianClient["process"]>[1]} definition - Process definition.
|
|
553
|
+
* @returns {import("./managed-process.js").default} Guardian-backed managed process.
|
|
554
|
+
*/
|
|
555
|
+
guardianProcess(key, definition) {
|
|
556
|
+
if (!this.guardian) throw new Error("Process guardian is not initialized")
|
|
557
|
+
return this.guardian.process(key, definition)
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** @param {import("./managed-process.js").default} processInstance - Guardian-backed process. */
|
|
561
|
+
async recoverGuardianProcess(processInstance) {
|
|
562
|
+
if (!("recover" in processInstance) || typeof processInstance.recover !== "function") throw new Error(`Managed process ${processInstance.id} is not guardian-backed`)
|
|
563
|
+
await processInstance.recover()
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/** Releases only resources created by a fenced startup loser. */
|
|
567
|
+
async abandonOwnerRecoveryAttempt() {
|
|
568
|
+
this.guardian?.disconnect()
|
|
569
|
+
await this.closeServer(this.proxyServer)
|
|
570
|
+
}
|
|
571
|
+
|
|
90
572
|
/** @returns {Promise<void>} Exposes control commands and begins periodic state persistence. */
|
|
91
573
|
async exposeControl() {
|
|
92
574
|
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
@@ -102,7 +584,9 @@ export default class RollbridgeDaemon {
|
|
|
102
584
|
this.startStatePersistence()
|
|
103
585
|
}
|
|
104
586
|
|
|
105
|
-
/**
|
|
587
|
+
/**
|
|
588
|
+
* @returns {Promise<void>} Starts the stable local proxy.
|
|
589
|
+
*/
|
|
106
590
|
async startProxy() {
|
|
107
591
|
const server = http.createServer((request, response) => this.proxyHttp(request, response))
|
|
108
592
|
|
|
@@ -111,7 +595,7 @@ export default class RollbridgeDaemon {
|
|
|
111
595
|
|
|
112
596
|
await new Promise((resolve, reject) => {
|
|
113
597
|
server.once("error", reject)
|
|
114
|
-
server.listen(this.config.proxy.
|
|
598
|
+
server.listen({host: this.config.proxy.host, port: this.config.proxy.port}, () => {
|
|
115
599
|
const address = server.address()
|
|
116
600
|
this.proxyPort = address && typeof address === "object" ? address.port : this.config.proxy.port
|
|
117
601
|
this.logger("proxy listening", {host: this.config.proxy.host, port: this.proxyPort})
|
|
@@ -120,35 +604,46 @@ export default class RollbridgeDaemon {
|
|
|
120
604
|
})
|
|
121
605
|
}
|
|
122
606
|
|
|
123
|
-
/**
|
|
124
|
-
|
|
607
|
+
/**
|
|
608
|
+
* @param {string} [socketPath] - Socket path to bind.
|
|
609
|
+
* @param {boolean} [applyMetadata] - Whether to apply configured mode and ownership.
|
|
610
|
+
* @returns {Promise<void>} Starts the control socket.
|
|
611
|
+
*/
|
|
612
|
+
async startControlServer(socketPath = this.config.control.path, applyMetadata = true) {
|
|
125
613
|
const server = net.createServer((socket) => this.handleControlSocket(socket))
|
|
126
614
|
|
|
127
615
|
this.controlServer = server
|
|
128
|
-
await this.prepareControlSocketPath()
|
|
616
|
+
await this.prepareControlSocketPath(socketPath)
|
|
129
617
|
|
|
130
618
|
await new Promise((resolve, reject) => {
|
|
131
619
|
server.once("error", reject)
|
|
132
|
-
server.listen(
|
|
620
|
+
server.listen(socketPath, () => {
|
|
133
621
|
this.controlSocketOwned = true
|
|
134
|
-
this.
|
|
622
|
+
this.boundControlPath = socketPath
|
|
623
|
+
this.logger("control socket listening", {path: socketPath})
|
|
135
624
|
resolve(undefined)
|
|
136
625
|
})
|
|
137
626
|
})
|
|
138
627
|
|
|
139
|
-
if (this.
|
|
140
|
-
|
|
141
|
-
}
|
|
628
|
+
if (applyMetadata) await this.applyControlSocketMetadata(socketPath)
|
|
629
|
+
}
|
|
142
630
|
|
|
143
|
-
|
|
631
|
+
/**
|
|
632
|
+
* @param {string} socketPath - Bound socket path.
|
|
633
|
+
* @returns {Promise<void>} Metadata application completion.
|
|
634
|
+
*/
|
|
635
|
+
async applyControlSocketMetadata(socketPath) {
|
|
636
|
+
if (this.config.control.mode !== undefined) await fs.chmod(socketPath, this.config.control.mode)
|
|
637
|
+
await this.applyControlSocketOwnership(socketPath)
|
|
144
638
|
}
|
|
145
639
|
|
|
146
640
|
/**
|
|
147
641
|
* Applies control.owner/control.group to the bound socket via chown, resolving names to ids.
|
|
642
|
+
* @param {string} [socketPath] - Bound socket path.
|
|
148
643
|
* @returns {Promise<void>} Resolves once ownership is applied (no-op when neither is set).
|
|
149
644
|
*/
|
|
150
|
-
async applyControlSocketOwnership() {
|
|
151
|
-
const {group, owner
|
|
645
|
+
async applyControlSocketOwnership(socketPath = this.config.control.path) {
|
|
646
|
+
const {group, owner} = this.config.control
|
|
152
647
|
|
|
153
648
|
if (owner === undefined && group === undefined) return
|
|
154
649
|
|
|
@@ -165,16 +660,19 @@ export default class RollbridgeDaemon {
|
|
|
165
660
|
}
|
|
166
661
|
}
|
|
167
662
|
|
|
168
|
-
/**
|
|
169
|
-
|
|
170
|
-
|
|
663
|
+
/**
|
|
664
|
+
* @param {string} [socketPath] - Socket path to inspect and prepare.
|
|
665
|
+
* @returns {Promise<void>} Removes a stale Unix socket before binding, or fails clearly when a daemon is alive.
|
|
666
|
+
*/
|
|
667
|
+
async prepareControlSocketPath(socketPath = this.config.control.path) {
|
|
668
|
+
const existing = await inspectControlSocket(socketPath)
|
|
171
669
|
|
|
172
670
|
if (existing.alive) {
|
|
173
|
-
throw new Error(controlSocketBusyMessage(
|
|
671
|
+
throw new Error(controlSocketBusyMessage(socketPath, existing))
|
|
174
672
|
}
|
|
175
673
|
|
|
176
674
|
try {
|
|
177
|
-
await fs.rm(
|
|
675
|
+
await fs.rm(socketPath, {force: true})
|
|
178
676
|
} catch (error) {
|
|
179
677
|
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return
|
|
180
678
|
throw error
|
|
@@ -322,12 +820,15 @@ export default class RollbridgeDaemon {
|
|
|
322
820
|
const data = /** @type {Record<string, JsonValue>} */ (command)
|
|
323
821
|
const commandName = data.command
|
|
324
822
|
|
|
823
|
+
if (!this.controlCommandsReady) throw new Error("Rollbridge replacement candidate is not committed and ready")
|
|
824
|
+
if (this.ownerRetired && commandName !== "status") throw new Error("Rollbridge owner authority has been transferred")
|
|
825
|
+
|
|
325
826
|
if (commandName === "deploy") {
|
|
326
|
-
return await this.deploy({
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
827
|
+
return await this.executeOwnerMutation("deploy", async () => await this.deploy({
|
|
828
|
+
releaseId: stringOrUndefined(data.releaseId),
|
|
829
|
+
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
830
|
+
revision: stringOrUndefined(data.revision)
|
|
831
|
+
}))
|
|
331
832
|
}
|
|
332
833
|
|
|
333
834
|
if (commandName === "status") {
|
|
@@ -339,19 +840,21 @@ export default class RollbridgeDaemon {
|
|
|
339
840
|
}
|
|
340
841
|
|
|
341
842
|
if (commandName === "stop") {
|
|
342
|
-
await this.
|
|
343
|
-
|
|
843
|
+
return await this.executeOwnerMutation("stop", async () => {
|
|
844
|
+
await this.stopRelease(stringOrUndefined(data.releaseId))
|
|
845
|
+
return this.status()
|
|
846
|
+
})
|
|
344
847
|
}
|
|
345
848
|
|
|
346
849
|
if (commandName === "restart") {
|
|
347
|
-
return await this.restartProcesses({
|
|
850
|
+
return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
|
|
348
851
|
policy: stringOrUndefined(data.policy),
|
|
349
852
|
processId: stringOrUndefined(data.processId)
|
|
350
|
-
})
|
|
853
|
+
}))
|
|
351
854
|
}
|
|
352
855
|
|
|
353
856
|
if (commandName === "rollback") {
|
|
354
|
-
return await this.rollback({releaseId: stringOrUndefined(data.releaseId)})
|
|
857
|
+
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
|
|
355
858
|
}
|
|
356
859
|
|
|
357
860
|
if (commandName === "shutdown") {
|
|
@@ -370,9 +873,149 @@ export default class RollbridgeDaemon {
|
|
|
370
873
|
return {message: "owner retired"}
|
|
371
874
|
}
|
|
372
875
|
|
|
876
|
+
if (commandName === "commit-owner-replacement") {
|
|
877
|
+
const replacementId = requiredString(data.replacementId, "replacementId")
|
|
878
|
+
|
|
879
|
+
if (!this.guardian) throw new Error("Atomic owner replacement requires the durable process guardian")
|
|
880
|
+
let committed = false
|
|
881
|
+
|
|
882
|
+
try {
|
|
883
|
+
await this.guardian.commitOwnerReplacement(replacementId)
|
|
884
|
+
committed = true
|
|
885
|
+
await this.retireCommittedOwner(controlSocket)
|
|
886
|
+
await this.guardian.finalizeOwnerReplacement(replacementId)
|
|
887
|
+
} finally {
|
|
888
|
+
if (committed) this.guardian.disconnect()
|
|
889
|
+
}
|
|
890
|
+
return {message: "owner replacement committed"}
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
if (commandName === "yield-owner-listeners") {
|
|
894
|
+
await this.yieldOwnerListeners({
|
|
895
|
+
completionSocket: controlSocket,
|
|
896
|
+
control: data.control === true,
|
|
897
|
+
proxy: data.proxy === true,
|
|
898
|
+
replacementId: requiredString(data.replacementId, "replacementId")
|
|
899
|
+
})
|
|
900
|
+
return {message: "owner listeners yielded"}
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
if (commandName === "abort-owner-listener-handoff") {
|
|
904
|
+
const replacementId = requiredString(data.replacementId, "replacementId")
|
|
905
|
+
|
|
906
|
+
if (!this.listenerHandoff || this.listenerHandoff.replacementId !== replacementId) throw new Error("Owner listener handoff is not the prepared transaction")
|
|
907
|
+
await this.resumeYieldedListeners()
|
|
908
|
+
return {message: "owner listeners resumed"}
|
|
909
|
+
}
|
|
910
|
+
|
|
373
911
|
throw new Error(`Unknown command: ${String(commandName)}`)
|
|
374
912
|
}
|
|
375
913
|
|
|
914
|
+
/**
|
|
915
|
+
* Runs one owner mutation under the guardian's prepare/commit exclusion fence.
|
|
916
|
+
* @param {string} operation - Mutation diagnostic name.
|
|
917
|
+
* @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
|
|
918
|
+
* @returns {Promise<Record<string, JsonValue>>} Operation result.
|
|
919
|
+
*/
|
|
920
|
+
async executeOwnerMutation(operation, callback) {
|
|
921
|
+
if (!this.guardian) return await callback()
|
|
922
|
+
const mutationId = await this.guardian.beginOwnerMutation(operation)
|
|
923
|
+
let result
|
|
924
|
+
let operationError
|
|
925
|
+
|
|
926
|
+
try {
|
|
927
|
+
result = await callback()
|
|
928
|
+
} catch (error) {
|
|
929
|
+
operationError = error instanceof Error ? error : new Error(String(error))
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
const finalizationErrors = /** @type {Error[]} */ ([])
|
|
933
|
+
|
|
934
|
+
try {
|
|
935
|
+
await this.publishOwnerState()
|
|
936
|
+
} catch (error) {
|
|
937
|
+
finalizationErrors.push(error instanceof Error ? error : new Error(String(error)))
|
|
938
|
+
}
|
|
939
|
+
try {
|
|
940
|
+
await this.guardian.endOwnerMutation(mutationId)
|
|
941
|
+
} catch (error) {
|
|
942
|
+
finalizationErrors.push(error instanceof Error ? error : new Error(String(error)))
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
if (operationError || finalizationErrors.length > 0) {
|
|
946
|
+
const errors = [...(operationError ? [operationError] : []), ...finalizationErrors]
|
|
947
|
+
|
|
948
|
+
if (errors.length === 1) throw errors[0]
|
|
949
|
+
throw new AggregateError(errors, `Owner mutation ${operation} failed: ${errors.map((error) => error.message).join("; ")}`)
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
return /** @type {Record<string, JsonValue>} */ (result)
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
/**
|
|
956
|
+
* Stops accepting only the listener endpoints that the prepared candidate must bind.
|
|
957
|
+
* Existing proxy connections remain owned by this daemon until they close.
|
|
958
|
+
* @param {{completionSocket?: net.Socket, control: boolean, proxy: boolean, replacementId: string}} options - Handoff request.
|
|
959
|
+
*/
|
|
960
|
+
async yieldOwnerListeners({completionSocket, control, proxy, replacementId}) {
|
|
961
|
+
if (!this.guardian) throw new Error("Owner listener handoff requires the durable process guardian")
|
|
962
|
+
if (this.listenerHandoff) throw new Error("Owner listeners are already yielded to a replacement candidate")
|
|
963
|
+
await this.guardian.validateOwnerReplacement(replacementId)
|
|
964
|
+
this.listenerHandoff = {control, proxy, replacementId}
|
|
965
|
+
this.listenerHandoffFailure = undefined
|
|
966
|
+
|
|
967
|
+
if (!completionSocket) throw new Error("Owner listener handoff requires its authenticated control session")
|
|
968
|
+
for (const release of this.releases.values()) release.pauseDrainForOwnerHandoff()
|
|
969
|
+
if (proxy) this.proxyClosePromise = this.closeServer(this.proxyServer)
|
|
970
|
+
if (control) {
|
|
971
|
+
this.controlClosePromise = this.closeServer(this.controlServer)
|
|
972
|
+
for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
|
|
973
|
+
await this.removeControlSocket()
|
|
974
|
+
}
|
|
975
|
+
for (const release of this.releases.values()) {
|
|
976
|
+
const publishConnections = () => {
|
|
977
|
+
if (completionSocket.destroyed) return
|
|
978
|
+
completionSocket.write(`${JSON.stringify({
|
|
979
|
+
connections: release.status().connections,
|
|
980
|
+
event: "owner-connection-state",
|
|
981
|
+
releaseId: release.releaseId
|
|
982
|
+
})}\n`)
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
publishConnections()
|
|
986
|
+
if (release.status().connectionCount > 0) release.once("drained", publishConnections)
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
const aborted = this.guardian.waitForEvent("replacement-aborted").then(async () => await this.resumeYieldedListeners())
|
|
990
|
+
const retired = this.guardian.waitForEvent("replacement-retired")
|
|
991
|
+
|
|
992
|
+
void Promise.race([aborted, retired]).catch((error) => {
|
|
993
|
+
if (this.ownerRetired) return
|
|
994
|
+
this.listenerHandoffFailure = error instanceof Error ? error : new Error(String(error))
|
|
995
|
+
this.logger("owner listener handoff recovery failed", {error: this.listenerHandoffFailure.message, replacementId})
|
|
996
|
+
})
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
/** Restores listener endpoints after a prepared candidate aborts before commit. */
|
|
1000
|
+
async resumeYieldedListeners() {
|
|
1001
|
+
const handoff = this.listenerHandoff
|
|
1002
|
+
|
|
1003
|
+
if (!handoff || this.ownerRetired) return
|
|
1004
|
+
for (const release of this.releases.values()) {
|
|
1005
|
+
release.resumeDrainAfterOwnerHandoff()
|
|
1006
|
+
if (release.state === "draining") void this.drainAndPrune(release, release.config)
|
|
1007
|
+
}
|
|
1008
|
+
if (handoff.proxy) {
|
|
1009
|
+
await this.startProxy()
|
|
1010
|
+
}
|
|
1011
|
+
if (handoff.control) {
|
|
1012
|
+
await this.startControlServer()
|
|
1013
|
+
}
|
|
1014
|
+
this.listenerHandoff = undefined
|
|
1015
|
+
this.listenerHandoffFailure = undefined
|
|
1016
|
+
this.logger("owner listeners resumed", {replacementId: handoff.replacementId})
|
|
1017
|
+
}
|
|
1018
|
+
|
|
376
1019
|
/**
|
|
377
1020
|
* Starts a new release, switches traffic, and drains the previous release.
|
|
378
1021
|
* @param {DeployArgs} args - Deploy args.
|
|
@@ -389,6 +1032,8 @@ export default class RollbridgeDaemon {
|
|
|
389
1032
|
const release = new ReleaseGroup({
|
|
390
1033
|
config: nextConfig,
|
|
391
1034
|
logger: this.logger,
|
|
1035
|
+
portReservations: this.portReservations,
|
|
1036
|
+
...(this.guardian ? {processFactory: (key, definition) => this.guardianProcess(key, definition)} : {}),
|
|
392
1037
|
releaseId: newReleaseId,
|
|
393
1038
|
releasePath,
|
|
394
1039
|
revision,
|
|
@@ -423,20 +1068,51 @@ export default class RollbridgeDaemon {
|
|
|
423
1068
|
this.logger("traffic switched", {previousReleaseId: previousRelease ? previousRelease.releaseId : null, releaseId: release.releaseId})
|
|
424
1069
|
|
|
425
1070
|
this.refreshServiceDefinitions(release)
|
|
426
|
-
|
|
1071
|
+
let retirementFailure
|
|
427
1072
|
|
|
428
1073
|
if (previousRelease) {
|
|
429
|
-
|
|
1074
|
+
try {
|
|
1075
|
+
const retirementConfig = previousRelease.preserveConfigOnRetirement ? previousRelease.config : nextConfig
|
|
1076
|
+
|
|
1077
|
+
await previousRelease.beginRetirement(retirementConfig)
|
|
1078
|
+
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
1079
|
+
} catch (error) {
|
|
1080
|
+
retirementFailure = previousRelease.retirementError ?? (error instanceof Error ? error.message : String(error))
|
|
1081
|
+
this.logger("release retirement quiescence failed", {error: retirementFailure, releaseId: previousRelease.releaseId})
|
|
1082
|
+
}
|
|
430
1083
|
}
|
|
431
1084
|
|
|
432
|
-
this.
|
|
1085
|
+
await this.replaceSingletons(release)
|
|
1086
|
+
|
|
1087
|
+
await this.persistState()
|
|
1088
|
+
await this.publishOwnerState()
|
|
433
1089
|
|
|
434
1090
|
return {
|
|
435
1091
|
activeReleaseId: release.releaseId,
|
|
436
|
-
previousReleaseId: previousRelease ? previousRelease.releaseId : null
|
|
1092
|
+
previousReleaseId: previousRelease ? previousRelease.releaseId : null,
|
|
1093
|
+
...(retirementFailure && previousRelease ? {retirement: {error: retirementFailure, releaseId: previousRelease.releaseId, status: "quiescence_failed"}} : {})
|
|
437
1094
|
}
|
|
438
1095
|
}
|
|
439
1096
|
|
|
1097
|
+
/**
|
|
1098
|
+
* Relinquishes only daemon authority/listeners after the guardian has committed a
|
|
1099
|
+
* prepared replacement. Guardian-owned processes and drains are never stopped.
|
|
1100
|
+
* @param {net.Socket | undefined} completionSocket - Commit response connection.
|
|
1101
|
+
*/
|
|
1102
|
+
async retireCommittedOwner(completionSocket) {
|
|
1103
|
+
this.ownerRetired = true
|
|
1104
|
+
if (this.persistTimer) clearInterval(this.persistTimer)
|
|
1105
|
+
this.persistTimer = undefined
|
|
1106
|
+
this.persistenceEnabled = false
|
|
1107
|
+
if (this.pendingWrite) await this.pendingWrite
|
|
1108
|
+
this.stateCleanupEnabled = false
|
|
1109
|
+
this.controlClosePromise = this.closeServer(this.controlServer)
|
|
1110
|
+
for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
|
|
1111
|
+
await this.removeControlSocket()
|
|
1112
|
+
void this.closeServer(this.proxyServer)
|
|
1113
|
+
this.logger("owner authority transferred", {activeReleaseId: this.activeRelease?.releaseId ?? null})
|
|
1114
|
+
}
|
|
1115
|
+
|
|
440
1116
|
/**
|
|
441
1117
|
* Rejects config changes that require rebinding daemon-owned resources or changing process topology.
|
|
442
1118
|
* @param {import("./config.js").RollbridgeConfig} nextConfig - Freshly loaded config.
|
|
@@ -449,6 +1125,7 @@ export default class RollbridgeDaemon {
|
|
|
449
1125
|
if (nextConfig.application !== this.config.application) restartRequired.push("application")
|
|
450
1126
|
if (!isDeepStrictEqual(nextConfig.control, this.config.control)) restartRequired.push("control")
|
|
451
1127
|
if (nextConfig.statePath !== this.config.statePath) restartRequired.push("statePath")
|
|
1128
|
+
if (!isDeepStrictEqual(nextConfig.ownerRecovery, this.config.ownerRecovery)) restartRequired.push("ownerRecovery")
|
|
452
1129
|
|
|
453
1130
|
if (nextConfig.proxy.host !== this.config.proxy.host) restartRequired.push("proxy.host")
|
|
454
1131
|
if (nextConfig.proxy.port !== this.config.proxy.port) restartRequired.push("proxy.port")
|
|
@@ -536,7 +1213,7 @@ export default class RollbridgeDaemon {
|
|
|
536
1213
|
if (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff") continue
|
|
537
1214
|
if (this.services.has(processConfig.id)) continue
|
|
538
1215
|
|
|
539
|
-
const service = release.buildProcess(processConfig, {shouldRestart: () => !this.stopping})
|
|
1216
|
+
const service = release.buildProcess(processConfig, {guardianKey: `service:${processConfig.id}`, shouldRestart: () => !this.stopping})
|
|
540
1217
|
|
|
541
1218
|
this.services.set(processConfig.id, service)
|
|
542
1219
|
|
|
@@ -546,6 +1223,7 @@ export default class RollbridgeDaemon {
|
|
|
546
1223
|
|
|
547
1224
|
try {
|
|
548
1225
|
await service.start("deploy")
|
|
1226
|
+
release.transferPortReservation(processConfig.id)
|
|
549
1227
|
startedServices.push(processConfig.id)
|
|
550
1228
|
} catch (error) {
|
|
551
1229
|
this.services.delete(processConfig.id)
|
|
@@ -570,6 +1248,9 @@ export default class RollbridgeDaemon {
|
|
|
570
1248
|
|
|
571
1249
|
await service.stop()
|
|
572
1250
|
this.services.delete(serviceId)
|
|
1251
|
+
const port = this.servicePorts[serviceId]
|
|
1252
|
+
|
|
1253
|
+
if (port !== undefined) this.portReservations.delete(port)
|
|
573
1254
|
delete this.servicePorts[serviceId]
|
|
574
1255
|
}
|
|
575
1256
|
}
|
|
@@ -610,7 +1291,7 @@ export default class RollbridgeDaemon {
|
|
|
610
1291
|
if (this.stopping) throw new Error("Rollbridge is shutting down")
|
|
611
1292
|
}
|
|
612
1293
|
|
|
613
|
-
const singleton = release.buildProcess(processConfig)
|
|
1294
|
+
const singleton = release.buildProcess(processConfig, {guardianKey: `singleton:${release.releaseId}:${processConfig.id}`})
|
|
614
1295
|
|
|
615
1296
|
this.singletons.set(processConfig.id, singleton)
|
|
616
1297
|
await singleton.start("deploy")
|
|
@@ -726,8 +1407,10 @@ export default class RollbridgeDaemon {
|
|
|
726
1407
|
} catch (error) {
|
|
727
1408
|
this.logger("release drain failed", {error: error instanceof Error ? error.message : String(error), releaseId: release.releaseId})
|
|
728
1409
|
} finally {
|
|
729
|
-
|
|
730
|
-
|
|
1410
|
+
if (!release.isDrainPausedForOwnerHandoff()) {
|
|
1411
|
+
this.pruneStoppedReleases()
|
|
1412
|
+
this.persistState()
|
|
1413
|
+
}
|
|
731
1414
|
}
|
|
732
1415
|
}
|
|
733
1416
|
|
|
@@ -753,25 +1436,40 @@ export default class RollbridgeDaemon {
|
|
|
753
1436
|
|
|
754
1437
|
/**
|
|
755
1438
|
* Persists a state snapshot (status plus recent events) to statePath, atomically and
|
|
756
|
-
* fire-and-forget. A failed write is logged
|
|
757
|
-
* @
|
|
1439
|
+
* fire-and-forget unless the caller awaits the returned write. A failed write is logged.
|
|
1440
|
+
* @param {{allowStopping?: boolean, throwOnError?: boolean}} [options] - Write behavior.
|
|
1441
|
+
* @returns {Promise<void> | undefined} The queued write, or undefined when persistence is disabled.
|
|
758
1442
|
*/
|
|
759
|
-
persistState() {
|
|
760
|
-
if (!this.statePath || !this.persistenceEnabled || this.stopping) return
|
|
1443
|
+
persistState({allowStopping = false, throwOnError = false} = {}) {
|
|
1444
|
+
if (!this.statePath || !this.persistenceEnabled || (this.stopping && !allowStopping)) return
|
|
761
1445
|
|
|
762
1446
|
const statePath = this.statePath
|
|
763
1447
|
const status = /** @type {Record<string, JsonValue>} */ (secretSafeStateValue(this.status()))
|
|
764
1448
|
const events = secretSafeStateValue(this.eventLog.recent())
|
|
765
|
-
const snapshot = {
|
|
1449
|
+
const snapshot = {
|
|
1450
|
+
...status,
|
|
1451
|
+
events,
|
|
1452
|
+
persistedAt: new Date().toISOString(),
|
|
1453
|
+
...(this.guardianIdentity ? {recovery: {
|
|
1454
|
+
configDigest: this.ownerRecoveryConfigDigest(),
|
|
1455
|
+
format: 1,
|
|
1456
|
+
guardian: this.guardianIdentity,
|
|
1457
|
+
reconnectGraceMs: this.config.ownerRecovery?.reconnectGraceMs
|
|
1458
|
+
}} : {})
|
|
1459
|
+
}
|
|
766
1460
|
|
|
767
1461
|
// Serialize writes (and track the tail) so shutdown can wait for an in-flight write before
|
|
768
1462
|
// clearing the file — otherwise a write started before shutdown could recreate it afterward.
|
|
769
1463
|
this.pendingWrite = Promise.resolve(this.pendingWrite)
|
|
770
1464
|
.catch(() => {})
|
|
771
1465
|
.then(() => writeState(statePath, snapshot))
|
|
1466
|
+
.then(() => this.publishOwnerState())
|
|
772
1467
|
.catch((error) => {
|
|
773
1468
|
this.logger("state persist failed", {error: error instanceof Error ? error.message : String(error)})
|
|
1469
|
+
if (throwOnError) throw error
|
|
774
1470
|
})
|
|
1471
|
+
|
|
1472
|
+
return this.pendingWrite
|
|
775
1473
|
}
|
|
776
1474
|
|
|
777
1475
|
/**
|
|
@@ -835,25 +1533,35 @@ export default class RollbridgeDaemon {
|
|
|
835
1533
|
clearInterval(this.persistTimer)
|
|
836
1534
|
this.persistTimer = undefined
|
|
837
1535
|
}
|
|
838
|
-
this.persistenceEnabled = false
|
|
839
1536
|
if (this.pendingWrite) await this.pendingWrite
|
|
840
1537
|
this.stateCleanupEnabled = false
|
|
841
1538
|
this.controlClosePromise = this.closeServer(this.controlServer)
|
|
842
1539
|
for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
|
|
1540
|
+
if (this.activeRelease) {
|
|
1541
|
+
await this.activeRelease.beginRetirement(this.activeRelease.config)
|
|
1542
|
+
this.activeRelease = undefined
|
|
1543
|
+
}
|
|
843
1544
|
await Promise.all([
|
|
844
1545
|
...[...this.services.values()].map((processInstance) => processInstance.quiesce()),
|
|
845
1546
|
...[...this.singletons.values()].map((processInstance) => processInstance.quiesce()),
|
|
846
1547
|
...[...this.startingReleases].map((release) => release.quiesce()),
|
|
847
1548
|
...[...this.releases.values()].map((release) => release.quiesce())
|
|
848
1549
|
])
|
|
1550
|
+
await this.persistState({allowStopping: true, throwOnError: true})
|
|
1551
|
+
this.persistenceEnabled = false
|
|
849
1552
|
await this.removeControlSocket()
|
|
850
1553
|
void this.closeServer(this.proxyServer)
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
1554
|
+
if (this.guardian) {
|
|
1555
|
+
await this.guardian.retireOwner()
|
|
1556
|
+
this.guardian.disconnect()
|
|
1557
|
+
} else {
|
|
1558
|
+
void Promise.allSettled([
|
|
1559
|
+
...[...this.services.values()].map((processInstance) => processInstance.stop()),
|
|
1560
|
+
...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
|
|
1561
|
+
...[...this.startingReleases].map((release) => release.stop()),
|
|
1562
|
+
...[...this.releases.values()].map((release) => release.stop())
|
|
1563
|
+
])
|
|
1564
|
+
}
|
|
857
1565
|
this.logger("external owner retired", {attestation, status: "draining"})
|
|
858
1566
|
}
|
|
859
1567
|
|
|
@@ -865,6 +1573,9 @@ export default class RollbridgeDaemon {
|
|
|
865
1573
|
this.stopping = true
|
|
866
1574
|
const cleanupErrors = /** @type {Error[]} */ ([])
|
|
867
1575
|
|
|
1576
|
+
this.incumbentListenerControl?.close()
|
|
1577
|
+
this.incumbentListenerControl = undefined
|
|
1578
|
+
|
|
868
1579
|
// server.close() stops new connections synchronously. Unlink immediately afterward so a
|
|
869
1580
|
// replacement can bind as soon as cleanup completes; existing connections remain usable for
|
|
870
1581
|
// the shutdown completion/error response.
|
|
@@ -889,6 +1600,8 @@ export default class RollbridgeDaemon {
|
|
|
889
1600
|
])
|
|
890
1601
|
const serviceStopResults = await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
|
|
891
1602
|
const stopResults = [...dependentStopResults, ...serviceStopResults]
|
|
1603
|
+
const guardian = this.guardian
|
|
1604
|
+
if (guardian) await captureShutdownError(cleanupErrors, "process guardian shutdown", () => guardian.shutdown())
|
|
892
1605
|
await captureShutdownError(cleanupErrors, "proxy server close", () => this.closeServer(this.proxyServer))
|
|
893
1606
|
|
|
894
1607
|
// Wait for any in-flight write first so it can't recreate or overwrite the final state (no
|
|
@@ -919,8 +1632,9 @@ export default class RollbridgeDaemon {
|
|
|
919
1632
|
async removeControlSocket() {
|
|
920
1633
|
if (!this.controlSocketOwned) return
|
|
921
1634
|
|
|
922
|
-
await fs.rm(this.config.control.path, {force: true})
|
|
1635
|
+
await fs.rm(this.boundControlPath || this.config.control.path, {force: true})
|
|
923
1636
|
this.controlSocketOwned = false
|
|
1637
|
+
this.boundControlPath = undefined
|
|
924
1638
|
}
|
|
925
1639
|
|
|
926
1640
|
/**
|
|
@@ -950,13 +1664,19 @@ export default class RollbridgeDaemon {
|
|
|
950
1664
|
application: this.config.application,
|
|
951
1665
|
bootstrap: this.bootstrap ? {...this.bootstrap} : undefined,
|
|
952
1666
|
control: {...this.config.control},
|
|
1667
|
+
daemonPid: process.pid,
|
|
953
1668
|
daemonRuntime: this.runtime ? {...this.runtime} : undefined,
|
|
1669
|
+
ownerRecovery: this.guardian ? {configDigest: this.ownerRecoveryConfigDigest()} : undefined,
|
|
1670
|
+
ownerTransition: this.ownerTransition ? {...this.ownerTransition} : undefined,
|
|
954
1671
|
orphans: [...this.orphans],
|
|
955
1672
|
proxy: {
|
|
956
1673
|
host: this.config.proxy.host,
|
|
957
1674
|
port: this.proxyPort ?? this.config.proxy.port,
|
|
958
1675
|
upstreamHost: this.config.proxy.upstreamHost
|
|
959
1676
|
},
|
|
1677
|
+
releaseReferences: [...this.releases.values()]
|
|
1678
|
+
.filter((release) => release.state === "active" || release.state === "draining")
|
|
1679
|
+
.map((release) => ({releaseId: release.releaseId, releasePath: release.releasePath})),
|
|
960
1680
|
releases: [...this.releases.values()].map((release) => release.status()),
|
|
961
1681
|
services: [...this.services.entries()].map(([id, processInstance]) => ({
|
|
962
1682
|
id,
|
|
@@ -970,6 +1690,162 @@ export default class RollbridgeDaemon {
|
|
|
970
1690
|
}
|
|
971
1691
|
}
|
|
972
1692
|
|
|
1693
|
+
/**
|
|
1694
|
+
* @param {import("./config.js").RollbridgeConfig} config - Normalized config.
|
|
1695
|
+
* @returns {string} Stable config authority digest.
|
|
1696
|
+
*/
|
|
1697
|
+
export function ownerConfigDigest(config) {
|
|
1698
|
+
return crypto.createHash("sha256").update(JSON.stringify(config)).digest("hex")
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
/**
|
|
1702
|
+
* Accepts only the two exact authenticated diagnostics emitted by pre-split guardians.
|
|
1703
|
+
* @param {string} diagnostic - Guardian response diagnostic.
|
|
1704
|
+
* @returns {boolean} Whether the response can proceed to full legacy process attestation.
|
|
1705
|
+
*/
|
|
1706
|
+
export function isLegacyGuardianPrepareDiagnostic(diagnostic) {
|
|
1707
|
+
return diagnostic === "Guardian prepare-owner-replacement requires a process key" ||
|
|
1708
|
+
diagnostic === "Unknown guardian command: prepare-owner-replacement"
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1711
|
+
/**
|
|
1712
|
+
* @param {OwnerRecoverySnapshot} snapshot - Durable pre-split snapshot.
|
|
1713
|
+
* @returns {string[]} Exact guardian registration keys present in the snapshot.
|
|
1714
|
+
*/
|
|
1715
|
+
function legacyGuardianKeys(snapshot) {
|
|
1716
|
+
const keys = []
|
|
1717
|
+
|
|
1718
|
+
for (const release of snapshot.releases) {
|
|
1719
|
+
for (const processStatus of release.processes) keys.push(`release:${release.releaseId}:${processStatus.id}`)
|
|
1720
|
+
}
|
|
1721
|
+
for (const service of snapshot.services) keys.push(`service:${service.id}`)
|
|
1722
|
+
if (snapshot.activeReleaseId) {
|
|
1723
|
+
for (const singleton of snapshot.singletons) keys.push(`singleton:${snapshot.activeReleaseId}:${singleton.id}`)
|
|
1724
|
+
}
|
|
1725
|
+
return keys
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
/**
|
|
1729
|
+
* Verifies the exact authenticated guardian process and socket without scanning other PIDs.
|
|
1730
|
+
* @param {number} pid - Persisted guardian PID.
|
|
1731
|
+
* @param {string} socketPath - Persisted guardian socket.
|
|
1732
|
+
*/
|
|
1733
|
+
async function verifyLegacyGuardianProcess(pid, socketPath) {
|
|
1734
|
+
const args = await processArguments(pid, "legacy guardian")
|
|
1735
|
+
const script = args.find((argument) => argument.endsWith("process-guardian.js"))
|
|
1736
|
+
|
|
1737
|
+
if (!script || !args.includes(socketPath)) throw new Error(`Persisted guardian PID ${pid} does not match the pre-split guardian command and socket`)
|
|
1738
|
+
await verifyProcessUser(pid, "legacy guardian")
|
|
1739
|
+
await verifyUnixSocketOwner(pid, socketPath, "legacy guardian")
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
/**
|
|
1743
|
+
* Verifies the exact incumbent daemon process and control socket without scanning other PIDs.
|
|
1744
|
+
* @param {number} pid - Incumbent PID from the daemon PID file.
|
|
1745
|
+
* @param {string | undefined} configPath - Exact daemon config path.
|
|
1746
|
+
* @param {string} socketPath - Persisted control socket.
|
|
1747
|
+
* @returns {Promise<string>} Linux process start-time identity.
|
|
1748
|
+
*/
|
|
1749
|
+
async function verifyLegacyDaemonProcess(pid, configPath, socketPath) {
|
|
1750
|
+
if (!configPath) throw new Error("Legacy disruptive replacement requires the daemon's exact config path")
|
|
1751
|
+
const args = await processArguments(pid, "legacy daemon")
|
|
1752
|
+
const daemonIndex = args.indexOf("daemon")
|
|
1753
|
+
const configIndex = args.indexOf("--config")
|
|
1754
|
+
|
|
1755
|
+
if (daemonIndex < 0 || configIndex < 0 || args[configIndex + 1] !== configPath) {
|
|
1756
|
+
throw new Error(`Daemon PID ${pid} does not match the exact pre-split daemon config command`)
|
|
1757
|
+
}
|
|
1758
|
+
await verifyProcessUser(pid, "legacy daemon")
|
|
1759
|
+
await verifyUnixSocketOwner(pid, socketPath, "legacy daemon")
|
|
1760
|
+
const stat = await fs.readFile(`/proc/${pid}/stat`, "utf8")
|
|
1761
|
+
const closingParenthesis = stat.lastIndexOf(")")
|
|
1762
|
+
const fields = stat.slice(closingParenthesis + 2).split(" ")
|
|
1763
|
+
const startTime = fields[19]
|
|
1764
|
+
|
|
1765
|
+
if (!startTime) throw new Error(`Could not attest start time for legacy daemon PID ${pid}`)
|
|
1766
|
+
return startTime
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
/**
|
|
1770
|
+
* @param {number} pid - Exact process PID.
|
|
1771
|
+
* @param {string} label - Diagnostic label.
|
|
1772
|
+
* @returns {Promise<string[]>} NUL-delimited argv.
|
|
1773
|
+
*/
|
|
1774
|
+
async function processArguments(pid, label) {
|
|
1775
|
+
try {
|
|
1776
|
+
return (await fs.readFile(`/proc/${pid}/cmdline`, "utf8")).split("\0").filter(Boolean)
|
|
1777
|
+
} catch (error) {
|
|
1778
|
+
throw new Error(`Could not verify exact ${label} PID ${pid}`, {cause: error})
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
/**
|
|
1783
|
+
* @param {number} pid - Exact process PID.
|
|
1784
|
+
* @param {string} label - Diagnostic label.
|
|
1785
|
+
*/
|
|
1786
|
+
async function verifyProcessUser(pid, label) {
|
|
1787
|
+
if (typeof process.getuid !== "function") throw new Error(`Cannot verify ${label} ownership on this platform`)
|
|
1788
|
+
const stats = await fs.stat(`/proc/${pid}`)
|
|
1789
|
+
|
|
1790
|
+
if (stats.uid !== process.getuid()) throw new Error(`Refusing ${label} PID ${pid} owned by another user`)
|
|
1791
|
+
}
|
|
1792
|
+
|
|
1793
|
+
/**
|
|
1794
|
+
* @param {number} pid - Exact process PID.
|
|
1795
|
+
* @param {string} socketPath - Exact Unix socket pathname.
|
|
1796
|
+
* @param {string} label - Diagnostic label.
|
|
1797
|
+
*/
|
|
1798
|
+
async function verifyUnixSocketOwner(pid, socketPath, label) {
|
|
1799
|
+
const rows = (await fs.readFile("/proc/net/unix", "utf8")).split("\n").map((line) => line.trim().split(/\s+/))
|
|
1800
|
+
const row = rows.find((columns) => columns[7] === socketPath)
|
|
1801
|
+
|
|
1802
|
+
if (!row?.[6]) throw new Error(`Could not find exact ${label} socket ${socketPath} in the kernel socket table`)
|
|
1803
|
+
const expected = `socket:[${row[6]}]`
|
|
1804
|
+
const descriptors = await fs.readdir(`/proc/${pid}/fd`)
|
|
1805
|
+
let owned = false
|
|
1806
|
+
|
|
1807
|
+
for (const descriptor of descriptors) {
|
|
1808
|
+
try {
|
|
1809
|
+
if (await fs.readlink(`/proc/${pid}/fd/${descriptor}`) === expected) {
|
|
1810
|
+
owned = true
|
|
1811
|
+
break
|
|
1812
|
+
}
|
|
1813
|
+
} catch (error) {
|
|
1814
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
if (!owned) throw new Error(`Exact ${label} PID ${pid} does not own socket ${socketPath}`)
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
/**
|
|
1821
|
+
* @param {Record<string, JsonValue>} status - Authenticated incumbent status response.
|
|
1822
|
+
* @param {OwnerRecoverySnapshot} persisted - Durable expected identity.
|
|
1823
|
+
*/
|
|
1824
|
+
function assertLegacyIncumbentStatus(status, persisted) {
|
|
1825
|
+
const control = status.control
|
|
1826
|
+
const runtime = status.daemonRuntime
|
|
1827
|
+
const ownerRecovery = status.ownerRecovery
|
|
1828
|
+
|
|
1829
|
+
if (status.application !== persisted.application || !control || typeof control !== "object" || Array.isArray(control) || control.path !== persisted.control.path ||
|
|
1830
|
+
!runtime || typeof runtime !== "object" || Array.isArray(runtime) || runtime.digest !== persisted.daemonRuntime?.digest ||
|
|
1831
|
+
!ownerRecovery || typeof ownerRecovery !== "object" || Array.isArray(ownerRecovery) || ownerRecovery.configDigest !== persisted.recovery.configDigest) {
|
|
1832
|
+
throw new Error("Responsive incumbent does not match the exact persisted pre-split daemon authority")
|
|
1833
|
+
}
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
/**
|
|
1837
|
+
* @param {string} candidatePath - Path that must not preexist.
|
|
1838
|
+
* @param {string} label - Diagnostic label.
|
|
1839
|
+
*/
|
|
1840
|
+
async function assertPathAbsent(candidatePath, label) {
|
|
1841
|
+
try {
|
|
1842
|
+
await fs.lstat(candidatePath)
|
|
1843
|
+
throw new Error(`${label} ${candidatePath} already exists; refusing legacy upgrade`)
|
|
1844
|
+
} catch (error) {
|
|
1845
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1848
|
+
|
|
973
1849
|
/**
|
|
974
1850
|
* @param {JsonValue} value - Value.
|
|
975
1851
|
* @returns {string | undefined} String value.
|
|
@@ -989,7 +1865,7 @@ function isShutdownControlLine(line) {
|
|
|
989
1865
|
try {
|
|
990
1866
|
const command = JSON.parse(line)
|
|
991
1867
|
|
|
992
|
-
return Boolean(command && typeof command === "object" && ["retire-owner", "shutdown"].includes(command.command))
|
|
1868
|
+
return Boolean(command && typeof command === "object" && ["commit-owner-replacement", "retire-owner", "shutdown"].includes(command.command))
|
|
993
1869
|
} catch {
|
|
994
1870
|
return false
|
|
995
1871
|
}
|
|
@@ -1049,6 +1925,16 @@ function requiredString(value, key) {
|
|
|
1049
1925
|
return value
|
|
1050
1926
|
}
|
|
1051
1927
|
|
|
1928
|
+
/**
|
|
1929
|
+
* @param {JsonValue} value - Value.
|
|
1930
|
+
* @param {string} key - Key.
|
|
1931
|
+
* @returns {number} Non-negative integer.
|
|
1932
|
+
*/
|
|
1933
|
+
function requiredNonNegativeInteger(value, key) {
|
|
1934
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) throw new Error(`${key} must be a non-negative integer`)
|
|
1935
|
+
return value
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1052
1938
|
/**
|
|
1053
1939
|
* @typedef {{releaseId: string, state: string, stoppedAt: string | undefined}} PrunableRelease
|
|
1054
1940
|
*/
|