rollbridge 0.1.31 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +11 -6
- package/README.md +28 -12
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +9 -0
- package/docs/cli.md +10 -7
- package/docs/config.md +41 -13
- package/docs/tensorbuzz-runbook.md +4 -2
- package/docs/velocious.md +19 -7
- package/docs/workers.md +18 -9
- package/examples/tensorbuzz.com.js +9 -5
- package/package.json +1 -1
- package/src/config.js +32 -2
- package/src/daemon.js +264 -20
- package/src/guardian-client.js +41 -3
- package/src/managed-process.js +85 -13
- package/src/process-guardian.js +13 -1
- package/src/release-group.js +53 -14
- package/test/config-validation.test.js +44 -0
- package/test/guardian-client.test.js +39 -0
- package/test/managed-process.test.js +33 -0
- package/test/owner-recovery.test.js +396 -6
- package/test/owner-replacement.test.js +107 -2
- package/test/rollbridge.test.js +333 -6
package/src/daemon.js
CHANGED
|
@@ -23,9 +23,12 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
23
23
|
* @typedef {{attestation?: string, releaseId: string, releasePath: string, revision: string}} BootstrapIdentity
|
|
24
24
|
* @typedef {{id: string, process: import("./managed-process.js").ManagedProcessStatus}} ProcessStatus
|
|
25
25
|
* @typedef {{disruptive: true, mode: "legacy-first-upgrade", reason: string}} OwnerTransition
|
|
26
|
-
* @typedef {
|
|
26
|
+
* @typedef {"candidate_ready" | "retiring_previous" | "previous_retired" | "activating_candidate" | "committed_pending" | "committed"} GenerationTransitionPhase
|
|
27
|
+
* @typedef {{candidateReleaseId: string, candidateReleasePath: string, candidateRevision: string, configDigest: string, error?: string, phase: GenerationTransitionPhase, previousReleaseId: string | null, startedAt: string, updatedAt: string}} GenerationTransition
|
|
28
|
+
* @typedef {{activeReleaseId: string | null, application: string, bootstrap: BootstrapIdentity | undefined, control: import("./config.js").ControlConfig, daemonPid: number, daemonRuntime: import("./daemon-runtime.js").DaemonRuntimeIdentity | undefined, generationTransition?: GenerationTransition, ownerRecovery: {configDigest: string} | undefined, ownerTransition?: OwnerTransition, orphans: {id: string, pid: number, releaseId: string | null}[], proxy: {host: string, port: number | undefined, upstreamHost: string}, releaseReferences: {releaseId: string, releasePath: string}[], releases: import("./release-group.js").ReleaseStatus[], services: ProcessStatus[], singletons: ProcessStatus[]}} DaemonStatus
|
|
27
29
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
28
|
-
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata}} OwnerRecoverySnapshot
|
|
30
|
+
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
31
|
+
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
29
32
|
* @typedef {{boundaryCrossed: boolean, incumbentControl: Awaited<ReturnType<typeof openControlSession>>, incumbentStartTime: string, prepared: {ownerState: JsonValue, replacementId: string}, recoverySnapshot: OwnerRecoverySnapshot}} LegacyOwnerBridge
|
|
30
33
|
*/
|
|
31
34
|
|
|
@@ -62,7 +65,9 @@ export default class RollbridgeDaemon {
|
|
|
62
65
|
this.servicePorts = /** @type {Record<string, number>} */ ({})
|
|
63
66
|
this.portReservations = /** @type {Set<number>} */ (new Set())
|
|
64
67
|
this.singletons = /** @type {Map<string, import("./managed-process.js").default>} */ (new Map())
|
|
68
|
+
this.singletonReleaseIds = /** @type {Map<string, string>} */ (new Map())
|
|
65
69
|
this.activeRelease = /** @type {ReleaseGroup | undefined} */ (undefined)
|
|
70
|
+
this.generationTransition = /** @type {GenerationTransition | undefined} */ (undefined)
|
|
66
71
|
this.proxy = httpProxy.createProxyServer({ws: true, xfwd: true})
|
|
67
72
|
this.proxyServer = /** @type {http.Server | undefined} */ (undefined)
|
|
68
73
|
this.controlServer = /** @type {net.Server | undefined} */ (undefined)
|
|
@@ -137,10 +142,42 @@ export default class RollbridgeDaemon {
|
|
|
137
142
|
this.watchOwnerReplacementEvents()
|
|
138
143
|
|
|
139
144
|
if (snapshot) {
|
|
140
|
-
|
|
145
|
+
if (!Array.isArray(snapshot.releases) || (snapshot.activeReleaseId !== null && typeof snapshot.activeReleaseId !== "string")) {
|
|
146
|
+
throw new Error("Owner recovery state is partial or corrupt; active release metadata is required.")
|
|
147
|
+
}
|
|
148
|
+
let recoveryConfig = this.config
|
|
149
|
+
let releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({})
|
|
150
|
+
let singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({})
|
|
151
|
+
let recoverySnapshot = snapshot
|
|
152
|
+
|
|
153
|
+
if ((recovery?.format ?? 0) >= 2) {
|
|
154
|
+
const ownerState = /** @type {PrivateOwnerState} */ (await this.guardian.ownerState())
|
|
155
|
+
|
|
156
|
+
if (!ownerState?.config || !ownerState.releaseConfigs || !ownerState.snapshot) {
|
|
157
|
+
throw new Error("Durable guardian state is missing exact release definitions; refusing partial owner recovery.")
|
|
158
|
+
}
|
|
159
|
+
recoveryConfig = ownerState.config
|
|
160
|
+
releaseConfigs = ownerState.releaseConfigs
|
|
161
|
+
singletonReleaseIds = ownerState.singletonReleaseIds || singletonReleaseIds
|
|
162
|
+
const guardianTransitionAt = Date.parse(ownerState.snapshot.generationTransition?.updatedAt || "")
|
|
163
|
+
const persistedTransitionAt = Date.parse(snapshot.generationTransition?.updatedAt || "")
|
|
164
|
+
|
|
165
|
+
if (Number.isFinite(guardianTransitionAt) && (!Number.isFinite(persistedTransitionAt) || guardianTransitionAt > persistedTransitionAt)) {
|
|
166
|
+
recoverySnapshot = ownerState.snapshot
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
await this.restoreOwnerState(recoverySnapshot, {config: recoveryConfig, releaseConfigs, resumeDrains: false, singletonReleaseIds})
|
|
170
|
+
this.persistenceEnabled = true
|
|
141
171
|
await this.guardian.reconcileInventory()
|
|
172
|
+
if (this.generationTransition && this.generationTransition.phase !== "committed" && !this.generationTransition.error) {
|
|
173
|
+
try {
|
|
174
|
+
await this.resumeGenerationTransition()
|
|
175
|
+
} catch (error) {
|
|
176
|
+
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: this.generationTransition.candidateReleaseId})
|
|
177
|
+
}
|
|
178
|
+
}
|
|
142
179
|
for (const release of this.releases.values()) {
|
|
143
|
-
if (release.state === "draining") void this.drainAndPrune(release, release.config)
|
|
180
|
+
if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
144
181
|
}
|
|
145
182
|
}
|
|
146
183
|
else {
|
|
@@ -166,7 +203,7 @@ export default class RollbridgeDaemon {
|
|
|
166
203
|
if (this.listenerHandoff || this.ownerRetired) return
|
|
167
204
|
for (const release of this.releases.values()) {
|
|
168
205
|
release.resumeDrainAfterOwnerHandoff()
|
|
169
|
-
if (release.state === "draining") void this.drainAndPrune(release, release.config)
|
|
206
|
+
if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
170
207
|
}
|
|
171
208
|
})
|
|
172
209
|
}
|
|
@@ -182,17 +219,24 @@ export default class RollbridgeDaemon {
|
|
|
182
219
|
* @param {import("./config.js").RollbridgeConfig} [options.config] - Owner config for daemon-wide processes.
|
|
183
220
|
* @param {Record<string, import("./config.js").RollbridgeConfig>} [options.releaseConfigs] - Exact generation configs.
|
|
184
221
|
* @param {boolean} [options.resumeDrains] - Whether to resume draining generations immediately.
|
|
222
|
+
* @param {Record<string, string>} [options.singletonReleaseIds] - Exact release owner for each singleton registration.
|
|
223
|
+
* @param {boolean} [options.synchronizeLifecycleRoles] - Whether the caller owns guardian role mutation authority.
|
|
185
224
|
*/
|
|
186
|
-
async restoreOwnerState(snapshot, {config = this.config, releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({}), resumeDrains = true} = {}) {
|
|
225
|
+
async restoreOwnerState(snapshot, {config = this.config, releaseConfigs = /** @type {Record<string, import("./config.js").RollbridgeConfig>} */ ({}), resumeDrains = true, singletonReleaseIds = snapshot.singletonReleaseIds || /** @type {Record<string, string>} */ ({}), synchronizeLifecycleRoles = true} = {}) {
|
|
187
226
|
if (!Array.isArray(snapshot.releases) || (snapshot.activeReleaseId !== null && typeof snapshot.activeReleaseId !== "string")) {
|
|
188
227
|
throw new Error("Owner recovery state is partial or corrupt; active release metadata is required.")
|
|
189
228
|
}
|
|
229
|
+
this.generationTransition = snapshot.generationTransition ? {...snapshot.generationTransition} : undefined
|
|
190
230
|
if (snapshot.activeReleaseId === null && snapshot.releases.length === 0) return
|
|
191
231
|
this.bootstrap = snapshot.bootstrap ? {...snapshot.bootstrap} : undefined
|
|
192
232
|
this.ownerTransition = snapshot.ownerTransition ? {...snapshot.ownerTransition} : undefined
|
|
233
|
+
const singletonOwnerReleaseIds = new Set(Object.values(singletonReleaseIds))
|
|
193
234
|
|
|
194
235
|
for (const releaseStatus of snapshot.releases) {
|
|
195
|
-
|
|
236
|
+
const transitionCandidate = this.generationTransition?.candidateReleaseId === releaseStatus.releaseId && this.generationTransition.phase !== "committed"
|
|
237
|
+
const singletonOwner = singletonOwnerReleaseIds.has(releaseStatus.releaseId)
|
|
238
|
+
|
|
239
|
+
if (releaseStatus.state !== "active" && releaseStatus.state !== "draining" && !transitionCandidate && !singletonOwner) continue
|
|
196
240
|
const release = new ReleaseGroup({
|
|
197
241
|
config: releaseConfigs[releaseStatus.releaseId] || config,
|
|
198
242
|
logger: this.logger,
|
|
@@ -205,7 +249,7 @@ export default class RollbridgeDaemon {
|
|
|
205
249
|
shouldStart: () => !this.stopping
|
|
206
250
|
})
|
|
207
251
|
|
|
208
|
-
await release.restore(releaseStatus)
|
|
252
|
+
await release.restore(releaseStatus, {synchronizeLifecycleRole: synchronizeLifecycleRoles})
|
|
209
253
|
this.releases.set(release.releaseId, release)
|
|
210
254
|
if (release.releaseId === snapshot.activeReleaseId) this.activeRelease = release
|
|
211
255
|
}
|
|
@@ -231,16 +275,21 @@ export default class RollbridgeDaemon {
|
|
|
231
275
|
}
|
|
232
276
|
}
|
|
233
277
|
for (const singletonStatus of snapshot.singletons) {
|
|
234
|
-
const
|
|
278
|
+
const singletonReleaseId = singletonReleaseIds[singletonStatus.id] || definitionRelease.releaseId
|
|
279
|
+
const singletonRelease = this.releases.get(singletonReleaseId)
|
|
280
|
+
|
|
281
|
+
if (!singletonRelease) throw new Error(`Owner recovery state contains singleton ${singletonStatus.id} for unknown release ${singletonReleaseId}.`)
|
|
282
|
+
const processConfig = singletonRelease.config.processes.find((candidate) => candidate.id === singletonStatus.id && candidate.policy === "singleton")
|
|
235
283
|
|
|
236
|
-
if (!processConfig) throw new Error(`Owner recovery state contains unknown singleton ${singletonStatus.id}.`)
|
|
237
|
-
const singleton =
|
|
284
|
+
if (!processConfig) throw new Error(`Owner recovery state contains unknown singleton ${singletonStatus.id} for release ${singletonReleaseId}.`)
|
|
285
|
+
const singleton = singletonRelease.buildProcess(processConfig, {guardianKey: `singleton:${singletonReleaseId}:${singletonStatus.id}`})
|
|
238
286
|
|
|
239
287
|
await this.recoverGuardianProcess(singleton)
|
|
240
288
|
this.singletons.set(singletonStatus.id, singleton)
|
|
289
|
+
this.singletonReleaseIds.set(singletonStatus.id, singletonReleaseId)
|
|
241
290
|
}
|
|
242
291
|
for (const release of this.releases.values()) {
|
|
243
|
-
if (resumeDrains && release.state === "draining") void this.drainAndPrune(release, release.config)
|
|
292
|
+
if (resumeDrains && release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
244
293
|
}
|
|
245
294
|
this.logger("owner state recovered", {activeReleaseId: this.activeRelease?.releaseId ?? null, releases: this.releases.size})
|
|
246
295
|
}
|
|
@@ -252,6 +301,7 @@ export default class RollbridgeDaemon {
|
|
|
252
301
|
authority: this.ownerAuthority(),
|
|
253
302
|
config: this.config,
|
|
254
303
|
releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
|
|
304
|
+
singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds),
|
|
255
305
|
snapshot: this.status()
|
|
256
306
|
})
|
|
257
307
|
}
|
|
@@ -289,10 +339,10 @@ export default class RollbridgeDaemon {
|
|
|
289
339
|
prepared = legacyBridge.prepared
|
|
290
340
|
}
|
|
291
341
|
const preparedStatus = await this.guardian.replacementStatus()
|
|
292
|
-
const transfer = /** @type {
|
|
342
|
+
const transfer = /** @type {PrivateOwnerState} */ (prepared.ownerState)
|
|
293
343
|
|
|
294
344
|
if (!transfer?.config || !transfer.snapshot) throw new Error("Committed owner published incomplete replacement state")
|
|
295
|
-
await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, releaseConfigs: transfer.releaseConfigs, resumeDrains: false})
|
|
345
|
+
await this.restoreOwnerState(transfer.snapshot, {config: transfer.config, releaseConfigs: transfer.releaseConfigs, resumeDrains: false, singletonReleaseIds: transfer.singletonReleaseIds, synchronizeLifecycleRoles: false})
|
|
296
346
|
for (const release of this.releases.values()) release.preserveConfigOnRetirement = true
|
|
297
347
|
this.logger("owner replacement candidate prepared", {activeReleaseId: this.activeRelease?.releaseId ?? null, replacementId: prepared.replacementId})
|
|
298
348
|
|
|
@@ -356,6 +406,7 @@ export default class RollbridgeDaemon {
|
|
|
356
406
|
authority: this.ownerAuthority(),
|
|
357
407
|
config: this.config,
|
|
358
408
|
releaseConfigs: Object.fromEntries([...this.releases].map(([releaseId, release]) => [releaseId, release.config])),
|
|
409
|
+
singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds),
|
|
359
410
|
snapshot: this.status()
|
|
360
411
|
})
|
|
361
412
|
|
|
@@ -381,12 +432,13 @@ export default class RollbridgeDaemon {
|
|
|
381
432
|
finalControlPublished = true
|
|
382
433
|
this.boundControlPath = this.config.control.path
|
|
383
434
|
}
|
|
435
|
+
await Promise.all([...this.releases.values()].map((release) => release.synchronizeLifecycleRoles()))
|
|
384
436
|
if (!legacyBridge && incumbentControl && [...this.releases.values()].some((release) => release.hasTransferredConnections())) {
|
|
385
437
|
this.incumbentListenerControl = incumbentControl
|
|
386
438
|
retainIncumbentControl = true
|
|
387
439
|
}
|
|
388
440
|
for (const release of this.releases.values()) {
|
|
389
|
-
if (release.state === "draining") void this.drainAndPrune(release, release.config)
|
|
441
|
+
if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
390
442
|
}
|
|
391
443
|
this.startStatePersistence()
|
|
392
444
|
await this.persistState({throwOnError: true})
|
|
@@ -509,6 +561,7 @@ export default class RollbridgeDaemon {
|
|
|
509
561
|
authority: persistedAuthority,
|
|
510
562
|
config: this.config,
|
|
511
563
|
releaseConfigs: Object.fromEntries(persisted.releases.map((release) => [release.releaseId, this.config])),
|
|
564
|
+
singletonReleaseIds: Object.fromEntries(persisted.singletons.map((singleton) => [singleton.id, persisted.activeReleaseId]).filter((entry) => entry[1] !== null)),
|
|
512
565
|
snapshot: persisted
|
|
513
566
|
}
|
|
514
567
|
const upgradedIdentity = /** @type {{pid?: number, socketPath: string, token: string}} */ ({
|
|
@@ -1022,7 +1075,7 @@ export default class RollbridgeDaemon {
|
|
|
1022
1075
|
if (!handoff || this.ownerRetired) return
|
|
1023
1076
|
for (const release of this.releases.values()) {
|
|
1024
1077
|
release.resumeDrainAfterOwnerHandoff()
|
|
1025
|
-
if (release.state === "draining") void this.drainAndPrune(release, release.config)
|
|
1078
|
+
if (release.state === "draining" && this.shouldResumeDrain(release)) void this.drainAndPrune(release, release.config)
|
|
1026
1079
|
}
|
|
1027
1080
|
if (handoff.proxy) {
|
|
1028
1081
|
await this.startProxy()
|
|
@@ -1048,6 +1101,16 @@ export default class RollbridgeDaemon {
|
|
|
1048
1101
|
this.assertReloadCompatible(nextConfig)
|
|
1049
1102
|
|
|
1050
1103
|
const newReleaseId = releaseId || revision || new Date().toISOString().replace(/[^0-9]/g, "")
|
|
1104
|
+
const transition = this.generationTransition
|
|
1105
|
+
|
|
1106
|
+
if (transition && transition.phase !== "committed") {
|
|
1107
|
+
this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
|
|
1108
|
+
return await this.resumeGenerationTransition()
|
|
1109
|
+
}
|
|
1110
|
+
if (transition?.phase === "committed" && transition.candidateReleaseId === newReleaseId && this.activeRelease?.releaseId === newReleaseId) {
|
|
1111
|
+
this.assertExactGenerationTransition(transition, {config: nextConfig, releaseId: newReleaseId, releasePath, revision: revision || newReleaseId})
|
|
1112
|
+
return {activeReleaseId: newReleaseId, previousReleaseId: transition.previousReleaseId}
|
|
1113
|
+
}
|
|
1051
1114
|
const release = new ReleaseGroup({
|
|
1052
1115
|
config: nextConfig,
|
|
1053
1116
|
logger: this.logger,
|
|
@@ -1080,6 +1143,25 @@ export default class RollbridgeDaemon {
|
|
|
1080
1143
|
|
|
1081
1144
|
const previousRelease = this.activeRelease
|
|
1082
1145
|
|
|
1146
|
+
if (nextConfig.processes.some((processConfig) => processConfig.lifecycle.activateCommand !== undefined)) {
|
|
1147
|
+
this.config = nextConfig
|
|
1148
|
+
this.releases.set(release.releaseId, release)
|
|
1149
|
+
const now = new Date().toISOString()
|
|
1150
|
+
|
|
1151
|
+
this.generationTransition = /** @type {GenerationTransition} */ ({
|
|
1152
|
+
candidateReleaseId: release.releaseId,
|
|
1153
|
+
candidateReleasePath: release.releasePath,
|
|
1154
|
+
candidateRevision: release.revision,
|
|
1155
|
+
configDigest: ownerConfigDigest(nextConfig),
|
|
1156
|
+
phase: "candidate_ready",
|
|
1157
|
+
previousReleaseId: previousRelease?.releaseId ?? null,
|
|
1158
|
+
startedAt: now,
|
|
1159
|
+
updatedAt: now
|
|
1160
|
+
})
|
|
1161
|
+
await this.checkpointGenerationTransition()
|
|
1162
|
+
return await this.resumeGenerationTransition()
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1083
1165
|
this.config = nextConfig
|
|
1084
1166
|
this.releases.set(release.releaseId, release)
|
|
1085
1167
|
release.activate()
|
|
@@ -1113,6 +1195,138 @@ export default class RollbridgeDaemon {
|
|
|
1113
1195
|
}
|
|
1114
1196
|
}
|
|
1115
1197
|
|
|
1198
|
+
/**
|
|
1199
|
+
* Continues one exact durable generation transition without an internal retry loop.
|
|
1200
|
+
* @returns {Promise<Record<string, JsonValue>>} Deploy result after commit.
|
|
1201
|
+
*/
|
|
1202
|
+
async resumeGenerationTransition() {
|
|
1203
|
+
const transition = this.generationTransition
|
|
1204
|
+
|
|
1205
|
+
if (!transition) throw new Error("No release generation transition to resume")
|
|
1206
|
+
const release = this.releases.get(transition.candidateReleaseId)
|
|
1207
|
+
const previousRelease = transition.previousReleaseId ? this.releases.get(transition.previousReleaseId) : undefined
|
|
1208
|
+
|
|
1209
|
+
if (!release) throw new Error(`Generation transition candidate ${transition.candidateReleaseId} is not retained`)
|
|
1210
|
+
if (transition.previousReleaseId && !previousRelease) throw new Error(`Generation transition previous release ${transition.previousReleaseId} is not retained`)
|
|
1211
|
+
|
|
1212
|
+
if (transition.phase === "candidate_ready") {
|
|
1213
|
+
if (previousRelease) await this.updateGenerationTransition("retiring_previous")
|
|
1214
|
+
else await this.updateGenerationTransition("previous_retired")
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
if (transition.phase === "retiring_previous") {
|
|
1218
|
+
try {
|
|
1219
|
+
const retirementConfig = previousRelease?.config
|
|
1220
|
+
|
|
1221
|
+
// Every entry into this journaled phase represents one explicit attempt. Reset the
|
|
1222
|
+
// process-local acknowledgement cache so exact resume can replay an ambiguous or
|
|
1223
|
+
// failed idempotent retirement once; there is no internal retry loop.
|
|
1224
|
+
await previousRelease?.beginRetirement(retirementConfig, {retry: true})
|
|
1225
|
+
} catch (error) {
|
|
1226
|
+
const failure = previousRelease?.retirementError ?? (error instanceof Error ? error.message : String(error))
|
|
1227
|
+
|
|
1228
|
+
await this.failGenerationTransition(failure)
|
|
1229
|
+
this.logger("release retirement quiescence failed", {error: failure, releaseId: previousRelease?.releaseId ?? null})
|
|
1230
|
+
throw error
|
|
1231
|
+
}
|
|
1232
|
+
await this.updateGenerationTransition("previous_retired")
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
if (transition.phase === "previous_retired") await this.updateGenerationTransition("activating_candidate")
|
|
1236
|
+
|
|
1237
|
+
if (transition.phase === "activating_candidate") {
|
|
1238
|
+
try {
|
|
1239
|
+
await release.activateGeneration()
|
|
1240
|
+
} catch (error) {
|
|
1241
|
+
const failure = error instanceof Error ? error.message : String(error)
|
|
1242
|
+
|
|
1243
|
+
await this.failGenerationTransition(failure)
|
|
1244
|
+
this.logger("release generation activation failed", {error: failure, releaseId: release.releaseId})
|
|
1245
|
+
throw error
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
// Activation acknowledgement and the logical proxy commit deliberately share one
|
|
1249
|
+
// synchronous continuation: no awaited failure boundary may leave jobs active while
|
|
1250
|
+
// Rollbridge still points new traffic at the retired generation.
|
|
1251
|
+
release.activate()
|
|
1252
|
+
this.activeRelease = release
|
|
1253
|
+
transition.phase = "committed_pending"
|
|
1254
|
+
transition.error = undefined
|
|
1255
|
+
transition.updatedAt = new Date().toISOString()
|
|
1256
|
+
this.logger("traffic switched", {previousReleaseId: previousRelease?.releaseId ?? null, releaseId: release.releaseId})
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
if (transition.phase === "committed_pending") {
|
|
1260
|
+
await this.checkpointGenerationTransition()
|
|
1261
|
+
this.refreshServiceDefinitions(release)
|
|
1262
|
+
if (previousRelease) {
|
|
1263
|
+
const retirementConfig = previousRelease.config
|
|
1264
|
+
|
|
1265
|
+
void this.drainAndPrune(previousRelease, retirementConfig)
|
|
1266
|
+
}
|
|
1267
|
+
try {
|
|
1268
|
+
await this.replaceSingletons(release)
|
|
1269
|
+
} catch (error) {
|
|
1270
|
+
await this.failGenerationTransition(error instanceof Error ? error.message : String(error))
|
|
1271
|
+
throw error
|
|
1272
|
+
}
|
|
1273
|
+
transition.phase = "committed"
|
|
1274
|
+
transition.error = undefined
|
|
1275
|
+
transition.updatedAt = new Date().toISOString()
|
|
1276
|
+
await this.checkpointGenerationTransition()
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
return {activeReleaseId: release.releaseId, previousReleaseId: previousRelease?.releaseId ?? null}
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1282
|
+
/**
|
|
1283
|
+
* @param {GenerationTransition} transition - Pending or committed exact transition.
|
|
1284
|
+
* @param {{config: import("./config.js").RollbridgeConfig, releaseId: string, releasePath: string, revision: string}} candidate - Requested identity.
|
|
1285
|
+
*/
|
|
1286
|
+
assertExactGenerationTransition(transition, candidate) {
|
|
1287
|
+
if (transition.candidateReleaseId !== candidate.releaseId || transition.candidateReleasePath !== candidate.releasePath || transition.candidateRevision !== candidate.revision || transition.configDigest !== ownerConfigDigest(candidate.config)) {
|
|
1288
|
+
throw new Error(`Release generation transition for ${transition.candidateReleaseId} is unresolved at ${transition.phase}; only the exact same release, path, revision, and config authority may resume it`)
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
/** @param {GenerationTransitionPhase} phase - Durable phase to enter. */
|
|
1293
|
+
async updateGenerationTransition(phase) {
|
|
1294
|
+
if (!this.generationTransition) throw new Error("No release generation transition to update")
|
|
1295
|
+
this.generationTransition.phase = phase
|
|
1296
|
+
this.generationTransition.error = undefined
|
|
1297
|
+
this.generationTransition.updatedAt = new Date().toISOString()
|
|
1298
|
+
await this.checkpointGenerationTransition()
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
/** @param {string} error - Exact failed phase diagnostic. */
|
|
1302
|
+
async failGenerationTransition(error) {
|
|
1303
|
+
if (!this.generationTransition) throw new Error("No release generation transition to fail")
|
|
1304
|
+
this.generationTransition.error = error
|
|
1305
|
+
this.generationTransition.updatedAt = new Date().toISOString()
|
|
1306
|
+
await this.checkpointGenerationTransition()
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
/** Persists and publishes the exact transition boundary before another external effect. */
|
|
1310
|
+
async checkpointGenerationTransition() {
|
|
1311
|
+
// Publish private exact definitions before the public secret-safe state can expose
|
|
1312
|
+
// a newer transition authority. A replacement can then reconstruct every guardian
|
|
1313
|
+
// registration without reading commands or environment values from statePath.
|
|
1314
|
+
await this.publishOwnerState()
|
|
1315
|
+
const write = this.persistState({throwOnError: true})
|
|
1316
|
+
|
|
1317
|
+
if (write) await write
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
/**
|
|
1321
|
+
* @param {ReleaseGroup} release - Retained release.
|
|
1322
|
+
* @returns {boolean} Whether its stop drain may run now.
|
|
1323
|
+
*/
|
|
1324
|
+
shouldResumeDrain(release) {
|
|
1325
|
+
const transition = this.generationTransition
|
|
1326
|
+
|
|
1327
|
+
return !transition || transition.phase === "committed_pending" || transition.phase === "committed" || transition.previousReleaseId !== release.releaseId
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1116
1330
|
/**
|
|
1117
1331
|
* Relinquishes only daemon authority/listeners after the guardian has committed a
|
|
1118
1332
|
* prepared replacement. Guardian-owned processes and drains are never stopped.
|
|
@@ -1181,6 +1395,7 @@ export default class RollbridgeDaemon {
|
|
|
1181
1395
|
* @returns {Promise<Record<string, JsonValue>>} The rollback result.
|
|
1182
1396
|
*/
|
|
1183
1397
|
async rollback({releaseId} = {}) {
|
|
1398
|
+
this.assertNoUnresolvedGenerationTransition("rollback")
|
|
1184
1399
|
const target = releaseId ? this.releases.get(releaseId) : this.previousRelease()
|
|
1185
1400
|
|
|
1186
1401
|
if (!target) {
|
|
@@ -1304,6 +1519,9 @@ export default class RollbridgeDaemon {
|
|
|
1304
1519
|
if (processConfig.policy !== "singleton") continue
|
|
1305
1520
|
|
|
1306
1521
|
const previous = this.singletons.get(processConfig.id)
|
|
1522
|
+
const previousReleaseId = this.singletonReleaseIds.get(processConfig.id)
|
|
1523
|
+
|
|
1524
|
+
if (previous && previousReleaseId === release.releaseId && previous.status().state === "running") continue
|
|
1307
1525
|
|
|
1308
1526
|
if (previous) {
|
|
1309
1527
|
await previous.stop()
|
|
@@ -1313,8 +1531,10 @@ export default class RollbridgeDaemon {
|
|
|
1313
1531
|
const singleton = release.buildProcess(processConfig, {guardianKey: `singleton:${release.releaseId}:${processConfig.id}`})
|
|
1314
1532
|
|
|
1315
1533
|
this.singletons.set(processConfig.id, singleton)
|
|
1534
|
+
this.singletonReleaseIds.set(processConfig.id, release.releaseId)
|
|
1316
1535
|
await singleton.start("deploy")
|
|
1317
1536
|
}
|
|
1537
|
+
this.pruneStoppedReleases()
|
|
1318
1538
|
}
|
|
1319
1539
|
|
|
1320
1540
|
/**
|
|
@@ -1328,6 +1548,7 @@ export default class RollbridgeDaemon {
|
|
|
1328
1548
|
* @returns {Promise<Record<string, JsonValue>>} The ids that were restarted.
|
|
1329
1549
|
*/
|
|
1330
1550
|
async restartProcesses({policy, processId} = {}) {
|
|
1551
|
+
this.assertNoUnresolvedGenerationTransition("restart")
|
|
1331
1552
|
if (policy === "proxied" || (processId !== undefined && this.isProxiedId(processId))) {
|
|
1332
1553
|
throw new Error('The proxied process cannot be restarted in place; use "rollbridge deploy" for a zero-downtime replacement.')
|
|
1333
1554
|
}
|
|
@@ -1341,7 +1562,7 @@ export default class RollbridgeDaemon {
|
|
|
1341
1562
|
for (const target of targets) {
|
|
1342
1563
|
this.logger("process restart requested", {processId: target.id})
|
|
1343
1564
|
await target.process.stop()
|
|
1344
|
-
await target.process.start("manual")
|
|
1565
|
+
await target.process.start("manual", target.process.lifecycle.activateCommand ? "active" : undefined)
|
|
1345
1566
|
}
|
|
1346
1567
|
|
|
1347
1568
|
return {restarted: targets.map((target) => target.id)}
|
|
@@ -1402,6 +1623,7 @@ export default class RollbridgeDaemon {
|
|
|
1402
1623
|
* @returns {Promise<void>} Resolves when stopped.
|
|
1403
1624
|
*/
|
|
1404
1625
|
async stopRelease(releaseId) {
|
|
1626
|
+
this.assertNoUnresolvedGenerationTransition("stop")
|
|
1405
1627
|
const release = releaseId ? this.releases.get(releaseId) : this.activeRelease
|
|
1406
1628
|
|
|
1407
1629
|
if (!release) throw new Error(`Release not found: ${releaseId || "active"}`)
|
|
@@ -1413,6 +1635,15 @@ export default class RollbridgeDaemon {
|
|
|
1413
1635
|
this.persistState()
|
|
1414
1636
|
}
|
|
1415
1637
|
|
|
1638
|
+
/** @param {string} operation - Mutating control operation. */
|
|
1639
|
+
assertNoUnresolvedGenerationTransition(operation) {
|
|
1640
|
+
const transition = this.generationTransition
|
|
1641
|
+
|
|
1642
|
+
if (transition && transition.phase !== "committed") {
|
|
1643
|
+
throw new Error(`Cannot ${operation} while release generation transition ${transition.candidateReleaseId} is unresolved at ${transition.phase}; resume the exact deploy first`)
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1416
1647
|
/**
|
|
1417
1648
|
* Drains and stops a retired release in the background, then prunes stopped releases.
|
|
1418
1649
|
* @param {ReleaseGroup} release - Release to drain and stop.
|
|
@@ -1435,9 +1666,13 @@ export default class RollbridgeDaemon {
|
|
|
1435
1666
|
|
|
1436
1667
|
/** @returns {void} Removes stopped releases beyond the retention policy. */
|
|
1437
1668
|
pruneStoppedReleases() {
|
|
1438
|
-
const
|
|
1669
|
+
const singletonOwnerReleaseIds = new Set(this.singletonReleaseIds.values())
|
|
1670
|
+
const statuses = [...this.releases.values()]
|
|
1671
|
+
.filter((release) => !singletonOwnerReleaseIds.has(release.releaseId))
|
|
1672
|
+
.map((release) => release.status())
|
|
1439
1673
|
|
|
1440
1674
|
for (const releaseId of releasesToPrune(statuses, this.config.releaseRetention, Date.now())) {
|
|
1675
|
+
this.releases.get(releaseId)?.releasePortReservations()
|
|
1441
1676
|
this.releases.delete(releaseId)
|
|
1442
1677
|
}
|
|
1443
1678
|
}
|
|
@@ -1469,9 +1704,10 @@ export default class RollbridgeDaemon {
|
|
|
1469
1704
|
...status,
|
|
1470
1705
|
events,
|
|
1471
1706
|
persistedAt: new Date().toISOString(),
|
|
1707
|
+
...(this.hasActivationLifecycle() ? {singletonReleaseIds: Object.fromEntries(this.singletonReleaseIds)} : {}),
|
|
1472
1708
|
...(this.guardianIdentity ? {recovery: {
|
|
1473
1709
|
configDigest: this.ownerRecoveryConfigDigest(),
|
|
1474
|
-
format: 1,
|
|
1710
|
+
format: this.hasActivationLifecycle() ? 2 : 1,
|
|
1475
1711
|
guardian: this.guardianIdentity,
|
|
1476
1712
|
reconnectGraceMs: this.config.ownerRecovery?.reconnectGraceMs
|
|
1477
1713
|
}} : {})
|
|
@@ -1491,6 +1727,11 @@ export default class RollbridgeDaemon {
|
|
|
1491
1727
|
return this.pendingWrite
|
|
1492
1728
|
}
|
|
1493
1729
|
|
|
1730
|
+
/** @returns {boolean} Whether the current authority uses explicit generation activation. */
|
|
1731
|
+
hasActivationLifecycle() {
|
|
1732
|
+
return Boolean(this.generationTransition) || this.config.processes.some((processConfig) => processConfig.lifecycle?.activateCommand !== undefined) || [...this.releases.values()].some((release) => release.config.processes.some((processConfig) => processConfig.lifecycle?.activateCommand !== undefined))
|
|
1733
|
+
}
|
|
1734
|
+
|
|
1494
1735
|
/**
|
|
1495
1736
|
* On startup, reads any state left by a previous daemon and reports managed processes whose
|
|
1496
1737
|
* pids are still alive — likely orphans from a daemon that did not shut down cleanly. This is
|
|
@@ -1678,6 +1919,8 @@ export default class RollbridgeDaemon {
|
|
|
1678
1919
|
// a cleared orphan must not reappear if the OS later recycles its pid for an unrelated process.
|
|
1679
1920
|
this.orphans = this.orphans.filter((orphan) => isProcessAlive(orphan.pid))
|
|
1680
1921
|
|
|
1922
|
+
const singletonOwnerReleaseIds = new Set(this.singletonReleaseIds.values())
|
|
1923
|
+
|
|
1681
1924
|
return {
|
|
1682
1925
|
activeReleaseId: this.activeRelease ? this.activeRelease.releaseId : null,
|
|
1683
1926
|
application: this.config.application,
|
|
@@ -1685,6 +1928,7 @@ export default class RollbridgeDaemon {
|
|
|
1685
1928
|
control: {...this.config.control},
|
|
1686
1929
|
daemonPid: process.pid,
|
|
1687
1930
|
daemonRuntime: this.runtime ? {...this.runtime} : undefined,
|
|
1931
|
+
generationTransition: this.generationTransition ? {...this.generationTransition} : undefined,
|
|
1688
1932
|
ownerRecovery: this.guardian ? {configDigest: this.ownerRecoveryConfigDigest()} : undefined,
|
|
1689
1933
|
ownerTransition: this.ownerTransition ? {...this.ownerTransition} : undefined,
|
|
1690
1934
|
orphans: [...this.orphans],
|
|
@@ -1694,7 +1938,7 @@ export default class RollbridgeDaemon {
|
|
|
1694
1938
|
upstreamHost: this.config.proxy.upstreamHost
|
|
1695
1939
|
},
|
|
1696
1940
|
releaseReferences: [...this.releases.values()]
|
|
1697
|
-
.filter((release) => release.state === "active" || release.state === "draining")
|
|
1941
|
+
.filter((release) => release.state === "active" || release.state === "draining" || singletonOwnerReleaseIds.has(release.releaseId) || (this.generationTransition?.phase !== "committed" && this.generationTransition?.candidateReleaseId === release.releaseId))
|
|
1698
1942
|
.map((release) => ({releaseId: release.releaseId, releasePath: release.releasePath})),
|
|
1699
1943
|
releases: [...this.releases.values()].map((release) => release.status()),
|
|
1700
1944
|
services: [...this.services.entries()].map(([id, processInstance]) => ({
|
package/src/guardian-client.js
CHANGED
|
@@ -310,7 +310,7 @@ export default class GuardianClient {
|
|
|
310
310
|
|
|
311
311
|
this.buffer = this.buffer.slice(newline + 1)
|
|
312
312
|
if (message.event) {
|
|
313
|
-
if (message.event === "process" || message.event === "status") this.processes.get(message.key)?.onGuardianEvent(message)
|
|
313
|
+
if (message.event === "process" || message.event === "process-log" || message.event === "status") this.processes.get(message.key)?.onGuardianEvent(message)
|
|
314
314
|
for (const handler of this.eventHandlers.get(message.event) || []) handler(message)
|
|
315
315
|
const waiter = this.events.get(message.event)?.shift()
|
|
316
316
|
|
|
@@ -354,10 +354,15 @@ class GuardianProcess extends ManagedProcess {
|
|
|
354
354
|
await this.ensureRegistered()
|
|
355
355
|
}
|
|
356
356
|
|
|
357
|
-
|
|
357
|
+
/**
|
|
358
|
+
* @param {import("./managed-process.js").ManagedProcessStartReason} [reason] - Start reason.
|
|
359
|
+
* @param {import("./managed-process.js").LifecycleRole} [lifecycleRole] - Desired role restored before running.
|
|
360
|
+
*/
|
|
361
|
+
async start(reason = "deploy", lifecycleRole) {
|
|
358
362
|
await this.ensureRegistered()
|
|
359
363
|
await this.pendingUpdate
|
|
360
|
-
|
|
364
|
+
if (lifecycleRole) this.lifecycleRole = lifecycleRole
|
|
365
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "start", key: this.key, lifecycleRole, reason}))
|
|
361
366
|
}
|
|
362
367
|
|
|
363
368
|
/** @param {import("./managed-process.js").ManagedProcessDefinition} definition - Updated definition. */
|
|
@@ -383,6 +388,27 @@ class GuardianProcess extends ManagedProcess {
|
|
|
383
388
|
await this.quiesce()
|
|
384
389
|
}
|
|
385
390
|
|
|
391
|
+
async requiesceStrict() {
|
|
392
|
+
await this.ensureRegistered()
|
|
393
|
+
await this.pendingUpdate
|
|
394
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "requiesce", key: this.key}))
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
async activateStrict() {
|
|
398
|
+
await this.ensureRegistered()
|
|
399
|
+
await this.pendingUpdate
|
|
400
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "activate", key: this.key}))
|
|
401
|
+
this.lifecycleRole = "active"
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/** @param {import("./managed-process.js").LifecycleRole} role - Exact generation role. */
|
|
405
|
+
async setLifecycleRole(role) {
|
|
406
|
+
await this.ensureRegistered()
|
|
407
|
+
await this.pendingUpdate
|
|
408
|
+
this.cachedStatus = asProcessStatus(await this.client.request({command: "set-lifecycle-role", key: this.key, lifecycleRole: role}))
|
|
409
|
+
this.lifecycleRole = role
|
|
410
|
+
}
|
|
411
|
+
|
|
386
412
|
async stop(options = {}) {
|
|
387
413
|
await this.ensureRegistered()
|
|
388
414
|
await this.pendingUpdate
|
|
@@ -396,6 +422,10 @@ class GuardianProcess extends ManagedProcess {
|
|
|
396
422
|
/** @param {Record<string, import("./json.js").JsonValue>} event - Guardian event. */
|
|
397
423
|
onGuardianEvent(event) {
|
|
398
424
|
if (event.status) this.cachedStatus = asProcessStatus(event.status)
|
|
425
|
+
if (event.event === "process-log") {
|
|
426
|
+
this.emit("log", asProcessLog(event.entry))
|
|
427
|
+
return
|
|
428
|
+
}
|
|
399
429
|
if (event.message === "process started") this.emit("started")
|
|
400
430
|
if (event.message === "process exited") this.emit("exit", event.data)
|
|
401
431
|
this.logger(typeof event.message === "string" ? event.message : "guardian process status", event.data && typeof event.data === "object" && !Array.isArray(event.data) ? event.data : {})
|
|
@@ -430,6 +460,14 @@ function asProcessStatus(value) {
|
|
|
430
460
|
return JSON.parse(JSON.stringify(value))
|
|
431
461
|
}
|
|
432
462
|
|
|
463
|
+
/**
|
|
464
|
+
* @param {import("./json.js").JsonValue} value - Protocol value.
|
|
465
|
+
* @returns {import("./managed-process.js").ManagedProcessLog} Process output entry.
|
|
466
|
+
*/
|
|
467
|
+
function asProcessLog(value) {
|
|
468
|
+
return JSON.parse(JSON.stringify(value))
|
|
469
|
+
}
|
|
470
|
+
|
|
433
471
|
/**
|
|
434
472
|
* @param {Error | string} error - Error-like value.
|
|
435
473
|
* @returns {string} Error message.
|