rollbridge 0.1.58 → 0.1.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/docs/config.md +40 -0
- package/docs/troubleshooting.md +43 -6
- package/package.json +1 -1
- package/src/config.js +91 -2
- package/src/daemon.js +10 -8
- package/src/guardian-client.js +13 -2
- package/src/health.js +99 -8
- package/src/process-guardian.js +125 -15
- package/src/process-order.js +26 -0
- package/src/release-group.js +157 -22
- package/test/config-validation.test.js +53 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/guardian-client.test.js +32 -0
- package/test/health.test.js +74 -1
- package/test/rollbridge.test.js +128 -0
package/src/process-guardian.js
CHANGED
|
@@ -12,6 +12,10 @@ import ManagedProcess from "./managed-process.js"
|
|
|
12
12
|
import {writeState} from "./state-store.js"
|
|
13
13
|
|
|
14
14
|
const OWNER_RESTART_RETRY_MS = 1000
|
|
15
|
+
// An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
|
|
16
|
+
// generation may legitimately drain for hours, but a mutation that holds the
|
|
17
|
+
// fence that long is stuck, so the lease is bounded and released on expiry.
|
|
18
|
+
const OWNER_MUTATION_LEASE_MS = 900000
|
|
15
19
|
|
|
16
20
|
/** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
|
|
17
21
|
|
|
@@ -32,7 +36,9 @@ const OWNER_RESTART_RETRY_MS = 1000
|
|
|
32
36
|
* @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
|
|
33
37
|
* @property {string} [replacementId] - Prepared replacement transaction id.
|
|
34
38
|
* @property {string} [mutationId] - Owner mutation lease id.
|
|
39
|
+
* @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
|
|
35
40
|
* @property {string} [operation] - Owner mutation diagnostic name.
|
|
41
|
+
* @property {string} [waitingFor] - What the held owner mutation is blocked on.
|
|
36
42
|
* @property {number} [ownerPid] - Exact claimant process PID.
|
|
37
43
|
* @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
|
|
38
44
|
* @property {string} [statePath] - Exact public recovery state path.
|
|
@@ -103,6 +109,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
|
|
|
103
109
|
let ownerMutationClient
|
|
104
110
|
/** @type {string | undefined} */
|
|
105
111
|
let ownerMutationId
|
|
112
|
+
/** @type {string | undefined} */
|
|
113
|
+
let ownerMutationOperation
|
|
114
|
+
/** @type {string | undefined} */
|
|
115
|
+
let ownerMutationWaitingFor
|
|
116
|
+
/** @type {number | undefined} */
|
|
117
|
+
let ownerMutationPid
|
|
118
|
+
/** @type {number | undefined} */
|
|
119
|
+
let ownerMutationStartedAt
|
|
120
|
+
let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
121
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
122
|
+
let ownerMutationLeaseTimer
|
|
123
|
+
/** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
|
|
124
|
+
let ownerMutationExpired
|
|
106
125
|
/** @type {net.Socket | undefined} */
|
|
107
126
|
let retiringClient
|
|
108
127
|
/** @type {string | undefined} */
|
|
@@ -154,10 +173,7 @@ const server = net.createServer((socket) => {
|
|
|
154
173
|
clearTimeout(waiter.timer)
|
|
155
174
|
waiter.reject(new Error("Owner claimant disconnected"))
|
|
156
175
|
}
|
|
157
|
-
if (ownerMutationClient === socket && ownerClient !== socket)
|
|
158
|
-
ownerMutationClient = undefined
|
|
159
|
-
ownerMutationId = undefined
|
|
160
|
-
}
|
|
176
|
+
if (ownerMutationClient === socket && ownerClient !== socket) clearOwnerMutationLease()
|
|
161
177
|
if (controlLessRetirement) {
|
|
162
178
|
const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
|
|
163
179
|
|
|
@@ -301,8 +317,7 @@ async function execute(request, socket) {
|
|
|
301
317
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
302
318
|
ownerClient = undefined
|
|
303
319
|
ownerClientPid = undefined
|
|
304
|
-
|
|
305
|
-
ownerMutationId = undefined
|
|
320
|
+
clearOwnerMutationLease()
|
|
306
321
|
ownerRevision += 1
|
|
307
322
|
grantNextOwner()
|
|
308
323
|
return {retired: true}
|
|
@@ -397,26 +412,48 @@ async function execute(request, socket) {
|
|
|
397
412
|
if (request.command === "begin-owner-mutation") {
|
|
398
413
|
requireOwner(socket, request.command)
|
|
399
414
|
if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
|
|
400
|
-
|
|
415
|
+
const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
|
|
416
|
+
|
|
417
|
+
if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
|
|
418
|
+
if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
|
|
401
419
|
ownerMutationClient = socket
|
|
402
420
|
ownerMutationId = crypto.randomUUID()
|
|
403
|
-
|
|
421
|
+
ownerMutationOperation = request.operation
|
|
422
|
+
ownerMutationWaitingFor = request.waitingFor
|
|
423
|
+
ownerMutationPid = request.ownerPid ?? ownerClientPid
|
|
424
|
+
ownerMutationStartedAt = Date.now()
|
|
425
|
+
ownerMutationLeaseMs = leaseMs
|
|
426
|
+
armOwnerMutationLease(ownerMutationId)
|
|
427
|
+
return {leaseMs, mutationId: ownerMutationId}
|
|
404
428
|
}
|
|
405
429
|
|
|
406
430
|
if (request.command === "end-owner-mutation") {
|
|
407
431
|
requireOwner(socket, request.command)
|
|
432
|
+
if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
|
|
433
|
+
const expired = ownerMutationExpired
|
|
434
|
+
|
|
435
|
+
ownerMutationExpired = undefined
|
|
436
|
+
throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
|
|
437
|
+
}
|
|
408
438
|
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
409
|
-
|
|
410
|
-
ownerMutationId = undefined
|
|
439
|
+
clearOwnerMutationLease()
|
|
411
440
|
return {ended: true}
|
|
412
441
|
}
|
|
413
442
|
|
|
443
|
+
if (request.command === "report-owner-mutation-wait") {
|
|
444
|
+
requireOwner(socket, request.command)
|
|
445
|
+
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
446
|
+
if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
|
|
447
|
+
ownerMutationWaitingFor = request.waitingFor.trim()
|
|
448
|
+
return {reported: true}
|
|
449
|
+
}
|
|
450
|
+
|
|
414
451
|
if (request.command === "prepare-owner-replacement") {
|
|
415
452
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
416
453
|
if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
|
|
417
454
|
if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
|
|
418
455
|
if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
|
|
419
|
-
if (ownerMutationClient) throw new Error(
|
|
456
|
+
if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
|
|
420
457
|
if (!ownerState) throw new Error("Committed owner has not published transferable state")
|
|
421
458
|
const currentAuthority = ownerAuthority(ownerState)
|
|
422
459
|
const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
|
|
@@ -715,6 +752,82 @@ async function execute(request, socket) {
|
|
|
715
752
|
return status
|
|
716
753
|
}
|
|
717
754
|
|
|
755
|
+
/**
|
|
756
|
+
* Names the mutation holding the owner-mutation fence: which mutation, which process,
|
|
757
|
+
* since when, and what it is blocked on. Refusals must be diagnosable without a live
|
|
758
|
+
* inspection of the daemon.
|
|
759
|
+
* @returns {string} Holder description used in refusal messages.
|
|
760
|
+
*/
|
|
761
|
+
function describeOwnerMutationLease() {
|
|
762
|
+
const startedAt = ownerMutationStartedAt ?? 0
|
|
763
|
+
const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
|
|
764
|
+
const holderPid = ownerMutationPid ?? ownerClientPid
|
|
765
|
+
|
|
766
|
+
return [
|
|
767
|
+
`owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
|
|
768
|
+
`mutation ${ownerMutationId || "unknown"}`,
|
|
769
|
+
`held by process ${holderPid ?? "unknown"}`,
|
|
770
|
+
`since ${new Date(startedAt).toISOString()}`,
|
|
771
|
+
`${elapsedSeconds}s elapsed`,
|
|
772
|
+
`waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
|
|
773
|
+
].join("; ")
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
/** @returns {void} */
|
|
777
|
+
function clearOwnerMutationLease() {
|
|
778
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
779
|
+
ownerMutationLeaseTimer = undefined
|
|
780
|
+
ownerMutationClient = undefined
|
|
781
|
+
ownerMutationId = undefined
|
|
782
|
+
ownerMutationOperation = undefined
|
|
783
|
+
ownerMutationWaitingFor = undefined
|
|
784
|
+
ownerMutationPid = undefined
|
|
785
|
+
ownerMutationStartedAt = undefined
|
|
786
|
+
ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* Arms the lease bound for one mutation.
|
|
791
|
+
* @param {string} mutationId - Lease id the timer belongs to.
|
|
792
|
+
* @returns {void}
|
|
793
|
+
*/
|
|
794
|
+
function armOwnerMutationLease(mutationId) {
|
|
795
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
796
|
+
ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
|
|
797
|
+
ownerMutationLeaseTimer.unref()
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
/**
|
|
801
|
+
* A mutation that outruns its lease is reported and released instead of holding the fence
|
|
802
|
+
* forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
|
|
803
|
+
* lease is gone when it tries to end it.
|
|
804
|
+
* @param {string} mutationId - Lease id the armed timer belongs to.
|
|
805
|
+
* @returns {void}
|
|
806
|
+
*/
|
|
807
|
+
function expireOwnerMutationLease(mutationId) {
|
|
808
|
+
ownerMutationLeaseTimer = undefined
|
|
809
|
+
if (!ownerMutationClient || ownerMutationId !== mutationId) return
|
|
810
|
+
const operation = ownerMutationOperation || "unnamed operation"
|
|
811
|
+
const leaseMs = ownerMutationLeaseMs
|
|
812
|
+
const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
|
|
813
|
+
const diagnostic = {
|
|
814
|
+
code: "OWNER_MUTATION_LEASE_EXPIRED",
|
|
815
|
+
elapsedMs,
|
|
816
|
+
event: "owner-mutation-expired",
|
|
817
|
+
leaseMs,
|
|
818
|
+
mutationId,
|
|
819
|
+
operation,
|
|
820
|
+
pid: ownerMutationPid ?? ownerClientPid,
|
|
821
|
+
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
825
|
+
if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
|
|
826
|
+
broadcast(diagnostic)
|
|
827
|
+
ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
|
|
828
|
+
clearOwnerMutationLease()
|
|
829
|
+
}
|
|
830
|
+
|
|
718
831
|
/**
|
|
719
832
|
* @param {net.Socket} socket - Requesting client.
|
|
720
833
|
* @param {string} command - Command name.
|
|
@@ -735,10 +848,7 @@ function releaseClosedOwner(socket) {
|
|
|
735
848
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
736
849
|
ownerClient = undefined
|
|
737
850
|
ownerClientPid = undefined
|
|
738
|
-
if (ownerMutationClient === socket)
|
|
739
|
-
ownerMutationClient = undefined
|
|
740
|
-
ownerMutationId = undefined
|
|
741
|
-
}
|
|
851
|
+
if (ownerMutationClient === socket) clearOwnerMutationLease()
|
|
742
852
|
if (replacementClient && replacementOwnerState) commitReplacement()
|
|
743
853
|
else {
|
|
744
854
|
if (incompleteRestartChild) {
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @typedef {import("./config.js").ProcessConfig} ProcessConfig
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const POLICY_START_RANK = /** @type {Record<ProcessConfig["policy"], number>} */ ({companion: 1, proxied: 2, service: 0, singleton: 3})
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Order in which Rollbridge starts processes, which is also the only order a `dependsOn`
|
|
11
|
+
* edge may point backwards in: daemon-wide persistent services are started before every
|
|
12
|
+
* release generation, then handoff services, companions, and the proxied process last.
|
|
13
|
+
* Config order is kept inside each group.
|
|
14
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
15
|
+
* @param {{includePersistentServices?: boolean}} [options] - Whether to include the
|
|
16
|
+
* daemon-wide persistent services that a release group never starts itself.
|
|
17
|
+
* @returns {ProcessConfig[]} Ordered process configs.
|
|
18
|
+
*/
|
|
19
|
+
export function processStartOrder(processes, {includePersistentServices = false} = {}) {
|
|
20
|
+
const persistentServices = includePersistentServices
|
|
21
|
+
? processes.filter((processConfig) => processConfig.policy === "service" && processConfig.deployStrategy === "persistent")
|
|
22
|
+
: []
|
|
23
|
+
const generationProcesses = processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
|
|
24
|
+
|
|
25
|
+
return [...persistentServices, ...generationProcesses.sort((left, right) => POLICY_START_RANK[left.policy] - POLICY_START_RANK[right.policy])]
|
|
26
|
+
}
|
package/src/release-group.js
CHANGED
|
@@ -4,8 +4,11 @@ import {spawn} from "node:child_process"
|
|
|
4
4
|
import {EventEmitter} from "node:events"
|
|
5
5
|
import ManagedProcess from "./managed-process.js"
|
|
6
6
|
import {findAvailablePort} from "./port-allocator.js"
|
|
7
|
+
import {processStartOrder} from "./process-order.js"
|
|
7
8
|
import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
|
|
8
|
-
import {waitForHealth} from "./health.js"
|
|
9
|
+
import {waitForHealth, waitForListeningPort} from "./health.js"
|
|
10
|
+
|
|
11
|
+
const DEPENDENCY_READY_INTERVAL_MS = 250
|
|
9
12
|
|
|
10
13
|
/**
|
|
11
14
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -13,6 +16,7 @@ import {waitForHealth} from "./health.js"
|
|
|
13
16
|
* @typedef {{http: number, websocket: number}} ReleaseConnections
|
|
14
17
|
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
15
18
|
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
|
|
19
|
+
* @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
|
|
16
20
|
*/
|
|
17
21
|
|
|
18
22
|
|
|
@@ -57,6 +61,14 @@ function envId(id) {
|
|
|
57
61
|
return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
|
|
58
62
|
}
|
|
59
63
|
|
|
64
|
+
/**
|
|
65
|
+
* @param {number | undefined} startedAtMs - When the measured phase began.
|
|
66
|
+
* @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
|
|
67
|
+
*/
|
|
68
|
+
function elapsedSince(startedAtMs) {
|
|
69
|
+
return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
|
|
70
|
+
}
|
|
71
|
+
|
|
60
72
|
/**
|
|
61
73
|
* @param {import("./config.js").ProcessConfig} processConfig - Process config.
|
|
62
74
|
* @param {number} index - Zero-based replica index.
|
|
@@ -107,16 +119,27 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
107
119
|
this.stoppedAt = /** @type {string | undefined} */ (undefined)
|
|
108
120
|
this.retirementError = /** @type {string | undefined} */ (undefined)
|
|
109
121
|
this.preserveConfigOnRetirement = false
|
|
122
|
+
this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
|
|
123
|
+
this.readyProcessIds = /** @type {Set<string>} */ (new Set())
|
|
124
|
+
this.startedProcessIds = /** @type {Set<string>} */ (new Set())
|
|
125
|
+
this.startupDependencyWaitMs = 0
|
|
126
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
|
|
127
|
+
this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
|
|
110
128
|
}
|
|
111
129
|
|
|
112
130
|
/** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
|
|
113
131
|
async start() {
|
|
114
132
|
this.state = "starting"
|
|
133
|
+
this.startupStartedAtMs = Date.now()
|
|
115
134
|
|
|
116
135
|
try {
|
|
117
136
|
await this.allocatePorts()
|
|
118
137
|
|
|
119
138
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
139
|
+
await this.waitForConfiguredDependencies(processConfig)
|
|
140
|
+
/** @type {string[]} */
|
|
141
|
+
const instanceIds = []
|
|
142
|
+
|
|
120
143
|
for (let index = 0; index < processConfig.replicas; index += 1) {
|
|
121
144
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
122
145
|
|
|
@@ -127,17 +150,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
127
150
|
if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
|
|
128
151
|
if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
|
|
129
152
|
await processInstance.start("deploy")
|
|
153
|
+
this.processLaunchAtMs.set(instanceId, Date.now())
|
|
154
|
+
this.startedProcessIds.add(instanceId)
|
|
155
|
+
instanceIds.push(instanceId)
|
|
130
156
|
}
|
|
131
157
|
|
|
132
158
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
133
|
-
await
|
|
134
|
-
health: processConfig.health,
|
|
135
|
-
host: this.config.proxy.upstreamHost,
|
|
136
|
-
port: this.ports[processConfig.id]
|
|
137
|
-
})
|
|
138
|
-
|
|
139
|
-
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
159
|
+
await this.waitForProxiedHealth(processConfig)
|
|
140
160
|
}
|
|
161
|
+
|
|
162
|
+
for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
|
|
163
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
141
164
|
}
|
|
142
165
|
} catch (error) {
|
|
143
166
|
this.state = "failed"
|
|
@@ -257,16 +280,132 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
257
280
|
}
|
|
258
281
|
|
|
259
282
|
/**
|
|
260
|
-
*
|
|
283
|
+
* Waits for the ports a process declares it depends on, on a budget that is not the
|
|
284
|
+
* health budget. The dependency leg of a start is not app boot, and charging it to the
|
|
285
|
+
* health deadline measures the dependency instead of the app.
|
|
286
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
|
|
287
|
+
* @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
|
|
288
|
+
*/
|
|
289
|
+
async waitForConfiguredDependencies(processConfig) {
|
|
290
|
+
if (processConfig.dependsOn.length === 0) return
|
|
291
|
+
|
|
292
|
+
for (const dependencyId of processConfig.dependsOn) {
|
|
293
|
+
const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
|
|
294
|
+
|
|
295
|
+
if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
|
|
296
|
+
|
|
297
|
+
this.logger("release process dependency wait", {
|
|
298
|
+
dependencyPort: port,
|
|
299
|
+
dependencyProcessId: dependencyId,
|
|
300
|
+
processId: processConfig.id,
|
|
301
|
+
releaseId: this.releaseId,
|
|
302
|
+
timeoutMs: processConfig.dependencyTimeoutMs
|
|
303
|
+
})
|
|
304
|
+
|
|
305
|
+
const waitStartedAtMs = Date.now()
|
|
306
|
+
|
|
307
|
+
try {
|
|
308
|
+
const {attempts, elapsedMs} = await waitForListeningPort({
|
|
309
|
+
host: this.config.proxy.upstreamHost,
|
|
310
|
+
intervalMs: DEPENDENCY_READY_INTERVAL_MS,
|
|
311
|
+
port,
|
|
312
|
+
timeoutMs: processConfig.dependencyTimeoutMs
|
|
313
|
+
})
|
|
314
|
+
|
|
315
|
+
this.startupDependencyWaitMs += elapsedMs
|
|
316
|
+
this.logger("release process dependency ready", {
|
|
317
|
+
attempts,
|
|
318
|
+
dependencyPort: port,
|
|
319
|
+
dependencyProcessId: dependencyId,
|
|
320
|
+
elapsedMs,
|
|
321
|
+
processId: processConfig.id,
|
|
322
|
+
releaseId: this.releaseId
|
|
323
|
+
})
|
|
324
|
+
} catch (error) {
|
|
325
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
326
|
+
deadlineMs: processConfig.dependencyTimeoutMs,
|
|
327
|
+
dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
|
|
328
|
+
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
329
|
+
failedProcessId: processConfig.id,
|
|
330
|
+
probes: undefined,
|
|
331
|
+
stage: "dependency"
|
|
332
|
+
})
|
|
333
|
+
throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Health checks the proxied process on its own boot budget and records what the boot cost.
|
|
342
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
|
|
343
|
+
* @returns {Promise<void>} Resolves once the process answers its health path.
|
|
344
|
+
*/
|
|
345
|
+
async waitForProxiedHealth(processConfig) {
|
|
346
|
+
const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
|
|
347
|
+
const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
|
|
348
|
+
|
|
349
|
+
try {
|
|
350
|
+
const {elapsedMs} = await waitForHealth({
|
|
351
|
+
health,
|
|
352
|
+
host: this.config.proxy.upstreamHost,
|
|
353
|
+
port: /** @type {number} */ (this.ports[processConfig.id]),
|
|
354
|
+
processId: processConfig.id
|
|
355
|
+
})
|
|
356
|
+
|
|
357
|
+
this.logger("release candidate health ready", {
|
|
358
|
+
bootMs: Date.now() - launchedAtMs,
|
|
359
|
+
deadlineMs: health.timeoutMs,
|
|
360
|
+
healthGateMs: elapsedMs,
|
|
361
|
+
path: health.path,
|
|
362
|
+
port: this.ports[processConfig.id],
|
|
363
|
+
processId: processConfig.id,
|
|
364
|
+
releaseId: this.releaseId,
|
|
365
|
+
waitedForDependenciesMs: this.startupDependencyWaitMs
|
|
366
|
+
})
|
|
367
|
+
} catch (error) {
|
|
368
|
+
const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
|
|
369
|
+
|
|
370
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
371
|
+
deadlineMs: health.timeoutMs,
|
|
372
|
+
dependencyWaitMs: this.startupDependencyWaitMs,
|
|
373
|
+
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
374
|
+
failedProcessId: processConfig.id,
|
|
375
|
+
probes: healthCheck?.attempts,
|
|
376
|
+
stage: "health"
|
|
377
|
+
})
|
|
378
|
+
throw error
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
|
|
261
384
|
* @param {Error | string} error - Startup failure.
|
|
262
385
|
* @param {{phase: string}} options - Diagnostic phase.
|
|
263
386
|
* @returns {void}
|
|
264
387
|
*/
|
|
265
388
|
logStartupFailure(error, {phase}) {
|
|
389
|
+
const diagnostics = this.startupFailureDiagnostics
|
|
390
|
+
const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
|
|
391
|
+
const lastLog = failedLogs?.at(-1)
|
|
392
|
+
const readyProcesses = [...this.readyProcessIds]
|
|
393
|
+
|
|
266
394
|
this.logger("release startup failed", {
|
|
395
|
+
deadlineMs: diagnostics?.deadlineMs ?? null,
|
|
396
|
+
dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
|
|
397
|
+
elapsedMs: diagnostics?.elapsedMs ?? null,
|
|
267
398
|
error: error instanceof Error ? error.message : error,
|
|
399
|
+
failedProcessId: diagnostics?.failedProcessId ?? null,
|
|
400
|
+
lastOutput: lastLog?.line ?? null,
|
|
401
|
+
lastOutputAt: lastLog?.at ?? null,
|
|
268
402
|
phase,
|
|
269
|
-
|
|
403
|
+
probes: diagnostics?.probes ?? null,
|
|
404
|
+
readyProcesses,
|
|
405
|
+
releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
406
|
+
releaseId: this.releaseId,
|
|
407
|
+
stage: diagnostics?.stage ?? null,
|
|
408
|
+
unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
|
|
270
409
|
})
|
|
271
410
|
|
|
272
411
|
for (const processInstance of this.processes.values()) {
|
|
@@ -291,12 +430,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
291
430
|
* @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
|
|
292
431
|
*/
|
|
293
432
|
releaseProcessStartOrder() {
|
|
294
|
-
|
|
295
|
-
const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
|
|
296
|
-
const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
|
|
297
|
-
const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
|
|
298
|
-
|
|
299
|
-
return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
|
|
433
|
+
return processStartOrder(this.config.processes)
|
|
300
434
|
}
|
|
301
435
|
|
|
302
436
|
/** @returns {void} Marks this release active. */
|
|
@@ -367,24 +501,25 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
367
501
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
368
502
|
this.assertCommittedGenerationRecoverable()
|
|
369
503
|
this.state = "starting"
|
|
504
|
+
this.startupStartedAtMs = Date.now()
|
|
370
505
|
try {
|
|
371
506
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
507
|
+
await this.waitForConfiguredDependencies(processConfig)
|
|
372
508
|
const instances = this.getProcesses(processConfig.id)
|
|
373
509
|
|
|
374
510
|
if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
|
|
375
|
-
for (const {process} of instances) {
|
|
511
|
+
for (const {id, process} of instances) {
|
|
376
512
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
377
513
|
await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
|
|
514
|
+
this.processLaunchAtMs.set(id, Date.now())
|
|
515
|
+
this.startedProcessIds.add(id)
|
|
378
516
|
}
|
|
379
517
|
|
|
380
518
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
381
|
-
await
|
|
382
|
-
health: processConfig.health,
|
|
383
|
-
host: this.config.proxy.upstreamHost,
|
|
384
|
-
port: this.ports[processConfig.id]
|
|
385
|
-
})
|
|
519
|
+
await this.waitForProxiedHealth(processConfig)
|
|
386
520
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
387
521
|
}
|
|
522
|
+
for (const {id} of instances) this.readyProcessIds.add(id)
|
|
388
523
|
}
|
|
389
524
|
} catch (error) {
|
|
390
525
|
await this.abortCommittedGenerationRestart()
|
|
@@ -587,6 +587,59 @@ test("validateConfig defaults health.startDelayMs to 0, accepts an override, and
|
|
|
587
587
|
expect(negative.issues.some((issue) => issue.message === "processes[0].health.startDelayMs must be a non-negative number")).toBeTruthy()
|
|
588
588
|
})
|
|
589
589
|
|
|
590
|
+
test("validateConfig defaults dependsOn and its own wait budget, and rejects dependencies it cannot wait on", () => {
|
|
591
|
+
/**
|
|
592
|
+
* @param {import("../src/json.js").JsonValue} processes - Processes under test.
|
|
593
|
+
* @returns {{config: import("../src/config.js").RollbridgeConfig, issues: import("../src/config.js").ConfigIssue[]}} Validation result.
|
|
594
|
+
*/
|
|
595
|
+
const validateProcesses = (processes) => validateConfig({
|
|
596
|
+
application: "demo",
|
|
597
|
+
control: {path: "/tmp/demo.sock"},
|
|
598
|
+
processes,
|
|
599
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
600
|
+
})
|
|
601
|
+
|
|
602
|
+
const base = [
|
|
603
|
+
{command: "run beacon", id: "beacon", policy: "service", port: 7330},
|
|
604
|
+
{command: "run jobs", dependsOn: ["beacon"], id: "jobs", policy: "service", port: {from: 7331, to: 7399}},
|
|
605
|
+
{command: "run web", dependsOn: ["jobs"], id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
|
|
606
|
+
]
|
|
607
|
+
const valid = validateProcesses(base)
|
|
608
|
+
|
|
609
|
+
expect(valid.issues).toEqual([])
|
|
610
|
+
expect(valid.config.processes[0].dependsOn).toEqual([])
|
|
611
|
+
expect(valid.config.processes[1].dependsOn).toEqual(["beacon"])
|
|
612
|
+
expect(valid.config.processes[1].dependencyTimeoutMs).toBe(30000)
|
|
613
|
+
|
|
614
|
+
const custom = validateProcesses([...base.slice(0, 2), {...base[2], dependencyTimeoutMs: 60000}])
|
|
615
|
+
|
|
616
|
+
expect(custom.issues).toEqual([])
|
|
617
|
+
expect(custom.config.processes[2].dependencyTimeoutMs).toBe(60000)
|
|
618
|
+
|
|
619
|
+
const invalid = validateProcesses([
|
|
620
|
+
{command: "run web", dependsOn: ["missing", "web", "jobs", "worker"], dependencyTimeoutMs: 0, id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
621
|
+
{command: "run jobs", id: "jobs", policy: "singleton"},
|
|
622
|
+
{command: "run worker", id: "worker", policy: "companion"}
|
|
623
|
+
])
|
|
624
|
+
const messages = invalid.issues.map((issue) => issue.message)
|
|
625
|
+
|
|
626
|
+
expect(messages.some((message) => message === 'Process "web" depends on unknown process "missing"')).toBeTruthy()
|
|
627
|
+
expect(messages.some((message) => message === 'Process "web" cannot depend on itself')).toBeTruthy()
|
|
628
|
+
expect(messages.some((message) => message === 'Process "web" cannot depend on singleton "jobs"')).toBeTruthy()
|
|
629
|
+
expect(messages.some((message) => message === 'Process "worker" has no port, so "web" cannot wait for it')).toBeTruthy()
|
|
630
|
+
expect(messages.some((message) => message === "processes[0].dependencyTimeoutMs must be a positive number")).toBeTruthy()
|
|
631
|
+
|
|
632
|
+
const backwards = validateProcesses([
|
|
633
|
+
{command: "run web", dependsOn: "web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
634
|
+
{command: "run worker", dependsOn: ["web"], id: "worker", policy: "companion", port: 7001}
|
|
635
|
+
])
|
|
636
|
+
const backwardsMessages = backwards.issues.map((issue) => issue.message)
|
|
637
|
+
|
|
638
|
+
expect(backwardsMessages.some((message) => message === "processes[0].dependsOn must be an array")).toBeTruthy()
|
|
639
|
+
expect(backwardsMessages.some((message) => message === 'Process "worker" depends on "web", which Rollbridge starts after it')).toBeTruthy()
|
|
640
|
+
expect(backwards.issues.every((issue) => typeof issue.fix === "string" && issue.fix.length > 0)).toBe(true)
|
|
641
|
+
})
|
|
642
|
+
|
|
590
643
|
test("validateConfig defaults releaseRetention, accepts overrides, and rejects bad values", () => {
|
|
591
644
|
/**
|
|
592
645
|
* @param {import("../src/json.js").JsonValue} releaseRetention - Retention config under test, or undefined.
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import http from "node:http"
|
|
4
|
+
import net from "node:net"
|
|
5
|
+
|
|
6
|
+
const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
|
|
7
|
+
const bootMs = Number(process.env.ROLLBRIDGE_BOOT_MS || "0")
|
|
8
|
+
const dependencyPort = process.env.ROLLBRIDGE_DEPENDENCY_PORT
|
|
9
|
+
const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
|
|
10
|
+
|
|
11
|
+
log(`starting (boot budget ${bootMs}ms)`)
|
|
12
|
+
|
|
13
|
+
if (dependencyPort) {
|
|
14
|
+
// The app performs its own dependency wait, exactly as the production web command does.
|
|
15
|
+
await waitForDependency(Number(dependencyPort))
|
|
16
|
+
log(`dependency ${dependencyPort} accepted a connection`)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
await sleep(bootMs)
|
|
20
|
+
|
|
21
|
+
const server = http.createServer((request, response) => {
|
|
22
|
+
if (request.url === "/ping") {
|
|
23
|
+
response.writeHead(200, {"Content-Type": "application/json"})
|
|
24
|
+
response.end(JSON.stringify({message: "Pong", releaseId}))
|
|
25
|
+
return
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
|
|
29
|
+
response.end(`${releaseId}\n`)
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
process.on("SIGTERM", () => {
|
|
33
|
+
server.close(() => process.exit(0))
|
|
34
|
+
setTimeout(() => process.exit(0), 10).unref()
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
server.listen(port, "127.0.0.1")
|
|
38
|
+
log(`listening on ${port}`)
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* @param {number} dependencyPortNumber - Port of the process this app depends on.
|
|
42
|
+
* @returns {Promise<void>} Resolves once the dependency accepts a connection.
|
|
43
|
+
*/
|
|
44
|
+
async function waitForDependency(dependencyPortNumber) {
|
|
45
|
+
const deadline = Date.now() + 10000
|
|
46
|
+
|
|
47
|
+
while (Date.now() < deadline) {
|
|
48
|
+
if (await tryConnect(dependencyPortNumber)) return
|
|
49
|
+
await sleep(50)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
log(`dependency ${dependencyPortNumber} never accepted a connection`)
|
|
53
|
+
process.exit(21)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @param {number} dependencyPortNumber - Port to connect to.
|
|
58
|
+
* @returns {Promise<boolean>} True when the dependency accepted the connection.
|
|
59
|
+
*/
|
|
60
|
+
function tryConnect(dependencyPortNumber) {
|
|
61
|
+
return new Promise((resolve) => {
|
|
62
|
+
const socket = net.createConnection({host: "127.0.0.1", port: dependencyPortNumber})
|
|
63
|
+
let settled = false
|
|
64
|
+
|
|
65
|
+
/** @param {boolean} connected - Whether the connection succeeded. */
|
|
66
|
+
const finish = (connected) => {
|
|
67
|
+
if (settled) return
|
|
68
|
+
settled = true
|
|
69
|
+
socket.destroy()
|
|
70
|
+
resolve(connected)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
socket.setTimeout(200)
|
|
74
|
+
socket.once("connect", () => finish(true))
|
|
75
|
+
socket.once("error", () => finish(false))
|
|
76
|
+
socket.once("timeout", () => finish(false))
|
|
77
|
+
})
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* @param {number} ms - Milliseconds.
|
|
82
|
+
* @returns {Promise<void>} Resolves after the delay.
|
|
83
|
+
*/
|
|
84
|
+
function sleep(ms) {
|
|
85
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* @param {string} line - Output line, so the failure record can name the app's last output.
|
|
90
|
+
* @returns {void}
|
|
91
|
+
*/
|
|
92
|
+
function log(line) {
|
|
93
|
+
console.log(`boot: ${process.pid} ${line}`)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* @param {string} key - Environment variable name.
|
|
98
|
+
* @returns {string} Environment variable value.
|
|
99
|
+
*/
|
|
100
|
+
function requiredEnv(key) {
|
|
101
|
+
const value = process.env[key]
|
|
102
|
+
|
|
103
|
+
if (!value) throw new Error(`${key} is required`)
|
|
104
|
+
|
|
105
|
+
return value
|
|
106
|
+
}
|