rollbridge 0.1.58 → 0.1.60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,6 +12,10 @@ import ManagedProcess from "./managed-process.js"
12
12
  import {writeState} from "./state-store.js"
13
13
 
14
14
  const OWNER_RESTART_RETRY_MS = 1000
15
+ // An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
16
+ // generation may legitimately drain for hours, but a mutation that holds the
17
+ // fence that long is stuck, so the lease is bounded and released on expiry.
18
+ const OWNER_MUTATION_LEASE_MS = 900000
15
19
 
16
20
  /** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
17
21
 
@@ -32,7 +36,9 @@ const OWNER_RESTART_RETRY_MS = 1000
32
36
  * @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
33
37
  * @property {string} [replacementId] - Prepared replacement transaction id.
34
38
  * @property {string} [mutationId] - Owner mutation lease id.
39
+ * @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
35
40
  * @property {string} [operation] - Owner mutation diagnostic name.
41
+ * @property {string} [waitingFor] - What the held owner mutation is blocked on.
36
42
  * @property {number} [ownerPid] - Exact claimant process PID.
37
43
  * @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
38
44
  * @property {string} [statePath] - Exact public recovery state path.
@@ -103,6 +109,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
103
109
  let ownerMutationClient
104
110
  /** @type {string | undefined} */
105
111
  let ownerMutationId
112
+ /** @type {string | undefined} */
113
+ let ownerMutationOperation
114
+ /** @type {string | undefined} */
115
+ let ownerMutationWaitingFor
116
+ /** @type {number | undefined} */
117
+ let ownerMutationPid
118
+ /** @type {number | undefined} */
119
+ let ownerMutationStartedAt
120
+ let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
121
+ /** @type {ReturnType<typeof setTimeout> | undefined} */
122
+ let ownerMutationLeaseTimer
123
+ /** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
124
+ let ownerMutationExpired
106
125
  /** @type {net.Socket | undefined} */
107
126
  let retiringClient
108
127
  /** @type {string | undefined} */
@@ -154,10 +173,7 @@ const server = net.createServer((socket) => {
154
173
  clearTimeout(waiter.timer)
155
174
  waiter.reject(new Error("Owner claimant disconnected"))
156
175
  }
157
- if (ownerMutationClient === socket && ownerClient !== socket) {
158
- ownerMutationClient = undefined
159
- ownerMutationId = undefined
160
- }
176
+ if (ownerMutationClient === socket && ownerClient !== socket) clearOwnerMutationLease()
161
177
  if (controlLessRetirement) {
162
178
  const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
163
179
 
@@ -301,8 +317,7 @@ async function execute(request, socket) {
301
317
  rememberDisconnectedOwnerSuccessors(socket)
302
318
  ownerClient = undefined
303
319
  ownerClientPid = undefined
304
- ownerMutationClient = undefined
305
- ownerMutationId = undefined
320
+ clearOwnerMutationLease()
306
321
  ownerRevision += 1
307
322
  grantNextOwner()
308
323
  return {retired: true}
@@ -397,26 +412,48 @@ async function execute(request, socket) {
397
412
  if (request.command === "begin-owner-mutation") {
398
413
  requireOwner(socket, request.command)
399
414
  if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
400
- if (ownerMutationClient) throw new Error("Another owner mutation is already in progress")
415
+ const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
416
+
417
+ if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
418
+ if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
401
419
  ownerMutationClient = socket
402
420
  ownerMutationId = crypto.randomUUID()
403
- return {mutationId: ownerMutationId}
421
+ ownerMutationOperation = request.operation
422
+ ownerMutationWaitingFor = request.waitingFor
423
+ ownerMutationPid = request.ownerPid ?? ownerClientPid
424
+ ownerMutationStartedAt = Date.now()
425
+ ownerMutationLeaseMs = leaseMs
426
+ armOwnerMutationLease(ownerMutationId)
427
+ return {leaseMs, mutationId: ownerMutationId}
404
428
  }
405
429
 
406
430
  if (request.command === "end-owner-mutation") {
407
431
  requireOwner(socket, request.command)
432
+ if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
433
+ const expired = ownerMutationExpired
434
+
435
+ ownerMutationExpired = undefined
436
+ throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
437
+ }
408
438
  if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
409
- ownerMutationClient = undefined
410
- ownerMutationId = undefined
439
+ clearOwnerMutationLease()
411
440
  return {ended: true}
412
441
  }
413
442
 
443
+ if (request.command === "report-owner-mutation-wait") {
444
+ requireOwner(socket, request.command)
445
+ if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
446
+ if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
447
+ ownerMutationWaitingFor = request.waitingFor.trim()
448
+ return {reported: true}
449
+ }
450
+
414
451
  if (request.command === "prepare-owner-replacement") {
415
452
  if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
416
453
  if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
417
454
  if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
418
455
  if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
419
- if (ownerMutationClient) throw new Error("Owner replacement cannot prepare while an owner mutation is in progress")
456
+ if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
420
457
  if (!ownerState) throw new Error("Committed owner has not published transferable state")
421
458
  const currentAuthority = ownerAuthority(ownerState)
422
459
  const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
@@ -715,6 +752,82 @@ async function execute(request, socket) {
715
752
  return status
716
753
  }
717
754
 
755
+ /**
756
+ * Names the mutation holding the owner-mutation fence: which mutation, which process,
757
+ * since when, and what it is blocked on. Refusals must be diagnosable without a live
758
+ * inspection of the daemon.
759
+ * @returns {string} Holder description used in refusal messages.
760
+ */
761
+ function describeOwnerMutationLease() {
762
+ const startedAt = ownerMutationStartedAt ?? 0
763
+ const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
764
+ const holderPid = ownerMutationPid ?? ownerClientPid
765
+
766
+ return [
767
+ `owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
768
+ `mutation ${ownerMutationId || "unknown"}`,
769
+ `held by process ${holderPid ?? "unknown"}`,
770
+ `since ${new Date(startedAt).toISOString()}`,
771
+ `${elapsedSeconds}s elapsed`,
772
+ `waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
773
+ ].join("; ")
774
+ }
775
+
776
+ /** @returns {void} */
777
+ function clearOwnerMutationLease() {
778
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
779
+ ownerMutationLeaseTimer = undefined
780
+ ownerMutationClient = undefined
781
+ ownerMutationId = undefined
782
+ ownerMutationOperation = undefined
783
+ ownerMutationWaitingFor = undefined
784
+ ownerMutationPid = undefined
785
+ ownerMutationStartedAt = undefined
786
+ ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
787
+ }
788
+
789
+ /**
790
+ * Arms the lease bound for one mutation.
791
+ * @param {string} mutationId - Lease id the timer belongs to.
792
+ * @returns {void}
793
+ */
794
+ function armOwnerMutationLease(mutationId) {
795
+ if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
796
+ ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
797
+ ownerMutationLeaseTimer.unref()
798
+ }
799
+
800
+ /**
801
+ * A mutation that outruns its lease is reported and released instead of holding the fence
802
+ * forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
803
+ * lease is gone when it tries to end it.
804
+ * @param {string} mutationId - Lease id the armed timer belongs to.
805
+ * @returns {void}
806
+ */
807
+ function expireOwnerMutationLease(mutationId) {
808
+ ownerMutationLeaseTimer = undefined
809
+ if (!ownerMutationClient || ownerMutationId !== mutationId) return
810
+ const operation = ownerMutationOperation || "unnamed operation"
811
+ const leaseMs = ownerMutationLeaseMs
812
+ const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
813
+ const diagnostic = {
814
+ code: "OWNER_MUTATION_LEASE_EXPIRED",
815
+ elapsedMs,
816
+ event: "owner-mutation-expired",
817
+ leaseMs,
818
+ mutationId,
819
+ operation,
820
+ pid: ownerMutationPid ?? ownerClientPid,
821
+ waitingFor: ownerMutationWaitingFor || "the operation to report completion"
822
+ }
823
+
824
+ process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
825
+ if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
826
+ broadcast(diagnostic)
827
+ ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
828
+ clearOwnerMutationLease()
829
+ }
830
+
718
831
  /**
719
832
  * @param {net.Socket} socket - Requesting client.
720
833
  * @param {string} command - Command name.
@@ -735,10 +848,7 @@ function releaseClosedOwner(socket) {
735
848
  rememberDisconnectedOwnerSuccessors(socket)
736
849
  ownerClient = undefined
737
850
  ownerClientPid = undefined
738
- if (ownerMutationClient === socket) {
739
- ownerMutationClient = undefined
740
- ownerMutationId = undefined
741
- }
851
+ if (ownerMutationClient === socket) clearOwnerMutationLease()
742
852
  if (replacementClient && replacementOwnerState) commitReplacement()
743
853
  else {
744
854
  if (incompleteRestartChild) {
@@ -0,0 +1,26 @@
1
+ // @ts-check
2
+
3
+ /**
4
+ * @typedef {import("./config.js").ProcessConfig} ProcessConfig
5
+ */
6
+
7
+ const POLICY_START_RANK = /** @type {Record<ProcessConfig["policy"], number>} */ ({companion: 1, proxied: 2, service: 0, singleton: 3})
8
+
9
+ /**
10
+ * Order in which Rollbridge starts processes, which is also the only order a `dependsOn`
11
+ * edge may point backwards in: daemon-wide persistent services are started before every
12
+ * release generation, then handoff services, companions, and the proxied process last.
13
+ * Config order is kept inside each group.
14
+ * @param {ProcessConfig[]} processes - Normalized processes.
15
+ * @param {{includePersistentServices?: boolean}} [options] - Whether to include the
16
+ * daemon-wide persistent services that a release group never starts itself.
17
+ * @returns {ProcessConfig[]} Ordered process configs.
18
+ */
19
+ export function processStartOrder(processes, {includePersistentServices = false} = {}) {
20
+ const persistentServices = includePersistentServices
21
+ ? processes.filter((processConfig) => processConfig.policy === "service" && processConfig.deployStrategy === "persistent")
22
+ : []
23
+ const generationProcesses = processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
24
+
25
+ return [...persistentServices, ...generationProcesses.sort((left, right) => POLICY_START_RANK[left.policy] - POLICY_START_RANK[right.policy])]
26
+ }
@@ -4,8 +4,11 @@ import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
5
  import ManagedProcess from "./managed-process.js"
6
6
  import {findAvailablePort} from "./port-allocator.js"
7
+ import {processStartOrder} from "./process-order.js"
7
8
  import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
8
- import {waitForHealth} from "./health.js"
9
+ import {waitForHealth, waitForListeningPort} from "./health.js"
10
+
11
+ const DEPENDENCY_READY_INTERVAL_MS = 250
9
12
 
10
13
  /**
11
14
  * @typedef {import("./json.js").JsonValue} JsonValue
@@ -13,6 +16,7 @@ import {waitForHealth} from "./health.js"
13
16
  * @typedef {{http: number, websocket: number}} ReleaseConnections
14
17
  * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
15
18
  * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
19
+ * @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
16
20
  */
17
21
 
18
22
 
@@ -57,6 +61,14 @@ function envId(id) {
57
61
  return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
58
62
  }
59
63
 
64
+ /**
65
+ * @param {number | undefined} startedAtMs - When the measured phase began.
66
+ * @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
67
+ */
68
+ function elapsedSince(startedAtMs) {
69
+ return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
70
+ }
71
+
60
72
  /**
61
73
  * @param {import("./config.js").ProcessConfig} processConfig - Process config.
62
74
  * @param {number} index - Zero-based replica index.
@@ -107,16 +119,27 @@ export default class ReleaseGroup extends EventEmitter {
107
119
  this.stoppedAt = /** @type {string | undefined} */ (undefined)
108
120
  this.retirementError = /** @type {string | undefined} */ (undefined)
109
121
  this.preserveConfigOnRetirement = false
122
+ this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
123
+ this.readyProcessIds = /** @type {Set<string>} */ (new Set())
124
+ this.startedProcessIds = /** @type {Set<string>} */ (new Set())
125
+ this.startupDependencyWaitMs = 0
126
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
127
+ this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
110
128
  }
111
129
 
112
130
  /** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
113
131
  async start() {
114
132
  this.state = "starting"
133
+ this.startupStartedAtMs = Date.now()
115
134
 
116
135
  try {
117
136
  await this.allocatePorts()
118
137
 
119
138
  for (const processConfig of this.releaseProcessStartOrder()) {
139
+ await this.waitForConfiguredDependencies(processConfig)
140
+ /** @type {string[]} */
141
+ const instanceIds = []
142
+
120
143
  for (let index = 0; index < processConfig.replicas; index += 1) {
121
144
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
122
145
 
@@ -127,17 +150,17 @@ export default class ReleaseGroup extends EventEmitter {
127
150
  if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
128
151
  if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
129
152
  await processInstance.start("deploy")
153
+ this.processLaunchAtMs.set(instanceId, Date.now())
154
+ this.startedProcessIds.add(instanceId)
155
+ instanceIds.push(instanceId)
130
156
  }
131
157
 
132
158
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
133
- await waitForHealth({
134
- health: processConfig.health,
135
- host: this.config.proxy.upstreamHost,
136
- port: this.ports[processConfig.id]
137
- })
138
-
139
- if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
159
+ await this.waitForProxiedHealth(processConfig)
140
160
  }
161
+
162
+ for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
163
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
141
164
  }
142
165
  } catch (error) {
143
166
  this.state = "failed"
@@ -257,16 +280,132 @@ export default class ReleaseGroup extends EventEmitter {
257
280
  }
258
281
 
259
282
  /**
260
- * Logs process diagnostics around failed startup cleanup.
283
+ * Waits for the ports a process declares it depends on, on a budget that is not the
284
+ * health budget. The dependency leg of a start is not app boot, and charging it to the
285
+ * health deadline measures the dependency instead of the app.
286
+ * @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
287
+ * @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
288
+ */
289
+ async waitForConfiguredDependencies(processConfig) {
290
+ if (processConfig.dependsOn.length === 0) return
291
+
292
+ for (const dependencyId of processConfig.dependsOn) {
293
+ const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
294
+
295
+ if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
296
+
297
+ this.logger("release process dependency wait", {
298
+ dependencyPort: port,
299
+ dependencyProcessId: dependencyId,
300
+ processId: processConfig.id,
301
+ releaseId: this.releaseId,
302
+ timeoutMs: processConfig.dependencyTimeoutMs
303
+ })
304
+
305
+ const waitStartedAtMs = Date.now()
306
+
307
+ try {
308
+ const {attempts, elapsedMs} = await waitForListeningPort({
309
+ host: this.config.proxy.upstreamHost,
310
+ intervalMs: DEPENDENCY_READY_INTERVAL_MS,
311
+ port,
312
+ timeoutMs: processConfig.dependencyTimeoutMs
313
+ })
314
+
315
+ this.startupDependencyWaitMs += elapsedMs
316
+ this.logger("release process dependency ready", {
317
+ attempts,
318
+ dependencyPort: port,
319
+ dependencyProcessId: dependencyId,
320
+ elapsedMs,
321
+ processId: processConfig.id,
322
+ releaseId: this.releaseId
323
+ })
324
+ } catch (error) {
325
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
326
+ deadlineMs: processConfig.dependencyTimeoutMs,
327
+ dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
328
+ elapsedMs: elapsedSince(this.startupStartedAtMs),
329
+ failedProcessId: processConfig.id,
330
+ probes: undefined,
331
+ stage: "dependency"
332
+ })
333
+ throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
334
+ }
335
+
336
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
337
+ }
338
+ }
339
+
340
+ /**
341
+ * Health checks the proxied process on its own boot budget and records what the boot cost.
342
+ * @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
343
+ * @returns {Promise<void>} Resolves once the process answers its health path.
344
+ */
345
+ async waitForProxiedHealth(processConfig) {
346
+ const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
347
+ const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
348
+
349
+ try {
350
+ const {elapsedMs} = await waitForHealth({
351
+ health,
352
+ host: this.config.proxy.upstreamHost,
353
+ port: /** @type {number} */ (this.ports[processConfig.id]),
354
+ processId: processConfig.id
355
+ })
356
+
357
+ this.logger("release candidate health ready", {
358
+ bootMs: Date.now() - launchedAtMs,
359
+ deadlineMs: health.timeoutMs,
360
+ healthGateMs: elapsedMs,
361
+ path: health.path,
362
+ port: this.ports[processConfig.id],
363
+ processId: processConfig.id,
364
+ releaseId: this.releaseId,
365
+ waitedForDependenciesMs: this.startupDependencyWaitMs
366
+ })
367
+ } catch (error) {
368
+ const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
369
+
370
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
371
+ deadlineMs: health.timeoutMs,
372
+ dependencyWaitMs: this.startupDependencyWaitMs,
373
+ elapsedMs: elapsedSince(this.startupStartedAtMs),
374
+ failedProcessId: processConfig.id,
375
+ probes: healthCheck?.attempts,
376
+ stage: "health"
377
+ })
378
+ throw error
379
+ }
380
+ }
381
+
382
+ /**
383
+ * Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
261
384
  * @param {Error | string} error - Startup failure.
262
385
  * @param {{phase: string}} options - Diagnostic phase.
263
386
  * @returns {void}
264
387
  */
265
388
  logStartupFailure(error, {phase}) {
389
+ const diagnostics = this.startupFailureDiagnostics
390
+ const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
391
+ const lastLog = failedLogs?.at(-1)
392
+ const readyProcesses = [...this.readyProcessIds]
393
+
266
394
  this.logger("release startup failed", {
395
+ deadlineMs: diagnostics?.deadlineMs ?? null,
396
+ dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
397
+ elapsedMs: diagnostics?.elapsedMs ?? null,
267
398
  error: error instanceof Error ? error.message : error,
399
+ failedProcessId: diagnostics?.failedProcessId ?? null,
400
+ lastOutput: lastLog?.line ?? null,
401
+ lastOutputAt: lastLog?.at ?? null,
268
402
  phase,
269
- releaseId: this.releaseId
403
+ probes: diagnostics?.probes ?? null,
404
+ readyProcesses,
405
+ releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
406
+ releaseId: this.releaseId,
407
+ stage: diagnostics?.stage ?? null,
408
+ unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
270
409
  })
271
410
 
272
411
  for (const processInstance of this.processes.values()) {
@@ -291,12 +430,7 @@ export default class ReleaseGroup extends EventEmitter {
291
430
  * @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
292
431
  */
293
432
  releaseProcessStartOrder() {
294
- const releaseProcesses = this.config.processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
295
- const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
296
- const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
297
- const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
298
-
299
- return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
433
+ return processStartOrder(this.config.processes)
300
434
  }
301
435
 
302
436
  /** @returns {void} Marks this release active. */
@@ -367,24 +501,25 @@ export default class ReleaseGroup extends EventEmitter {
367
501
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
368
502
  this.assertCommittedGenerationRecoverable()
369
503
  this.state = "starting"
504
+ this.startupStartedAtMs = Date.now()
370
505
  try {
371
506
  for (const processConfig of this.releaseProcessStartOrder()) {
507
+ await this.waitForConfiguredDependencies(processConfig)
372
508
  const instances = this.getProcesses(processConfig.id)
373
509
 
374
510
  if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
375
- for (const {process} of instances) {
511
+ for (const {id, process} of instances) {
376
512
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
377
513
  await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
514
+ this.processLaunchAtMs.set(id, Date.now())
515
+ this.startedProcessIds.add(id)
378
516
  }
379
517
 
380
518
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
381
- await waitForHealth({
382
- health: processConfig.health,
383
- host: this.config.proxy.upstreamHost,
384
- port: this.ports[processConfig.id]
385
- })
519
+ await this.waitForProxiedHealth(processConfig)
386
520
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
387
521
  }
522
+ for (const {id} of instances) this.readyProcessIds.add(id)
388
523
  }
389
524
  } catch (error) {
390
525
  await this.abortCommittedGenerationRestart()
@@ -587,6 +587,59 @@ test("validateConfig defaults health.startDelayMs to 0, accepts an override, and
587
587
  expect(negative.issues.some((issue) => issue.message === "processes[0].health.startDelayMs must be a non-negative number")).toBeTruthy()
588
588
  })
589
589
 
590
+ test("validateConfig defaults dependsOn and its own wait budget, and rejects dependencies it cannot wait on", () => {
591
+ /**
592
+ * @param {import("../src/json.js").JsonValue} processes - Processes under test.
593
+ * @returns {{config: import("../src/config.js").RollbridgeConfig, issues: import("../src/config.js").ConfigIssue[]}} Validation result.
594
+ */
595
+ const validateProcesses = (processes) => validateConfig({
596
+ application: "demo",
597
+ control: {path: "/tmp/demo.sock"},
598
+ processes,
599
+ proxy: {host: "127.0.0.1", port: 8182}
600
+ })
601
+
602
+ const base = [
603
+ {command: "run beacon", id: "beacon", policy: "service", port: 7330},
604
+ {command: "run jobs", dependsOn: ["beacon"], id: "jobs", policy: "service", port: {from: 7331, to: 7399}},
605
+ {command: "run web", dependsOn: ["jobs"], id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
606
+ ]
607
+ const valid = validateProcesses(base)
608
+
609
+ expect(valid.issues).toEqual([])
610
+ expect(valid.config.processes[0].dependsOn).toEqual([])
611
+ expect(valid.config.processes[1].dependsOn).toEqual(["beacon"])
612
+ expect(valid.config.processes[1].dependencyTimeoutMs).toBe(30000)
613
+
614
+ const custom = validateProcesses([...base.slice(0, 2), {...base[2], dependencyTimeoutMs: 60000}])
615
+
616
+ expect(custom.issues).toEqual([])
617
+ expect(custom.config.processes[2].dependencyTimeoutMs).toBe(60000)
618
+
619
+ const invalid = validateProcesses([
620
+ {command: "run web", dependsOn: ["missing", "web", "jobs", "worker"], dependencyTimeoutMs: 0, id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
621
+ {command: "run jobs", id: "jobs", policy: "singleton"},
622
+ {command: "run worker", id: "worker", policy: "companion"}
623
+ ])
624
+ const messages = invalid.issues.map((issue) => issue.message)
625
+
626
+ expect(messages.some((message) => message === 'Process "web" depends on unknown process "missing"')).toBeTruthy()
627
+ expect(messages.some((message) => message === 'Process "web" cannot depend on itself')).toBeTruthy()
628
+ expect(messages.some((message) => message === 'Process "web" cannot depend on singleton "jobs"')).toBeTruthy()
629
+ expect(messages.some((message) => message === 'Process "worker" has no port, so "web" cannot wait for it')).toBeTruthy()
630
+ expect(messages.some((message) => message === "processes[0].dependencyTimeoutMs must be a positive number")).toBeTruthy()
631
+
632
+ const backwards = validateProcesses([
633
+ {command: "run web", dependsOn: "web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
634
+ {command: "run worker", dependsOn: ["web"], id: "worker", policy: "companion", port: 7001}
635
+ ])
636
+ const backwardsMessages = backwards.issues.map((issue) => issue.message)
637
+
638
+ expect(backwardsMessages.some((message) => message === "processes[0].dependsOn must be an array")).toBeTruthy()
639
+ expect(backwardsMessages.some((message) => message === 'Process "worker" depends on "web", which Rollbridge starts after it')).toBeTruthy()
640
+ expect(backwards.issues.every((issue) => typeof issue.fix === "string" && issue.fix.length > 0)).toBe(true)
641
+ })
642
+
590
643
  test("validateConfig defaults releaseRetention, accepts overrides, and rejects bad values", () => {
591
644
  /**
592
645
  * @param {import("../src/json.js").JsonValue} releaseRetention - Retention config under test, or undefined.
@@ -0,0 +1,106 @@
1
+ // @ts-check
2
+
3
+ import http from "node:http"
4
+ import net from "node:net"
5
+
6
+ const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
7
+ const bootMs = Number(process.env.ROLLBRIDGE_BOOT_MS || "0")
8
+ const dependencyPort = process.env.ROLLBRIDGE_DEPENDENCY_PORT
9
+ const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
10
+
11
+ log(`starting (boot budget ${bootMs}ms)`)
12
+
13
+ if (dependencyPort) {
14
+ // The app performs its own dependency wait, exactly as the production web command does.
15
+ await waitForDependency(Number(dependencyPort))
16
+ log(`dependency ${dependencyPort} accepted a connection`)
17
+ }
18
+
19
+ await sleep(bootMs)
20
+
21
+ const server = http.createServer((request, response) => {
22
+ if (request.url === "/ping") {
23
+ response.writeHead(200, {"Content-Type": "application/json"})
24
+ response.end(JSON.stringify({message: "Pong", releaseId}))
25
+ return
26
+ }
27
+
28
+ response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
29
+ response.end(`${releaseId}\n`)
30
+ })
31
+
32
+ process.on("SIGTERM", () => {
33
+ server.close(() => process.exit(0))
34
+ setTimeout(() => process.exit(0), 10).unref()
35
+ })
36
+
37
+ server.listen(port, "127.0.0.1")
38
+ log(`listening on ${port}`)
39
+
40
+ /**
41
+ * @param {number} dependencyPortNumber - Port of the process this app depends on.
42
+ * @returns {Promise<void>} Resolves once the dependency accepts a connection.
43
+ */
44
+ async function waitForDependency(dependencyPortNumber) {
45
+ const deadline = Date.now() + 10000
46
+
47
+ while (Date.now() < deadline) {
48
+ if (await tryConnect(dependencyPortNumber)) return
49
+ await sleep(50)
50
+ }
51
+
52
+ log(`dependency ${dependencyPortNumber} never accepted a connection`)
53
+ process.exit(21)
54
+ }
55
+
56
+ /**
57
+ * @param {number} dependencyPortNumber - Port to connect to.
58
+ * @returns {Promise<boolean>} True when the dependency accepted the connection.
59
+ */
60
+ function tryConnect(dependencyPortNumber) {
61
+ return new Promise((resolve) => {
62
+ const socket = net.createConnection({host: "127.0.0.1", port: dependencyPortNumber})
63
+ let settled = false
64
+
65
+ /** @param {boolean} connected - Whether the connection succeeded. */
66
+ const finish = (connected) => {
67
+ if (settled) return
68
+ settled = true
69
+ socket.destroy()
70
+ resolve(connected)
71
+ }
72
+
73
+ socket.setTimeout(200)
74
+ socket.once("connect", () => finish(true))
75
+ socket.once("error", () => finish(false))
76
+ socket.once("timeout", () => finish(false))
77
+ })
78
+ }
79
+
80
+ /**
81
+ * @param {number} ms - Milliseconds.
82
+ * @returns {Promise<void>} Resolves after the delay.
83
+ */
84
+ function sleep(ms) {
85
+ return new Promise((resolve) => setTimeout(resolve, ms))
86
+ }
87
+
88
+ /**
89
+ * @param {string} line - Output line, so the failure record can name the app's last output.
90
+ * @returns {void}
91
+ */
92
+ function log(line) {
93
+ console.log(`boot: ${process.pid} ${line}`)
94
+ }
95
+
96
+ /**
97
+ * @param {string} key - Environment variable name.
98
+ * @returns {string} Environment variable value.
99
+ */
100
+ function requiredEnv(key) {
101
+ const value = process.env[key]
102
+
103
+ if (!value) throw new Error(`${key} is required`)
104
+
105
+ return value
106
+ }