rollbridge 0.1.59 → 0.1.61

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,8 +4,11 @@ import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
5
  import ManagedProcess from "./managed-process.js"
6
6
  import {findAvailablePort} from "./port-allocator.js"
7
+ import {processStartOrder} from "./process-order.js"
7
8
  import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
8
- import {waitForHealth} from "./health.js"
9
+ import {waitForHealth, waitForListeningPort} from "./health.js"
10
+
11
+ const DEPENDENCY_READY_INTERVAL_MS = 250
9
12
 
10
13
  /**
11
14
  * @typedef {import("./json.js").JsonValue} JsonValue
@@ -13,6 +16,7 @@ import {waitForHealth} from "./health.js"
13
16
  * @typedef {{http: number, websocket: number}} ReleaseConnections
14
17
  * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
15
18
  * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
19
+ * @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
16
20
  */
17
21
 
18
22
 
@@ -57,6 +61,14 @@ function envId(id) {
57
61
  return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
58
62
  }
59
63
 
64
+ /**
65
+ * @param {number | undefined} startedAtMs - When the measured phase began.
66
+ * @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
67
+ */
68
+ function elapsedSince(startedAtMs) {
69
+ return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
70
+ }
71
+
60
72
  /**
61
73
  * @param {import("./config.js").ProcessConfig} processConfig - Process config.
62
74
  * @param {number} index - Zero-based replica index.
@@ -107,16 +119,27 @@ export default class ReleaseGroup extends EventEmitter {
107
119
  this.stoppedAt = /** @type {string | undefined} */ (undefined)
108
120
  this.retirementError = /** @type {string | undefined} */ (undefined)
109
121
  this.preserveConfigOnRetirement = false
122
+ this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
123
+ this.readyProcessIds = /** @type {Set<string>} */ (new Set())
124
+ this.startedProcessIds = /** @type {Set<string>} */ (new Set())
125
+ this.startupDependencyWaitMs = 0
126
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
127
+ this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
110
128
  }
111
129
 
112
130
  /** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
113
131
  async start() {
114
132
  this.state = "starting"
133
+ this.startupStartedAtMs = Date.now()
115
134
 
116
135
  try {
117
136
  await this.allocatePorts()
118
137
 
119
138
  for (const processConfig of this.releaseProcessStartOrder()) {
139
+ await this.waitForConfiguredDependencies(processConfig)
140
+ /** @type {string[]} */
141
+ const instanceIds = []
142
+
120
143
  for (let index = 0; index < processConfig.replicas; index += 1) {
121
144
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
122
145
 
@@ -127,17 +150,17 @@ export default class ReleaseGroup extends EventEmitter {
127
150
  if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
128
151
  if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
129
152
  await processInstance.start("deploy")
153
+ this.processLaunchAtMs.set(instanceId, Date.now())
154
+ this.startedProcessIds.add(instanceId)
155
+ instanceIds.push(instanceId)
130
156
  }
131
157
 
132
158
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
133
- await waitForHealth({
134
- health: processConfig.health,
135
- host: this.config.proxy.upstreamHost,
136
- port: this.ports[processConfig.id]
137
- })
138
-
139
- if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
159
+ await this.waitForProxiedHealth(processConfig)
140
160
  }
161
+
162
+ for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
163
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
141
164
  }
142
165
  } catch (error) {
143
166
  this.state = "failed"
@@ -257,16 +280,132 @@ export default class ReleaseGroup extends EventEmitter {
257
280
  }
258
281
 
259
282
  /**
260
- * Logs process diagnostics around failed startup cleanup.
283
+ * Waits for the ports a process declares it depends on, on a budget that is not the
284
+ * health budget. The dependency leg of a start is not app boot, and charging it to the
285
+ * health deadline measures the dependency instead of the app.
286
+ * @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
287
+ * @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
288
+ */
289
+ async waitForConfiguredDependencies(processConfig) {
290
+ if (processConfig.dependsOn.length === 0) return
291
+
292
+ for (const dependencyId of processConfig.dependsOn) {
293
+ const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
294
+
295
+ if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
296
+
297
+ this.logger("release process dependency wait", {
298
+ dependencyPort: port,
299
+ dependencyProcessId: dependencyId,
300
+ processId: processConfig.id,
301
+ releaseId: this.releaseId,
302
+ timeoutMs: processConfig.dependencyTimeoutMs
303
+ })
304
+
305
+ const waitStartedAtMs = Date.now()
306
+
307
+ try {
308
+ const {attempts, elapsedMs} = await waitForListeningPort({
309
+ host: this.config.proxy.upstreamHost,
310
+ intervalMs: DEPENDENCY_READY_INTERVAL_MS,
311
+ port,
312
+ timeoutMs: processConfig.dependencyTimeoutMs
313
+ })
314
+
315
+ this.startupDependencyWaitMs += elapsedMs
316
+ this.logger("release process dependency ready", {
317
+ attempts,
318
+ dependencyPort: port,
319
+ dependencyProcessId: dependencyId,
320
+ elapsedMs,
321
+ processId: processConfig.id,
322
+ releaseId: this.releaseId
323
+ })
324
+ } catch (error) {
325
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
326
+ deadlineMs: processConfig.dependencyTimeoutMs,
327
+ dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
328
+ elapsedMs: elapsedSince(this.startupStartedAtMs),
329
+ failedProcessId: processConfig.id,
330
+ probes: undefined,
331
+ stage: "dependency"
332
+ })
333
+ throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
334
+ }
335
+
336
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
337
+ }
338
+ }
339
+
340
+ /**
341
+ * Health checks the proxied process on its own boot budget and records what the boot cost.
342
+ * @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
343
+ * @returns {Promise<void>} Resolves once the process answers its health path.
344
+ */
345
+ async waitForProxiedHealth(processConfig) {
346
+ const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
347
+ const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
348
+
349
+ try {
350
+ const {elapsedMs} = await waitForHealth({
351
+ health,
352
+ host: this.config.proxy.upstreamHost,
353
+ port: /** @type {number} */ (this.ports[processConfig.id]),
354
+ processId: processConfig.id
355
+ })
356
+
357
+ this.logger("release candidate health ready", {
358
+ bootMs: Date.now() - launchedAtMs,
359
+ deadlineMs: health.timeoutMs,
360
+ healthGateMs: elapsedMs,
361
+ path: health.path,
362
+ port: this.ports[processConfig.id],
363
+ processId: processConfig.id,
364
+ releaseId: this.releaseId,
365
+ waitedForDependenciesMs: this.startupDependencyWaitMs
366
+ })
367
+ } catch (error) {
368
+ const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
369
+
370
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
371
+ deadlineMs: health.timeoutMs,
372
+ dependencyWaitMs: this.startupDependencyWaitMs,
373
+ elapsedMs: elapsedSince(this.startupStartedAtMs),
374
+ failedProcessId: processConfig.id,
375
+ probes: healthCheck?.attempts,
376
+ stage: "health"
377
+ })
378
+ throw error
379
+ }
380
+ }
381
+
382
+ /**
383
+ * Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
261
384
  * @param {Error | string} error - Startup failure.
262
385
  * @param {{phase: string}} options - Diagnostic phase.
263
386
  * @returns {void}
264
387
  */
265
388
  logStartupFailure(error, {phase}) {
389
+ const diagnostics = this.startupFailureDiagnostics
390
+ const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
391
+ const lastLog = failedLogs?.at(-1)
392
+ const readyProcesses = [...this.readyProcessIds]
393
+
266
394
  this.logger("release startup failed", {
395
+ deadlineMs: diagnostics?.deadlineMs ?? null,
396
+ dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
397
+ elapsedMs: diagnostics?.elapsedMs ?? null,
267
398
  error: error instanceof Error ? error.message : error,
399
+ failedProcessId: diagnostics?.failedProcessId ?? null,
400
+ lastOutput: lastLog?.line ?? null,
401
+ lastOutputAt: lastLog?.at ?? null,
268
402
  phase,
269
- releaseId: this.releaseId
403
+ probes: diagnostics?.probes ?? null,
404
+ readyProcesses,
405
+ releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
406
+ releaseId: this.releaseId,
407
+ stage: diagnostics?.stage ?? null,
408
+ unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
270
409
  })
271
410
 
272
411
  for (const processInstance of this.processes.values()) {
@@ -291,12 +430,7 @@ export default class ReleaseGroup extends EventEmitter {
291
430
  * @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
292
431
  */
293
432
  releaseProcessStartOrder() {
294
- const releaseProcesses = this.config.processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
295
- const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
296
- const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
297
- const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
298
-
299
- return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
433
+ return processStartOrder(this.config.processes)
300
434
  }
301
435
 
302
436
  /** @returns {void} Marks this release active. */
@@ -367,24 +501,25 @@ export default class ReleaseGroup extends EventEmitter {
367
501
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
368
502
  this.assertCommittedGenerationRecoverable()
369
503
  this.state = "starting"
504
+ this.startupStartedAtMs = Date.now()
370
505
  try {
371
506
  for (const processConfig of this.releaseProcessStartOrder()) {
507
+ await this.waitForConfiguredDependencies(processConfig)
372
508
  const instances = this.getProcesses(processConfig.id)
373
509
 
374
510
  if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
375
- for (const {process} of instances) {
511
+ for (const {id, process} of instances) {
376
512
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
377
513
  await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
514
+ this.processLaunchAtMs.set(id, Date.now())
515
+ this.startedProcessIds.add(id)
378
516
  }
379
517
 
380
518
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
381
- await waitForHealth({
382
- health: processConfig.health,
383
- host: this.config.proxy.upstreamHost,
384
- port: this.ports[processConfig.id]
385
- })
519
+ await this.waitForProxiedHealth(processConfig)
386
520
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
387
521
  }
522
+ for (const {id} of instances) this.readyProcessIds.add(id)
388
523
  }
389
524
  } catch (error) {
390
525
  await this.abortCommittedGenerationRestart()
package/tensorbuzz.yml CHANGED
@@ -1,3 +1,20 @@
1
+ # TensorBuzz build config for Rollbridge.
2
+ #
3
+ # `paths` (STOE-1179): this gate installs the dependency tree and runs typecheck, ESLint and the
4
+ # whole Velocious suite, and every one of those reads code, `examples/`, and the package/tooling
5
+ # manifests — no gate input is Markdown. A delivery that changes nothing but Markdown therefore
6
+ # has nothing for this lane to check, and TensorBuzz records it as a terminal `skipped` build
7
+ # instead of spending a container and ~3 minutes on it. `test/tensorbuzz-config.test.js` asserts
8
+ # the split against every tracked path so a later edit cannot carve a gate input out of the lane.
9
+ #
10
+ # Ordered rules, last match wins, so the negation has to come after `"**"`. Quote every negation:
11
+ # an unquoted `!x` is a YAML tag, and the config validator fails the whole file rather than
12
+ # filtering anything. The lane still runs whenever `tensorbuzz.yml` itself changes, whenever the
13
+ # changed-file list is missing, empty, or capped by the host, and on every manual rebuild.
14
+ paths:
15
+ - "**"
16
+ - "!**/*.md"
17
+
1
18
  before_script:
2
19
  - npm install
3
20
  script:
@@ -587,6 +587,108 @@ test("validateConfig defaults health.startDelayMs to 0, accepts an override, and
587
587
  expect(negative.issues.some((issue) => issue.message === "processes[0].health.startDelayMs must be a non-negative number")).toBeTruthy()
588
588
  })
589
589
 
590
+ test("validateConfig gives an unconfigured proxied process the measured production boot budget", () => {
591
+ /**
592
+ * @param {Record<string, import("../src/json.js").JsonValue>} [processConfig] - Extra keys for the proxied process.
593
+ * @returns {import("../src/config.js").RollbridgeConfig} Normalized config.
594
+ */
595
+ const normalized = (processConfig = {}) => validateConfig({
596
+ application: "demo",
597
+ control: {path: "/tmp/demo.sock"},
598
+ processes: [
599
+ {command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}, ...processConfig}
600
+ ],
601
+ proxy: {host: "127.0.0.1", port: 8182}
602
+ }).config
603
+
604
+ const defaults = normalized()
605
+
606
+ expect(defaults.proxy.healthTimeoutMs).toBe(120000)
607
+ // The per-process gate inherits the release budget, so a project that configures no health
608
+ // timeout at all gets a deadline above the TensorBuzz boot measured on 2026-11-06 instead of
609
+ // the 30000ms that killed two production deploys.
610
+ expect(defaults.processes[0].health?.timeoutMs).toBe(120000)
611
+ expect(defaults.processes[0].health?.intervalMs).toBe(250)
612
+
613
+ const releaseBudget = normalized({health: {path: "/healthz"}})
614
+
615
+ expect(releaseBudget.proxy.healthTimeoutMs).toBe(120000)
616
+ expect(releaseBudget.processes[0].health?.path).toBe("/healthz")
617
+ expect(releaseBudget.processes[0].health?.timeoutMs).toBe(120000)
618
+
619
+ // Both configuration levels still win over the default: the release-level budget and the
620
+ // per-process budget, which is what a project with a known boot time sets.
621
+ const configuredProxy = validateConfig({
622
+ application: "demo",
623
+ control: {path: "/tmp/demo.sock"},
624
+ processes: [
625
+ {command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
626
+ ],
627
+ proxy: {healthTimeoutMs: 45000, host: "127.0.0.1", port: 8182}
628
+ }).config
629
+
630
+ expect(configuredProxy.proxy.healthTimeoutMs).toBe(45000)
631
+ expect(configuredProxy.processes[0].health?.timeoutMs).toBe(45000)
632
+
633
+ const perProcess = normalized({health: {path: "/ping", timeoutMs: 900}})
634
+
635
+ expect(perProcess.proxy.healthTimeoutMs).toBe(120000)
636
+ expect(perProcess.processes[0].health?.timeoutMs).toBe(900)
637
+ })
638
+
639
+ test("validateConfig defaults dependsOn and its own wait budget, and rejects dependencies it cannot wait on", () => {
640
+ /**
641
+ * @param {import("../src/json.js").JsonValue} processes - Processes under test.
642
+ * @returns {{config: import("../src/config.js").RollbridgeConfig, issues: import("../src/config.js").ConfigIssue[]}} Validation result.
643
+ */
644
+ const validateProcesses = (processes) => validateConfig({
645
+ application: "demo",
646
+ control: {path: "/tmp/demo.sock"},
647
+ processes,
648
+ proxy: {host: "127.0.0.1", port: 8182}
649
+ })
650
+
651
+ const base = [
652
+ {command: "run beacon", id: "beacon", policy: "service", port: 7330},
653
+ {command: "run jobs", dependsOn: ["beacon"], id: "jobs", policy: "service", port: {from: 7331, to: 7399}},
654
+ {command: "run web", dependsOn: ["jobs"], id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
655
+ ]
656
+ const valid = validateProcesses(base)
657
+
658
+ expect(valid.issues).toEqual([])
659
+ expect(valid.config.processes[0].dependsOn).toEqual([])
660
+ expect(valid.config.processes[1].dependsOn).toEqual(["beacon"])
661
+ expect(valid.config.processes[1].dependencyTimeoutMs).toBe(30000)
662
+
663
+ const custom = validateProcesses([...base.slice(0, 2), {...base[2], dependencyTimeoutMs: 60000}])
664
+
665
+ expect(custom.issues).toEqual([])
666
+ expect(custom.config.processes[2].dependencyTimeoutMs).toBe(60000)
667
+
668
+ const invalid = validateProcesses([
669
+ {command: "run web", dependsOn: ["missing", "web", "jobs", "worker"], dependencyTimeoutMs: 0, id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
670
+ {command: "run jobs", id: "jobs", policy: "singleton"},
671
+ {command: "run worker", id: "worker", policy: "companion"}
672
+ ])
673
+ const messages = invalid.issues.map((issue) => issue.message)
674
+
675
+ expect(messages.some((message) => message === 'Process "web" depends on unknown process "missing"')).toBeTruthy()
676
+ expect(messages.some((message) => message === 'Process "web" cannot depend on itself')).toBeTruthy()
677
+ expect(messages.some((message) => message === 'Process "web" cannot depend on singleton "jobs"')).toBeTruthy()
678
+ expect(messages.some((message) => message === 'Process "worker" has no port, so "web" cannot wait for it')).toBeTruthy()
679
+ expect(messages.some((message) => message === "processes[0].dependencyTimeoutMs must be a positive number")).toBeTruthy()
680
+
681
+ const backwards = validateProcesses([
682
+ {command: "run web", dependsOn: "web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
683
+ {command: "run worker", dependsOn: ["web"], id: "worker", policy: "companion", port: 7001}
684
+ ])
685
+ const backwardsMessages = backwards.issues.map((issue) => issue.message)
686
+
687
+ expect(backwardsMessages.some((message) => message === "processes[0].dependsOn must be an array")).toBeTruthy()
688
+ expect(backwardsMessages.some((message) => message === 'Process "worker" depends on "web", which Rollbridge starts after it')).toBeTruthy()
689
+ expect(backwards.issues.every((issue) => typeof issue.fix === "string" && issue.fix.length > 0)).toBe(true)
690
+ })
691
+
590
692
  test("validateConfig defaults releaseRetention, accepts overrides, and rejects bad values", () => {
591
693
  /**
592
694
  * @param {import("../src/json.js").JsonValue} releaseRetention - Retention config under test, or undefined.
@@ -0,0 +1,106 @@
1
+ // @ts-check
2
+
3
+ import http from "node:http"
4
+ import net from "node:net"
5
+
6
+ const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
7
+ const bootMs = Number(process.env.ROLLBRIDGE_BOOT_MS || "0")
8
+ const dependencyPort = process.env.ROLLBRIDGE_DEPENDENCY_PORT
9
+ const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
10
+
11
+ log(`starting (boot budget ${bootMs}ms)`)
12
+
13
+ if (dependencyPort) {
14
+ // The app performs its own dependency wait, exactly as the production web command does.
15
+ await waitForDependency(Number(dependencyPort))
16
+ log(`dependency ${dependencyPort} accepted a connection`)
17
+ }
18
+
19
+ await sleep(bootMs)
20
+
21
+ const server = http.createServer((request, response) => {
22
+ if (request.url === "/ping") {
23
+ response.writeHead(200, {"Content-Type": "application/json"})
24
+ response.end(JSON.stringify({message: "Pong", releaseId}))
25
+ return
26
+ }
27
+
28
+ response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
29
+ response.end(`${releaseId}\n`)
30
+ })
31
+
32
+ process.on("SIGTERM", () => {
33
+ server.close(() => process.exit(0))
34
+ setTimeout(() => process.exit(0), 10).unref()
35
+ })
36
+
37
+ server.listen(port, "127.0.0.1")
38
+ log(`listening on ${port}`)
39
+
40
+ /**
41
+ * @param {number} dependencyPortNumber - Port of the process this app depends on.
42
+ * @returns {Promise<void>} Resolves once the dependency accepts a connection.
43
+ */
44
+ async function waitForDependency(dependencyPortNumber) {
45
+ const deadline = Date.now() + 10000
46
+
47
+ while (Date.now() < deadline) {
48
+ if (await tryConnect(dependencyPortNumber)) return
49
+ await sleep(50)
50
+ }
51
+
52
+ log(`dependency ${dependencyPortNumber} never accepted a connection`)
53
+ process.exit(21)
54
+ }
55
+
56
+ /**
57
+ * @param {number} dependencyPortNumber - Port to connect to.
58
+ * @returns {Promise<boolean>} True when the dependency accepted the connection.
59
+ */
60
+ function tryConnect(dependencyPortNumber) {
61
+ return new Promise((resolve) => {
62
+ const socket = net.createConnection({host: "127.0.0.1", port: dependencyPortNumber})
63
+ let settled = false
64
+
65
+ /** @param {boolean} connected - Whether the connection succeeded. */
66
+ const finish = (connected) => {
67
+ if (settled) return
68
+ settled = true
69
+ socket.destroy()
70
+ resolve(connected)
71
+ }
72
+
73
+ socket.setTimeout(200)
74
+ socket.once("connect", () => finish(true))
75
+ socket.once("error", () => finish(false))
76
+ socket.once("timeout", () => finish(false))
77
+ })
78
+ }
79
+
80
+ /**
81
+ * @param {number} ms - Milliseconds.
82
+ * @returns {Promise<void>} Resolves after the delay.
83
+ */
84
+ function sleep(ms) {
85
+ return new Promise((resolve) => setTimeout(resolve, ms))
86
+ }
87
+
88
+ /**
89
+ * @param {string} line - Output line, so the failure record can name the app's last output.
90
+ * @returns {void}
91
+ */
92
+ function log(line) {
93
+ console.log(`boot: ${process.pid} ${line}`)
94
+ }
95
+
96
+ /**
97
+ * @param {string} key - Environment variable name.
98
+ * @returns {string} Environment variable value.
99
+ */
100
+ function requiredEnv(key) {
101
+ const value = process.env[key]
102
+
103
+ if (!value) throw new Error(`${key} is required`)
104
+
105
+ return value
106
+ }
@@ -1,8 +1,9 @@
1
1
  // @ts-check
2
2
 
3
3
  import http from "node:http"
4
+ import net from "node:net"
4
5
  import {describe, expect, test} from "@velocious/testing"
5
- import {waitForHealth} from "../src/health.js"
6
+ import {waitForHealth, waitForListeningPort} from "../src/health.js"
6
7
 
7
8
  describe("health", () => {
8
9
 
@@ -31,6 +32,18 @@ async function startHealthServer() {
31
32
  }
32
33
  }
33
34
 
35
+ /**
36
+ * @param {net.Server} server - Server to bind.
37
+ * @returns {Promise<number>} Ephemeral port it bound.
38
+ */
39
+ async function bindPort(server) {
40
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(undefined)))
41
+
42
+ const address = server.address()
43
+
44
+ return address && typeof address === "object" ? address.port : 0
45
+ }
46
+
34
47
  test("waitForHealth delays the first probe by startDelayMs", async () => {
35
48
  const server = await startHealthServer()
36
49
 
@@ -62,4 +75,64 @@ test("waitForHealth probes immediately when startDelayMs is 0", async () => {
62
75
  await server.close()
63
76
  }
64
77
  })
78
+
79
+ test("waitForHealth names the deadline and the time it spent when the app never becomes healthy", async () => {
80
+ const server = http.createServer((_request, response) => {
81
+ response.writeHead(500)
82
+ response.end("bad")
83
+ })
84
+
85
+ try {
86
+ const port = await bindPort(server)
87
+
88
+ /** @type {import("../src/health.js").HealthCheckError} */
89
+ const failure = /** @type {import("../src/health.js").HealthCheckError} */ (await waitForHealth({
90
+ health: {intervalMs: 25, path: "/ping", startDelayMs: 0, timeoutMs: 300},
91
+ host: "127.0.0.1",
92
+ port,
93
+ processId: "web"
94
+ }).catch((error) => error instanceof Error ? error : new Error(String(error))))
95
+
96
+ expect(String(failure.message)).toMatch(/process web: \d+ probes, \d+ms elapsed of the 300ms health deadline/)
97
+ expect(failure.healthCheck?.deadlineMs).toBe(300)
98
+ expect({value: Boolean(failure.healthCheck && failure.healthCheck.elapsedMs >= 300), context: `expected the health gate to have spent its whole budget, was ${failure.healthCheck?.elapsedMs}ms`}).toMatchObject({value: true})
99
+ expect({value: Boolean(failure.healthCheck && failure.healthCheck.attempts >= 1), context: `expected at least one probe, was ${failure.healthCheck?.attempts}`}).toMatchObject({value: true})
100
+ expect(failure.healthCheck?.processId).toBe("web")
101
+ } finally {
102
+ await server.close()
103
+ }
104
+ })
105
+
106
+ test("waitForListeningPort waits for a dependency port on a budget of its own", async () => {
107
+ const server = net.createServer()
108
+ const port = await bindPort(server)
109
+
110
+ await new Promise((resolve) => server.close(() => resolve(undefined)))
111
+
112
+ const waiting = waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 2000})
113
+
114
+ await new Promise((resolve) => setTimeout(resolve, 300))
115
+ await new Promise((resolve) => server.listen(port, "127.0.0.1", () => resolve(undefined)))
116
+
117
+ try {
118
+ const result = await waiting
119
+
120
+ expect({value: Boolean(result.attempts >= 2), context: `expected more than one connection attempt, was ${result.attempts}`}).toMatchObject({value: true})
121
+ expect({value: Boolean(result.elapsedMs >= 250), context: `expected the dependency wait to take ~300ms, was ${result.elapsedMs}ms`}).toMatchObject({value: true})
122
+ } finally {
123
+ await new Promise((resolve) => server.close(() => resolve(undefined)))
124
+ }
125
+ })
126
+
127
+ test("waitForListeningPort names the budget it ran out on when the dependency never binds", async () => {
128
+ const server = net.createServer()
129
+ const port = await bindPort(server)
130
+
131
+ await new Promise((resolve) => server.close(() => resolve(undefined)))
132
+
133
+ /** @type {Error} */
134
+ const failure = /** @type {Error} */ (await waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 200}).catch((error) => error instanceof Error ? error : new Error(String(error))))
135
+
136
+ expect(String(failure.message)).toMatch(/did not accept a connection within 200ms/)
137
+ })
65
138
  })
@@ -22,6 +22,15 @@ const binPath = path.join(repoRoot, "bin", "rollbridge")
22
22
  const dummyAppPath = path.join(repoRoot, "test", "fixtures", "dummy-app.js")
23
23
  const legacyDaemonPath = path.join(repoRoot, "test", "fixtures", "pre-split3-daemon-runner.js")
24
24
  const partialGuardianPath = path.join(repoRoot, "test", "fixtures", "partial-owner-replacement-process-guardian.js")
25
+ // Budget for the retired-drain fixture below. It spawns six child processes (a daemon, a worker and
26
+ // a proxied web process for each of two releases, plus a partial owner-replacement guardian) and then
27
+ // waits for the incumbent's retired drain to resume, so its cost tracks host contention, not the
28
+ // behaviour under test. Measured on the same machine in the same invocation: 1.457s at load average
29
+ // 8.3 and 7.105s at load average 24.5, both with the drain completing. The previous 5000ms cap sat
30
+ // between those two numbers, which is why the spec reddened on a busy runner while nothing in the
31
+ // drain actually stalled. 20000ms is 2.8x the measured worst case; the wait stays event-gated, so a
32
+ // release that never resumes its drain still fails, and now fails naming the release record.
33
+ const PARTIAL_DRAIN_FIXTURE_TIMEOUT_MS = 20000
25
34
 
26
35
  test("partial owner-replacement guardian crosses the authenticated legacy bridge before retired-owner commit", async () => {
27
36
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-"))
@@ -324,7 +333,7 @@ test("partial guardian replacement persists the coordinator only after ownership
324
333
  }
325
334
  })
326
335
 
327
- test("failed partial upgrade resumes an incumbent retired release drain", {timeoutMs: 5000}, async () => {
336
+ test("failed partial upgrade resumes an incumbent retired release drain", {timeoutMs: PARTIAL_DRAIN_FIXTURE_TIMEOUT_MS}, async () => {
328
337
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-drain-"))
329
338
  const socketPath = path.join(root, "rollbridge.sock")
330
339
  const statePath = path.join(root, "state.json")
@@ -382,7 +391,18 @@ test("failed partial upgrade resumes an incumbent retired release drain", {timeo
382
391
  retainedConnection.destroy()
383
392
  await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
384
393
  await releaseDrained
385
- expect(releaseState(await sendControlCommand({command: {command: "status"}, path: socketPath}), "v1")).toBe("stopped")
394
+ const drainedStatus = await sendControlCommand({command: {command: "status"}, path: socketPath})
395
+
396
+ expect(releaseState(drainedStatus, "v1")).toBe("stopped")
397
+ // The resumed drain must also be visible in the retired release's own record, not only as the
398
+ // stdout line the wait is keyed on: a release cleared without draining its processes leaves no
399
+ // drainStartedAt/stoppedAt pair behind.
400
+ const drainedRelease = /** @type {{drainStartedAt?: string, releaseId: string, stoppedAt?: string}[]} */ (drainedStatus.releases)
401
+ .find(({releaseId}) => releaseId === "v1")
402
+
403
+ expect(typeof drainedRelease?.drainStartedAt).toBe("string")
404
+ expect(typeof drainedRelease?.stoppedAt).toBe("string")
405
+ expect(Date.parse(/** @type {string} */ (drainedRelease?.stoppedAt))).toBeGreaterThanOrEqual(Date.parse(/** @type {string} */ (drainedRelease?.drainStartedAt)))
386
406
  const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
387
407
 
388
408
  await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")