rollbridge 0.1.59 → 0.1.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/docs/config.md +42 -2
- package/docs/troubleshooting.md +22 -6
- package/docs/velocious.md +1 -1
- package/examples/tensorbuzz.com.js +1 -1
- package/package.json +1 -1
- package/src/config.js +104 -3
- package/src/health.js +99 -8
- package/src/process-order.js +26 -0
- package/src/release-group.js +157 -22
- package/tensorbuzz.yml +17 -0
- package/test/config-validation.test.js +102 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/health.test.js +74 -1
- package/test/owner-replacement.test.js +22 -2
- package/test/rollbridge.test.js +202 -14
- package/test/tensorbuzz-config.test.js +228 -0
package/src/release-group.js
CHANGED
|
@@ -4,8 +4,11 @@ import {spawn} from "node:child_process"
|
|
|
4
4
|
import {EventEmitter} from "node:events"
|
|
5
5
|
import ManagedProcess from "./managed-process.js"
|
|
6
6
|
import {findAvailablePort} from "./port-allocator.js"
|
|
7
|
+
import {processStartOrder} from "./process-order.js"
|
|
7
8
|
import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
|
|
8
|
-
import {waitForHealth} from "./health.js"
|
|
9
|
+
import {waitForHealth, waitForListeningPort} from "./health.js"
|
|
10
|
+
|
|
11
|
+
const DEPENDENCY_READY_INTERVAL_MS = 250
|
|
9
12
|
|
|
10
13
|
/**
|
|
11
14
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -13,6 +16,7 @@ import {waitForHealth} from "./health.js"
|
|
|
13
16
|
* @typedef {{http: number, websocket: number}} ReleaseConnections
|
|
14
17
|
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
15
18
|
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
|
|
19
|
+
* @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
|
|
16
20
|
*/
|
|
17
21
|
|
|
18
22
|
|
|
@@ -57,6 +61,14 @@ function envId(id) {
|
|
|
57
61
|
return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
|
|
58
62
|
}
|
|
59
63
|
|
|
64
|
+
/**
|
|
65
|
+
* @param {number | undefined} startedAtMs - When the measured phase began.
|
|
66
|
+
* @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
|
|
67
|
+
*/
|
|
68
|
+
function elapsedSince(startedAtMs) {
|
|
69
|
+
return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
|
|
70
|
+
}
|
|
71
|
+
|
|
60
72
|
/**
|
|
61
73
|
* @param {import("./config.js").ProcessConfig} processConfig - Process config.
|
|
62
74
|
* @param {number} index - Zero-based replica index.
|
|
@@ -107,16 +119,27 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
107
119
|
this.stoppedAt = /** @type {string | undefined} */ (undefined)
|
|
108
120
|
this.retirementError = /** @type {string | undefined} */ (undefined)
|
|
109
121
|
this.preserveConfigOnRetirement = false
|
|
122
|
+
this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
|
|
123
|
+
this.readyProcessIds = /** @type {Set<string>} */ (new Set())
|
|
124
|
+
this.startedProcessIds = /** @type {Set<string>} */ (new Set())
|
|
125
|
+
this.startupDependencyWaitMs = 0
|
|
126
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
|
|
127
|
+
this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
|
|
110
128
|
}
|
|
111
129
|
|
|
112
130
|
/** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
|
|
113
131
|
async start() {
|
|
114
132
|
this.state = "starting"
|
|
133
|
+
this.startupStartedAtMs = Date.now()
|
|
115
134
|
|
|
116
135
|
try {
|
|
117
136
|
await this.allocatePorts()
|
|
118
137
|
|
|
119
138
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
139
|
+
await this.waitForConfiguredDependencies(processConfig)
|
|
140
|
+
/** @type {string[]} */
|
|
141
|
+
const instanceIds = []
|
|
142
|
+
|
|
120
143
|
for (let index = 0; index < processConfig.replicas; index += 1) {
|
|
121
144
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
122
145
|
|
|
@@ -127,17 +150,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
127
150
|
if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
|
|
128
151
|
if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
|
|
129
152
|
await processInstance.start("deploy")
|
|
153
|
+
this.processLaunchAtMs.set(instanceId, Date.now())
|
|
154
|
+
this.startedProcessIds.add(instanceId)
|
|
155
|
+
instanceIds.push(instanceId)
|
|
130
156
|
}
|
|
131
157
|
|
|
132
158
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
133
|
-
await
|
|
134
|
-
health: processConfig.health,
|
|
135
|
-
host: this.config.proxy.upstreamHost,
|
|
136
|
-
port: this.ports[processConfig.id]
|
|
137
|
-
})
|
|
138
|
-
|
|
139
|
-
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
159
|
+
await this.waitForProxiedHealth(processConfig)
|
|
140
160
|
}
|
|
161
|
+
|
|
162
|
+
for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
|
|
163
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
141
164
|
}
|
|
142
165
|
} catch (error) {
|
|
143
166
|
this.state = "failed"
|
|
@@ -257,16 +280,132 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
257
280
|
}
|
|
258
281
|
|
|
259
282
|
/**
|
|
260
|
-
*
|
|
283
|
+
* Waits for the ports a process declares it depends on, on a budget that is not the
|
|
284
|
+
* health budget. The dependency leg of a start is not app boot, and charging it to the
|
|
285
|
+
* health deadline measures the dependency instead of the app.
|
|
286
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
|
|
287
|
+
* @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
|
|
288
|
+
*/
|
|
289
|
+
async waitForConfiguredDependencies(processConfig) {
|
|
290
|
+
if (processConfig.dependsOn.length === 0) return
|
|
291
|
+
|
|
292
|
+
for (const dependencyId of processConfig.dependsOn) {
|
|
293
|
+
const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
|
|
294
|
+
|
|
295
|
+
if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
|
|
296
|
+
|
|
297
|
+
this.logger("release process dependency wait", {
|
|
298
|
+
dependencyPort: port,
|
|
299
|
+
dependencyProcessId: dependencyId,
|
|
300
|
+
processId: processConfig.id,
|
|
301
|
+
releaseId: this.releaseId,
|
|
302
|
+
timeoutMs: processConfig.dependencyTimeoutMs
|
|
303
|
+
})
|
|
304
|
+
|
|
305
|
+
const waitStartedAtMs = Date.now()
|
|
306
|
+
|
|
307
|
+
try {
|
|
308
|
+
const {attempts, elapsedMs} = await waitForListeningPort({
|
|
309
|
+
host: this.config.proxy.upstreamHost,
|
|
310
|
+
intervalMs: DEPENDENCY_READY_INTERVAL_MS,
|
|
311
|
+
port,
|
|
312
|
+
timeoutMs: processConfig.dependencyTimeoutMs
|
|
313
|
+
})
|
|
314
|
+
|
|
315
|
+
this.startupDependencyWaitMs += elapsedMs
|
|
316
|
+
this.logger("release process dependency ready", {
|
|
317
|
+
attempts,
|
|
318
|
+
dependencyPort: port,
|
|
319
|
+
dependencyProcessId: dependencyId,
|
|
320
|
+
elapsedMs,
|
|
321
|
+
processId: processConfig.id,
|
|
322
|
+
releaseId: this.releaseId
|
|
323
|
+
})
|
|
324
|
+
} catch (error) {
|
|
325
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
326
|
+
deadlineMs: processConfig.dependencyTimeoutMs,
|
|
327
|
+
dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
|
|
328
|
+
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
329
|
+
failedProcessId: processConfig.id,
|
|
330
|
+
probes: undefined,
|
|
331
|
+
stage: "dependency"
|
|
332
|
+
})
|
|
333
|
+
throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Health checks the proxied process on its own boot budget and records what the boot cost.
|
|
342
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
|
|
343
|
+
* @returns {Promise<void>} Resolves once the process answers its health path.
|
|
344
|
+
*/
|
|
345
|
+
async waitForProxiedHealth(processConfig) {
|
|
346
|
+
const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
|
|
347
|
+
const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
|
|
348
|
+
|
|
349
|
+
try {
|
|
350
|
+
const {elapsedMs} = await waitForHealth({
|
|
351
|
+
health,
|
|
352
|
+
host: this.config.proxy.upstreamHost,
|
|
353
|
+
port: /** @type {number} */ (this.ports[processConfig.id]),
|
|
354
|
+
processId: processConfig.id
|
|
355
|
+
})
|
|
356
|
+
|
|
357
|
+
this.logger("release candidate health ready", {
|
|
358
|
+
bootMs: Date.now() - launchedAtMs,
|
|
359
|
+
deadlineMs: health.timeoutMs,
|
|
360
|
+
healthGateMs: elapsedMs,
|
|
361
|
+
path: health.path,
|
|
362
|
+
port: this.ports[processConfig.id],
|
|
363
|
+
processId: processConfig.id,
|
|
364
|
+
releaseId: this.releaseId,
|
|
365
|
+
waitedForDependenciesMs: this.startupDependencyWaitMs
|
|
366
|
+
})
|
|
367
|
+
} catch (error) {
|
|
368
|
+
const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
|
|
369
|
+
|
|
370
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
371
|
+
deadlineMs: health.timeoutMs,
|
|
372
|
+
dependencyWaitMs: this.startupDependencyWaitMs,
|
|
373
|
+
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
374
|
+
failedProcessId: processConfig.id,
|
|
375
|
+
probes: healthCheck?.attempts,
|
|
376
|
+
stage: "health"
|
|
377
|
+
})
|
|
378
|
+
throw error
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
|
|
261
384
|
* @param {Error | string} error - Startup failure.
|
|
262
385
|
* @param {{phase: string}} options - Diagnostic phase.
|
|
263
386
|
* @returns {void}
|
|
264
387
|
*/
|
|
265
388
|
logStartupFailure(error, {phase}) {
|
|
389
|
+
const diagnostics = this.startupFailureDiagnostics
|
|
390
|
+
const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
|
|
391
|
+
const lastLog = failedLogs?.at(-1)
|
|
392
|
+
const readyProcesses = [...this.readyProcessIds]
|
|
393
|
+
|
|
266
394
|
this.logger("release startup failed", {
|
|
395
|
+
deadlineMs: diagnostics?.deadlineMs ?? null,
|
|
396
|
+
dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
|
|
397
|
+
elapsedMs: diagnostics?.elapsedMs ?? null,
|
|
267
398
|
error: error instanceof Error ? error.message : error,
|
|
399
|
+
failedProcessId: diagnostics?.failedProcessId ?? null,
|
|
400
|
+
lastOutput: lastLog?.line ?? null,
|
|
401
|
+
lastOutputAt: lastLog?.at ?? null,
|
|
268
402
|
phase,
|
|
269
|
-
|
|
403
|
+
probes: diagnostics?.probes ?? null,
|
|
404
|
+
readyProcesses,
|
|
405
|
+
releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
406
|
+
releaseId: this.releaseId,
|
|
407
|
+
stage: diagnostics?.stage ?? null,
|
|
408
|
+
unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
|
|
270
409
|
})
|
|
271
410
|
|
|
272
411
|
for (const processInstance of this.processes.values()) {
|
|
@@ -291,12 +430,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
291
430
|
* @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
|
|
292
431
|
*/
|
|
293
432
|
releaseProcessStartOrder() {
|
|
294
|
-
|
|
295
|
-
const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
|
|
296
|
-
const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
|
|
297
|
-
const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
|
|
298
|
-
|
|
299
|
-
return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
|
|
433
|
+
return processStartOrder(this.config.processes)
|
|
300
434
|
}
|
|
301
435
|
|
|
302
436
|
/** @returns {void} Marks this release active. */
|
|
@@ -367,24 +501,25 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
367
501
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
368
502
|
this.assertCommittedGenerationRecoverable()
|
|
369
503
|
this.state = "starting"
|
|
504
|
+
this.startupStartedAtMs = Date.now()
|
|
370
505
|
try {
|
|
371
506
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
507
|
+
await this.waitForConfiguredDependencies(processConfig)
|
|
372
508
|
const instances = this.getProcesses(processConfig.id)
|
|
373
509
|
|
|
374
510
|
if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
|
|
375
|
-
for (const {process} of instances) {
|
|
511
|
+
for (const {id, process} of instances) {
|
|
376
512
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
377
513
|
await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
|
|
514
|
+
this.processLaunchAtMs.set(id, Date.now())
|
|
515
|
+
this.startedProcessIds.add(id)
|
|
378
516
|
}
|
|
379
517
|
|
|
380
518
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
381
|
-
await
|
|
382
|
-
health: processConfig.health,
|
|
383
|
-
host: this.config.proxy.upstreamHost,
|
|
384
|
-
port: this.ports[processConfig.id]
|
|
385
|
-
})
|
|
519
|
+
await this.waitForProxiedHealth(processConfig)
|
|
386
520
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
387
521
|
}
|
|
522
|
+
for (const {id} of instances) this.readyProcessIds.add(id)
|
|
388
523
|
}
|
|
389
524
|
} catch (error) {
|
|
390
525
|
await this.abortCommittedGenerationRestart()
|
package/tensorbuzz.yml
CHANGED
|
@@ -1,3 +1,20 @@
|
|
|
1
|
+
# TensorBuzz build config for Rollbridge.
|
|
2
|
+
#
|
|
3
|
+
# `paths` (STOE-1179): this gate installs the dependency tree and runs typecheck, ESLint and the
|
|
4
|
+
# whole Velocious suite, and every one of those reads code, `examples/`, and the package/tooling
|
|
5
|
+
# manifests — no gate input is Markdown. A delivery that changes nothing but Markdown therefore
|
|
6
|
+
# has nothing for this lane to check, and TensorBuzz records it as a terminal `skipped` build
|
|
7
|
+
# instead of spending a container and ~3 minutes on it. `test/tensorbuzz-config.test.js` asserts
|
|
8
|
+
# the split against every tracked path so a later edit cannot carve a gate input out of the lane.
|
|
9
|
+
#
|
|
10
|
+
# Ordered rules, last match wins, so the negation has to come after `"**"`. Quote every negation:
|
|
11
|
+
# an unquoted `!x` is a YAML tag, and the config validator fails the whole file rather than
|
|
12
|
+
# filtering anything. The lane still runs whenever `tensorbuzz.yml` itself changes, whenever the
|
|
13
|
+
# changed-file list is missing, empty, or capped by the host, and on every manual rebuild.
|
|
14
|
+
paths:
|
|
15
|
+
- "**"
|
|
16
|
+
- "!**/*.md"
|
|
17
|
+
|
|
1
18
|
before_script:
|
|
2
19
|
- npm install
|
|
3
20
|
script:
|
|
@@ -587,6 +587,108 @@ test("validateConfig defaults health.startDelayMs to 0, accepts an override, and
|
|
|
587
587
|
expect(negative.issues.some((issue) => issue.message === "processes[0].health.startDelayMs must be a non-negative number")).toBeTruthy()
|
|
588
588
|
})
|
|
589
589
|
|
|
590
|
+
test("validateConfig gives an unconfigured proxied process the measured production boot budget", () => {
|
|
591
|
+
/**
|
|
592
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} [processConfig] - Extra keys for the proxied process.
|
|
593
|
+
* @returns {import("../src/config.js").RollbridgeConfig} Normalized config.
|
|
594
|
+
*/
|
|
595
|
+
const normalized = (processConfig = {}) => validateConfig({
|
|
596
|
+
application: "demo",
|
|
597
|
+
control: {path: "/tmp/demo.sock"},
|
|
598
|
+
processes: [
|
|
599
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}, ...processConfig}
|
|
600
|
+
],
|
|
601
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
602
|
+
}).config
|
|
603
|
+
|
|
604
|
+
const defaults = normalized()
|
|
605
|
+
|
|
606
|
+
expect(defaults.proxy.healthTimeoutMs).toBe(120000)
|
|
607
|
+
// The per-process gate inherits the release budget, so a project that configures no health
|
|
608
|
+
// timeout at all gets a deadline above the TensorBuzz boot measured on 2026-11-06 instead of
|
|
609
|
+
// the 30000ms that killed two production deploys.
|
|
610
|
+
expect(defaults.processes[0].health?.timeoutMs).toBe(120000)
|
|
611
|
+
expect(defaults.processes[0].health?.intervalMs).toBe(250)
|
|
612
|
+
|
|
613
|
+
const releaseBudget = normalized({health: {path: "/healthz"}})
|
|
614
|
+
|
|
615
|
+
expect(releaseBudget.proxy.healthTimeoutMs).toBe(120000)
|
|
616
|
+
expect(releaseBudget.processes[0].health?.path).toBe("/healthz")
|
|
617
|
+
expect(releaseBudget.processes[0].health?.timeoutMs).toBe(120000)
|
|
618
|
+
|
|
619
|
+
// Both configuration levels still win over the default: the release-level budget and the
|
|
620
|
+
// per-process budget, which is what a project with a known boot time sets.
|
|
621
|
+
const configuredProxy = validateConfig({
|
|
622
|
+
application: "demo",
|
|
623
|
+
control: {path: "/tmp/demo.sock"},
|
|
624
|
+
processes: [
|
|
625
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
|
|
626
|
+
],
|
|
627
|
+
proxy: {healthTimeoutMs: 45000, host: "127.0.0.1", port: 8182}
|
|
628
|
+
}).config
|
|
629
|
+
|
|
630
|
+
expect(configuredProxy.proxy.healthTimeoutMs).toBe(45000)
|
|
631
|
+
expect(configuredProxy.processes[0].health?.timeoutMs).toBe(45000)
|
|
632
|
+
|
|
633
|
+
const perProcess = normalized({health: {path: "/ping", timeoutMs: 900}})
|
|
634
|
+
|
|
635
|
+
expect(perProcess.proxy.healthTimeoutMs).toBe(120000)
|
|
636
|
+
expect(perProcess.processes[0].health?.timeoutMs).toBe(900)
|
|
637
|
+
})
|
|
638
|
+
|
|
639
|
+
test("validateConfig defaults dependsOn and its own wait budget, and rejects dependencies it cannot wait on", () => {
|
|
640
|
+
/**
|
|
641
|
+
* @param {import("../src/json.js").JsonValue} processes - Processes under test.
|
|
642
|
+
* @returns {{config: import("../src/config.js").RollbridgeConfig, issues: import("../src/config.js").ConfigIssue[]}} Validation result.
|
|
643
|
+
*/
|
|
644
|
+
const validateProcesses = (processes) => validateConfig({
|
|
645
|
+
application: "demo",
|
|
646
|
+
control: {path: "/tmp/demo.sock"},
|
|
647
|
+
processes,
|
|
648
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
649
|
+
})
|
|
650
|
+
|
|
651
|
+
const base = [
|
|
652
|
+
{command: "run beacon", id: "beacon", policy: "service", port: 7330},
|
|
653
|
+
{command: "run jobs", dependsOn: ["beacon"], id: "jobs", policy: "service", port: {from: 7331, to: 7399}},
|
|
654
|
+
{command: "run web", dependsOn: ["jobs"], id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
|
|
655
|
+
]
|
|
656
|
+
const valid = validateProcesses(base)
|
|
657
|
+
|
|
658
|
+
expect(valid.issues).toEqual([])
|
|
659
|
+
expect(valid.config.processes[0].dependsOn).toEqual([])
|
|
660
|
+
expect(valid.config.processes[1].dependsOn).toEqual(["beacon"])
|
|
661
|
+
expect(valid.config.processes[1].dependencyTimeoutMs).toBe(30000)
|
|
662
|
+
|
|
663
|
+
const custom = validateProcesses([...base.slice(0, 2), {...base[2], dependencyTimeoutMs: 60000}])
|
|
664
|
+
|
|
665
|
+
expect(custom.issues).toEqual([])
|
|
666
|
+
expect(custom.config.processes[2].dependencyTimeoutMs).toBe(60000)
|
|
667
|
+
|
|
668
|
+
const invalid = validateProcesses([
|
|
669
|
+
{command: "run web", dependsOn: ["missing", "web", "jobs", "worker"], dependencyTimeoutMs: 0, id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
670
|
+
{command: "run jobs", id: "jobs", policy: "singleton"},
|
|
671
|
+
{command: "run worker", id: "worker", policy: "companion"}
|
|
672
|
+
])
|
|
673
|
+
const messages = invalid.issues.map((issue) => issue.message)
|
|
674
|
+
|
|
675
|
+
expect(messages.some((message) => message === 'Process "web" depends on unknown process "missing"')).toBeTruthy()
|
|
676
|
+
expect(messages.some((message) => message === 'Process "web" cannot depend on itself')).toBeTruthy()
|
|
677
|
+
expect(messages.some((message) => message === 'Process "web" cannot depend on singleton "jobs"')).toBeTruthy()
|
|
678
|
+
expect(messages.some((message) => message === 'Process "worker" has no port, so "web" cannot wait for it')).toBeTruthy()
|
|
679
|
+
expect(messages.some((message) => message === "processes[0].dependencyTimeoutMs must be a positive number")).toBeTruthy()
|
|
680
|
+
|
|
681
|
+
const backwards = validateProcesses([
|
|
682
|
+
{command: "run web", dependsOn: "web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
683
|
+
{command: "run worker", dependsOn: ["web"], id: "worker", policy: "companion", port: 7001}
|
|
684
|
+
])
|
|
685
|
+
const backwardsMessages = backwards.issues.map((issue) => issue.message)
|
|
686
|
+
|
|
687
|
+
expect(backwardsMessages.some((message) => message === "processes[0].dependsOn must be an array")).toBeTruthy()
|
|
688
|
+
expect(backwardsMessages.some((message) => message === 'Process "worker" depends on "web", which Rollbridge starts after it')).toBeTruthy()
|
|
689
|
+
expect(backwards.issues.every((issue) => typeof issue.fix === "string" && issue.fix.length > 0)).toBe(true)
|
|
690
|
+
})
|
|
691
|
+
|
|
590
692
|
test("validateConfig defaults releaseRetention, accepts overrides, and rejects bad values", () => {
|
|
591
693
|
/**
|
|
592
694
|
* @param {import("../src/json.js").JsonValue} releaseRetention - Retention config under test, or undefined.
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import http from "node:http"
|
|
4
|
+
import net from "node:net"
|
|
5
|
+
|
|
6
|
+
const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
|
|
7
|
+
const bootMs = Number(process.env.ROLLBRIDGE_BOOT_MS || "0")
|
|
8
|
+
const dependencyPort = process.env.ROLLBRIDGE_DEPENDENCY_PORT
|
|
9
|
+
const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
|
|
10
|
+
|
|
11
|
+
log(`starting (boot budget ${bootMs}ms)`)
|
|
12
|
+
|
|
13
|
+
if (dependencyPort) {
|
|
14
|
+
// The app performs its own dependency wait, exactly as the production web command does.
|
|
15
|
+
await waitForDependency(Number(dependencyPort))
|
|
16
|
+
log(`dependency ${dependencyPort} accepted a connection`)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
await sleep(bootMs)
|
|
20
|
+
|
|
21
|
+
const server = http.createServer((request, response) => {
|
|
22
|
+
if (request.url === "/ping") {
|
|
23
|
+
response.writeHead(200, {"Content-Type": "application/json"})
|
|
24
|
+
response.end(JSON.stringify({message: "Pong", releaseId}))
|
|
25
|
+
return
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
|
|
29
|
+
response.end(`${releaseId}\n`)
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
process.on("SIGTERM", () => {
|
|
33
|
+
server.close(() => process.exit(0))
|
|
34
|
+
setTimeout(() => process.exit(0), 10).unref()
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
server.listen(port, "127.0.0.1")
|
|
38
|
+
log(`listening on ${port}`)
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* @param {number} dependencyPortNumber - Port of the process this app depends on.
|
|
42
|
+
* @returns {Promise<void>} Resolves once the dependency accepts a connection.
|
|
43
|
+
*/
|
|
44
|
+
async function waitForDependency(dependencyPortNumber) {
|
|
45
|
+
const deadline = Date.now() + 10000
|
|
46
|
+
|
|
47
|
+
while (Date.now() < deadline) {
|
|
48
|
+
if (await tryConnect(dependencyPortNumber)) return
|
|
49
|
+
await sleep(50)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
log(`dependency ${dependencyPortNumber} never accepted a connection`)
|
|
53
|
+
process.exit(21)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @param {number} dependencyPortNumber - Port to connect to.
|
|
58
|
+
* @returns {Promise<boolean>} True when the dependency accepted the connection.
|
|
59
|
+
*/
|
|
60
|
+
function tryConnect(dependencyPortNumber) {
|
|
61
|
+
return new Promise((resolve) => {
|
|
62
|
+
const socket = net.createConnection({host: "127.0.0.1", port: dependencyPortNumber})
|
|
63
|
+
let settled = false
|
|
64
|
+
|
|
65
|
+
/** @param {boolean} connected - Whether the connection succeeded. */
|
|
66
|
+
const finish = (connected) => {
|
|
67
|
+
if (settled) return
|
|
68
|
+
settled = true
|
|
69
|
+
socket.destroy()
|
|
70
|
+
resolve(connected)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
socket.setTimeout(200)
|
|
74
|
+
socket.once("connect", () => finish(true))
|
|
75
|
+
socket.once("error", () => finish(false))
|
|
76
|
+
socket.once("timeout", () => finish(false))
|
|
77
|
+
})
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* @param {number} ms - Milliseconds.
|
|
82
|
+
* @returns {Promise<void>} Resolves after the delay.
|
|
83
|
+
*/
|
|
84
|
+
function sleep(ms) {
|
|
85
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* @param {string} line - Output line, so the failure record can name the app's last output.
|
|
90
|
+
* @returns {void}
|
|
91
|
+
*/
|
|
92
|
+
function log(line) {
|
|
93
|
+
console.log(`boot: ${process.pid} ${line}`)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* @param {string} key - Environment variable name.
|
|
98
|
+
* @returns {string} Environment variable value.
|
|
99
|
+
*/
|
|
100
|
+
function requiredEnv(key) {
|
|
101
|
+
const value = process.env[key]
|
|
102
|
+
|
|
103
|
+
if (!value) throw new Error(`${key} is required`)
|
|
104
|
+
|
|
105
|
+
return value
|
|
106
|
+
}
|
package/test/health.test.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import http from "node:http"
|
|
4
|
+
import net from "node:net"
|
|
4
5
|
import {describe, expect, test} from "@velocious/testing"
|
|
5
|
-
import {waitForHealth} from "../src/health.js"
|
|
6
|
+
import {waitForHealth, waitForListeningPort} from "../src/health.js"
|
|
6
7
|
|
|
7
8
|
describe("health", () => {
|
|
8
9
|
|
|
@@ -31,6 +32,18 @@ async function startHealthServer() {
|
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
|
|
35
|
+
/**
|
|
36
|
+
* @param {net.Server} server - Server to bind.
|
|
37
|
+
* @returns {Promise<number>} Ephemeral port it bound.
|
|
38
|
+
*/
|
|
39
|
+
async function bindPort(server) {
|
|
40
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(undefined)))
|
|
41
|
+
|
|
42
|
+
const address = server.address()
|
|
43
|
+
|
|
44
|
+
return address && typeof address === "object" ? address.port : 0
|
|
45
|
+
}
|
|
46
|
+
|
|
34
47
|
test("waitForHealth delays the first probe by startDelayMs", async () => {
|
|
35
48
|
const server = await startHealthServer()
|
|
36
49
|
|
|
@@ -62,4 +75,64 @@ test("waitForHealth probes immediately when startDelayMs is 0", async () => {
|
|
|
62
75
|
await server.close()
|
|
63
76
|
}
|
|
64
77
|
})
|
|
78
|
+
|
|
79
|
+
test("waitForHealth names the deadline and the time it spent when the app never becomes healthy", async () => {
|
|
80
|
+
const server = http.createServer((_request, response) => {
|
|
81
|
+
response.writeHead(500)
|
|
82
|
+
response.end("bad")
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
try {
|
|
86
|
+
const port = await bindPort(server)
|
|
87
|
+
|
|
88
|
+
/** @type {import("../src/health.js").HealthCheckError} */
|
|
89
|
+
const failure = /** @type {import("../src/health.js").HealthCheckError} */ (await waitForHealth({
|
|
90
|
+
health: {intervalMs: 25, path: "/ping", startDelayMs: 0, timeoutMs: 300},
|
|
91
|
+
host: "127.0.0.1",
|
|
92
|
+
port,
|
|
93
|
+
processId: "web"
|
|
94
|
+
}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
95
|
+
|
|
96
|
+
expect(String(failure.message)).toMatch(/process web: \d+ probes, \d+ms elapsed of the 300ms health deadline/)
|
|
97
|
+
expect(failure.healthCheck?.deadlineMs).toBe(300)
|
|
98
|
+
expect({value: Boolean(failure.healthCheck && failure.healthCheck.elapsedMs >= 300), context: `expected the health gate to have spent its whole budget, was ${failure.healthCheck?.elapsedMs}ms`}).toMatchObject({value: true})
|
|
99
|
+
expect({value: Boolean(failure.healthCheck && failure.healthCheck.attempts >= 1), context: `expected at least one probe, was ${failure.healthCheck?.attempts}`}).toMatchObject({value: true})
|
|
100
|
+
expect(failure.healthCheck?.processId).toBe("web")
|
|
101
|
+
} finally {
|
|
102
|
+
await server.close()
|
|
103
|
+
}
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
test("waitForListeningPort waits for a dependency port on a budget of its own", async () => {
|
|
107
|
+
const server = net.createServer()
|
|
108
|
+
const port = await bindPort(server)
|
|
109
|
+
|
|
110
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
111
|
+
|
|
112
|
+
const waiting = waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 2000})
|
|
113
|
+
|
|
114
|
+
await new Promise((resolve) => setTimeout(resolve, 300))
|
|
115
|
+
await new Promise((resolve) => server.listen(port, "127.0.0.1", () => resolve(undefined)))
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const result = await waiting
|
|
119
|
+
|
|
120
|
+
expect({value: Boolean(result.attempts >= 2), context: `expected more than one connection attempt, was ${result.attempts}`}).toMatchObject({value: true})
|
|
121
|
+
expect({value: Boolean(result.elapsedMs >= 250), context: `expected the dependency wait to take ~300ms, was ${result.elapsedMs}ms`}).toMatchObject({value: true})
|
|
122
|
+
} finally {
|
|
123
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
124
|
+
}
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
test("waitForListeningPort names the budget it ran out on when the dependency never binds", async () => {
|
|
128
|
+
const server = net.createServer()
|
|
129
|
+
const port = await bindPort(server)
|
|
130
|
+
|
|
131
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
132
|
+
|
|
133
|
+
/** @type {Error} */
|
|
134
|
+
const failure = /** @type {Error} */ (await waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 200}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
135
|
+
|
|
136
|
+
expect(String(failure.message)).toMatch(/did not accept a connection within 200ms/)
|
|
137
|
+
})
|
|
65
138
|
})
|
|
@@ -22,6 +22,15 @@ const binPath = path.join(repoRoot, "bin", "rollbridge")
|
|
|
22
22
|
const dummyAppPath = path.join(repoRoot, "test", "fixtures", "dummy-app.js")
|
|
23
23
|
const legacyDaemonPath = path.join(repoRoot, "test", "fixtures", "pre-split3-daemon-runner.js")
|
|
24
24
|
const partialGuardianPath = path.join(repoRoot, "test", "fixtures", "partial-owner-replacement-process-guardian.js")
|
|
25
|
+
// Budget for the retired-drain fixture below. It spawns six child processes (a daemon, a worker and
|
|
26
|
+
// a proxied web process for each of two releases, plus a partial owner-replacement guardian) and then
|
|
27
|
+
// waits for the incumbent's retired drain to resume, so its cost tracks host contention, not the
|
|
28
|
+
// behaviour under test. Measured on the same machine in the same invocation: 1.457s at load average
|
|
29
|
+
// 8.3 and 7.105s at load average 24.5, both with the drain completing. The previous 5000ms cap sat
|
|
30
|
+
// between those two numbers, which is why the spec reddened on a busy runner while nothing in the
|
|
31
|
+
// drain actually stalled. 20000ms is 2.8x the measured worst case; the wait stays event-gated, so a
|
|
32
|
+
// release that never resumes its drain still fails, and now fails naming the release record.
|
|
33
|
+
const PARTIAL_DRAIN_FIXTURE_TIMEOUT_MS = 20000
|
|
25
34
|
|
|
26
35
|
test("partial owner-replacement guardian crosses the authenticated legacy bridge before retired-owner commit", async () => {
|
|
27
36
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-"))
|
|
@@ -324,7 +333,7 @@ test("partial guardian replacement persists the coordinator only after ownership
|
|
|
324
333
|
}
|
|
325
334
|
})
|
|
326
335
|
|
|
327
|
-
test("failed partial upgrade resumes an incumbent retired release drain", {timeoutMs:
|
|
336
|
+
test("failed partial upgrade resumes an incumbent retired release drain", {timeoutMs: PARTIAL_DRAIN_FIXTURE_TIMEOUT_MS}, async () => {
|
|
328
337
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-drain-"))
|
|
329
338
|
const socketPath = path.join(root, "rollbridge.sock")
|
|
330
339
|
const statePath = path.join(root, "state.json")
|
|
@@ -382,7 +391,18 @@ test("failed partial upgrade resumes an incumbent retired release drain", {timeo
|
|
|
382
391
|
retainedConnection.destroy()
|
|
383
392
|
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
384
393
|
await releaseDrained
|
|
385
|
-
|
|
394
|
+
const drainedStatus = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
395
|
+
|
|
396
|
+
expect(releaseState(drainedStatus, "v1")).toBe("stopped")
|
|
397
|
+
// The resumed drain must also be visible in the retired release's own record, not only as the
|
|
398
|
+
// stdout line the wait is keyed on: a release cleared without draining its processes leaves no
|
|
399
|
+
// drainStartedAt/stoppedAt pair behind.
|
|
400
|
+
const drainedRelease = /** @type {{drainStartedAt?: string, releaseId: string, stoppedAt?: string}[]} */ (drainedStatus.releases)
|
|
401
|
+
.find(({releaseId}) => releaseId === "v1")
|
|
402
|
+
|
|
403
|
+
expect(typeof drainedRelease?.drainStartedAt).toBe("string")
|
|
404
|
+
expect(typeof drainedRelease?.stoppedAt).toBe("string")
|
|
405
|
+
expect(Date.parse(/** @type {string} */ (drainedRelease?.stoppedAt))).toBeGreaterThanOrEqual(Date.parse(/** @type {string} */ (drainedRelease?.drainStartedAt)))
|
|
386
406
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
|
|
387
407
|
|
|
388
408
|
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|