rollbridge 0.1.59 → 0.1.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/docs/config.md +40 -0
- package/docs/troubleshooting.md +19 -6
- package/package.json +1 -1
- package/src/config.js +91 -2
- package/src/health.js +99 -8
- package/src/process-order.js +26 -0
- package/src/release-group.js +157 -22
- package/test/config-validation.test.js +53 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/health.test.js +74 -1
- package/test/rollbridge.test.js +128 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
### Added
|
|
2
|
+
|
|
3
|
+
- `processes[].dependsOn` names the same-release processes a process needs, and
|
|
4
|
+
Rollbridge now waits for each dependency's allocated port **before** spawning
|
|
5
|
+
the dependent, on `processes[].dependencyTimeoutMs` (default 30 seconds) — a
|
|
6
|
+
budget of its own that is never charged to `health.timeoutMs`. Each wait is
|
|
7
|
+
logged as `release process dependency wait` and `release process dependency
|
|
8
|
+
ready` with `attempts` and `elapsedMs`. `validate` rejects a dependency on
|
|
9
|
+
itself, an unknown process, a `singleton`, a process with no `port`, and a
|
|
10
|
+
process Rollbridge starts after the dependent, which also rules out cycles.
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
|
|
14
|
+
- A release-scoped process no longer spends the proxied health deadline waiting
|
|
15
|
+
for a dependency. Measured on production 2026-11-06: `background-jobs-main`
|
|
16
|
+
took 11 817 ms to start listening (14:33:39.175 → 14:33:50.992) and the web
|
|
17
|
+
command's own `wait-port` hop then ate part of the same 30 000 ms that
|
|
18
|
+
`proxy.healthTimeoutMs` and `health.timeoutMs` allowed for the whole start
|
|
19
|
+
sequence, leaving the app about 16 seconds to boot before Rollbridge killed a
|
|
20
|
+
release that was still starting normally. The health gate now measures only
|
|
21
|
+
the proxied process's own boot and reports `bootMs`, `healthGateMs`, and
|
|
22
|
+
`waitedForDependenciesMs` in `release candidate health ready`.
|
|
23
|
+
- A failed release start is diagnostic. The health failure now reads
|
|
24
|
+
`Health check failed for <url>: <reason> (process <id>: 18 probes, 941ms
|
|
25
|
+
elapsed of the 900ms health deadline)` instead of naming only the connection
|
|
26
|
+
error, and `release startup failed` carries `deadlineMs`, `dependencyWaitMs`,
|
|
27
|
+
`elapsedMs`, `releaseElapsedMs`, `stage` (`dependency` or `health`), `probes`,
|
|
28
|
+
`failedProcessId`, `readyProcesses`, `unreadyProcesses`, and the failed
|
|
29
|
+
process's `lastOutput`/`lastOutputAt`. A dependency gate that expires fails
|
|
30
|
+
the deploy as `Process <id> could not start because dependency <id> was not
|
|
31
|
+
ready: …` on its own budget; the health gate is unchanged, so a genuinely
|
|
32
|
+
unhealthy release still fails and the incumbent keeps serving.
|
package/docs/config.md
CHANGED
|
@@ -233,6 +233,8 @@ legacyTakeover: {
|
|
|
233
233
|
| `env` | object of string → string | `{}` | Extra environment variables (values templated). Merged over the injected `ROLLBRIDGE_*` vars. |
|
|
234
234
|
| `port` | number or `{from, to}` | unset | Port (or range) allocated per release. **Required for the `proxied` process.** A plain number `n` means the fixed port `n` (`{from: n, to: n}`). |
|
|
235
235
|
| `health` | object or `false` | enabled with defaults | Health check for the `proxied` process; set `false` to disable (see below). |
|
|
236
|
+
| `dependsOn` | array of process ids | `[]` | Processes whose allocated port must accept a connection before Rollbridge starts this process (see [`processes[].dependsOn`](#processesdependson)). |
|
|
237
|
+
| `dependencyTimeoutMs` | positive number | `30000` | Budget for each `dependsOn` wait. Charged to its own clock, never to `health.timeoutMs`. |
|
|
236
238
|
| `stopSignal` | signal name (e.g. `"SIGTERM"`, `"SIGINT"`, `"SIGQUIT"`) | `"SIGTERM"` | Signal sent to gracefully stop the process; after `gracefulStopMs` it is `SIGKILL`ed. Use a worker's quit signal so it finishes in-flight work before exiting. |
|
|
237
239
|
| `nonBlockingDrain` | boolean | `false` | When a release is retired, drain this process **immediately** (in parallel with the proxied connection drain) instead of after it. Companion processes only — typically background workers (see below). |
|
|
238
240
|
| `lifecycle` | object | no hooks | Command hooks run when gracefully stopping the process (see below). |
|
|
@@ -447,6 +449,43 @@ release). Set `health: false` to disable it.
|
|
|
447
449
|
| `health.intervalMs` | number | `250` | Delay between probes. |
|
|
448
450
|
| `health.startDelayMs` | non-negative number | `0` | Wait this long after the process starts before the first probe (runs before the `timeoutMs` window). |
|
|
449
451
|
|
|
452
|
+
### `processes[].dependsOn`
|
|
453
|
+
|
|
454
|
+
A release-scoped process that talks to a same-release service usually waits for
|
|
455
|
+
that service inside its own command (`wait-port … && exec <app>`). That wait
|
|
456
|
+
runs after Rollbridge has started the process, so it is charged against
|
|
457
|
+
`health.timeoutMs`: a dependency that takes 12 seconds to bind leaves the app 18
|
|
458
|
+
seconds of a 30-second health deadline, and the deploy fails while the app is
|
|
459
|
+
still booting normally.
|
|
460
|
+
|
|
461
|
+
Declare the dependency instead, and Rollbridge waits for each `dependsOn`
|
|
462
|
+
process's allocated port **before** spawning the dependent, on a budget of its
|
|
463
|
+
own (`dependencyTimeoutMs`):
|
|
464
|
+
|
|
465
|
+
```js
|
|
466
|
+
{
|
|
467
|
+
id: "web",
|
|
468
|
+
policy: "proxied",
|
|
469
|
+
dependsOn: ["background-jobs-main"],
|
|
470
|
+
dependencyTimeoutMs: 30000,
|
|
471
|
+
command: "exec node build/bin/velocious.js server --port {{port}}",
|
|
472
|
+
port: {from: 14500, to: 14599},
|
|
473
|
+
health: {path: "/ping", timeoutMs: 120000}
|
|
474
|
+
}
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
`health.timeoutMs` then measures only the app's own boot. Both waits are logged
|
|
478
|
+
separately: `release process dependency wait` / `release process dependency
|
|
479
|
+
ready` (with `attempts` and `elapsedMs`) and `release candidate health ready`
|
|
480
|
+
(with `bootMs`, `healthGateMs`, and `waitedForDependenciesMs`).
|
|
481
|
+
|
|
482
|
+
`dependsOn` accepts ids of processes Rollbridge starts before this one —
|
|
483
|
+
persistent services, handoff services, companions, and the proxied process. A
|
|
484
|
+
dependency must have a `port` (there is nothing to wait for otherwise), may not
|
|
485
|
+
be a `singleton` (singleton ownership is not per-release), and may not be this
|
|
486
|
+
process itself. Dependencies that Rollbridge would start after this process are
|
|
487
|
+
rejected, which also rules out cycles.
|
|
488
|
+
|
|
450
489
|
## `compose[]`
|
|
451
490
|
|
|
452
491
|
Optional Docker Compose services this app updates. Each entry is a unit Rollbridge
|
|
@@ -548,6 +587,7 @@ Rollbridge sets these in every managed process's environment (the process's own
|
|
|
548
587
|
- `replicas` must be a positive integer; `replicas > 1` is allowed only on a `companion` process without a `port`. Process ids must not contain `#` (reserved for replica instance ids).
|
|
549
588
|
- `lifecycle.activateCommand`/`quietCommand`/`drainCommand`/`stopCommand` must be non-empty strings when set, and `lifecycle.drainTimeoutMs` a non-negative number; `lifecycle.drainCommand` requires a positive `lifecycle.drainTimeoutMs`. `activateCommand` is allowed on at most one handoff service, requires that service's `quietCommand`, and requires `statePath` plus `ownerRecovery`. A `lifecycle.stopCommand` may not be combined with a custom `stopSignal` (the `stopCommand` runs instead of the signal, so the signal would be ignored).
|
|
550
589
|
- `nonBlockingDrain` must be a boolean, and is allowed only on a `companion` process.
|
|
590
|
+
- `dependsOn` must be an array of configured process ids: not the process itself, not a `singleton`, not a process without a `port`, and not a process Rollbridge starts after it. `dependencyTimeoutMs` must be a positive number.
|
|
551
591
|
- `statePath` must be a string when set.
|
|
552
592
|
- `compose` must be a non-empty array of unit mappings. Each unit needs `id`, `service`, `projectDirectory`, and a valid `image` reference (a tag, or a `@sha256:` digest with 64 hex characters; no whitespace, no leading `-`).
|
|
553
593
|
- Compose unit `id`s must be unique and must not collide with a process `id`.
|
package/docs/troubleshooting.md
CHANGED
|
@@ -54,14 +54,27 @@ Health check failed for http://127.0.0.1:18182/ping: HTTP 503
|
|
|
54
54
|
new release never went live; the previous release stays active.
|
|
55
55
|
|
|
56
56
|
**Diagnose.** The new release's `proxied` process didn't return a healthy
|
|
57
|
-
response in time.
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
57
|
+
response in time. The reason names what it spent that budget on —
|
|
58
|
+
`(process web: 18 probes, 941ms elapsed of the 900ms health deadline)` — and the
|
|
59
|
+
`release startup failed` event carries `deadlineMs`, `elapsedMs`,
|
|
60
|
+
`dependencyWaitMs`, `probes`, `stage` (`"dependency"` or `"health"`),
|
|
61
|
+
`failedProcessId`, `readyProcesses`/`unreadyProcesses`, and the process's
|
|
62
|
+
`lastOutput`. Check that process's output with `rollbridge logs --process <id>`
|
|
63
|
+
and its state/`exitCode` with `rollbridge status`. Common causes: the app doesn't
|
|
64
|
+
listen on the templated `{{port}}`, the `health.path` returns a non-2xx status, or
|
|
65
|
+
the app boots slower than `health.timeoutMs`.
|
|
66
|
+
|
|
67
|
+
If the boot itself is fine but the app waits for a same-release service inside
|
|
68
|
+
its own command, that wait is inside the health deadline. `stage: "dependency"`
|
|
69
|
+
plus a `release process dependency ready` event shows the wait Rollbridge did on
|
|
70
|
+
the dependency's own budget.
|
|
61
71
|
|
|
62
72
|
**Fix.** Make the proxied command bind `{{port}}` and serve `health.path` with a
|
|
63
|
-
2xx status. For slow boots, raise `health.timeoutMs`
|
|
64
|
-
so probing begins after the app is up.
|
|
73
|
+
2xx status. For slow boots, raise `health.timeoutMs` to above the measured boot
|
|
74
|
+
time (or set `health.startDelayMs` so probing begins after the app is up). For a
|
|
75
|
+
same-release service dependency, declare `dependsOn` (see
|
|
76
|
+
[`processes[].dependsOn`](config.md#processesdependson)) so the wait runs on
|
|
77
|
+
`dependencyTimeoutMs` instead of eating the boot budget.
|
|
65
78
|
|
|
66
79
|
## Port conflicts / exhausted ranges
|
|
67
80
|
|
package/package.json
CHANGED
package/src/config.js
CHANGED
|
@@ -6,6 +6,7 @@ import {createRequire} from "node:module"
|
|
|
6
6
|
import os from "node:os"
|
|
7
7
|
import path from "node:path"
|
|
8
8
|
import {pathToFileURL} from "node:url"
|
|
9
|
+
import {processStartOrder} from "./process-order.js"
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -17,7 +18,7 @@ import {pathToFileURL} from "node:url"
|
|
|
17
18
|
* @typedef {{activateCommand?: string, activateTimeoutMs?: number, drainCommand?: string, drainTimeoutMs: number, quietCommand?: string, reactivateCommand?: string, stopCommand?: string}} LifecycleConfig
|
|
18
19
|
* @typedef {number | "indefinite"} StopTimeoutMs
|
|
19
20
|
* @typedef {"persistent" | "handoff"} ServiceDeployStrategy
|
|
20
|
-
* @typedef {{cwd?: string, deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
|
|
21
|
+
* @typedef {{cwd?: string, dependencyTimeoutMs: number, dependsOn: string[], deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
|
|
21
22
|
* @typedef {{group?: number | string, mode?: number, owner?: number | string, path: string}} ControlConfig
|
|
22
23
|
* @typedef {{drainTimeoutMs: number, forceStopTimeoutMs: number, healthPath: string, healthTimeoutMs: number, host: string, port: number, upstreamHost: string}} ProxyConfig
|
|
23
24
|
* @typedef {{includes: string[], name: string}} LegacyTakeoverProcessConfig
|
|
@@ -35,6 +36,13 @@ import {pathToFileURL} from "node:url"
|
|
|
35
36
|
|
|
36
37
|
const PROCESS_POLICIES = new Set(["proxied", "companion", "singleton", "service"])
|
|
37
38
|
const DEFAULT_CONFIG_FILENAMES = ["rollbridge.js"]
|
|
39
|
+
// Budget for one process waiting for a dependency's port, separate from the health budget.
|
|
40
|
+
// Measured on TensorBuzz production (server3, 2026-11-06): the candidate's
|
|
41
|
+
// background-jobs-main took 11817ms (14:33:39.175 start -> 14:33:50.992 listening) before
|
|
42
|
+
// the web process could reach its own boot, and the production command chain already waited
|
|
43
|
+
// 30000ms per hop. 30000ms keeps that dependency budget, so the health deadline measures
|
|
44
|
+
// only the app boot.
|
|
45
|
+
const DEFAULT_DEPENDENCY_TIMEOUT_MS = 30000
|
|
38
46
|
const commonJsRequire = createRequire(import.meta.url)
|
|
39
47
|
|
|
40
48
|
/**
|
|
@@ -232,7 +240,7 @@ function normalizeProcess(value, index, proxy, issues) {
|
|
|
232
240
|
if (!isPlainObject(value)) {
|
|
233
241
|
issues.push({fix: `Define processes[${index}] as a mapping with id, policy, and command.`, message: `processes[${index}] must be an object`})
|
|
234
242
|
|
|
235
|
-
return {command: "", cwd: undefined, deployStrategy: "persistent", env: {}, gracefulStopMs: proxy.forceStopTimeoutMs, health: undefined, id: "", lifecycle: {drainTimeoutMs: 0}, memory: undefined, nonBlockingDrain: false, outputLines: 50, policy: "companion", port: undefined, replicas: 1, restart: defaultRestartConfig(), restartDelayMs: 1000, stopSignal: "SIGTERM"}
|
|
243
|
+
return {command: "", cwd: undefined, dependencyTimeoutMs: DEFAULT_DEPENDENCY_TIMEOUT_MS, dependsOn: [], deployStrategy: "persistent", env: {}, gracefulStopMs: proxy.forceStopTimeoutMs, health: undefined, id: "", lifecycle: {drainTimeoutMs: 0}, memory: undefined, nonBlockingDrain: false, outputLines: 50, policy: "companion", port: undefined, replicas: 1, restart: defaultRestartConfig(), restartDelayMs: 1000, stopSignal: "SIGTERM"}
|
|
236
244
|
}
|
|
237
245
|
|
|
238
246
|
const source = value
|
|
@@ -240,6 +248,8 @@ function normalizeProcess(value, index, proxy, issues) {
|
|
|
240
248
|
return {
|
|
241
249
|
command: normalizeString(source.command, `processes[${index}].command`, issues),
|
|
242
250
|
cwd: source.cwd === undefined ? undefined : normalizeString(source.cwd, `processes[${index}].cwd`, issues),
|
|
251
|
+
dependencyTimeoutMs: normalizeDependencyTimeoutMs(source.dependencyTimeoutMs, `processes[${index}].dependencyTimeoutMs`, issues),
|
|
252
|
+
dependsOn: normalizeDependsOn(source.dependsOn, `processes[${index}].dependsOn`, issues),
|
|
243
253
|
deployStrategy: normalizeDeployStrategy(source.deployStrategy, `processes[${index}].deployStrategy`, issues),
|
|
244
254
|
env: normalizeEnv(source.env, `processes[${index}].env`, issues),
|
|
245
255
|
gracefulStopMs: normalizeStopTimeout(source.gracefulStopMs, `processes[${index}].gracefulStopMs`, proxy.forceStopTimeoutMs, issues),
|
|
@@ -765,6 +775,34 @@ function normalizeDeployStrategy(value, key, issues) {
|
|
|
765
775
|
return "persistent"
|
|
766
776
|
}
|
|
767
777
|
|
|
778
|
+
/**
|
|
779
|
+
* @param {JsonValue} value - Raw dependency list.
|
|
780
|
+
* @param {string} key - Config key.
|
|
781
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
782
|
+
* @returns {string[]} Ids of the processes this process waits for before it starts.
|
|
783
|
+
*/
|
|
784
|
+
function normalizeDependsOn(value, key, issues) {
|
|
785
|
+
return normalizeStringList(value, key, issues)
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* @param {JsonValue} value - Raw dependency wait budget.
|
|
790
|
+
* @param {string} key - Config key.
|
|
791
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
792
|
+
* @returns {number} Milliseconds this process may spend waiting for its dependency ports.
|
|
793
|
+
*/
|
|
794
|
+
function normalizeDependencyTimeoutMs(value, key, issues) {
|
|
795
|
+
const timeoutMs = normalizeNumber(value, key, issues, {default: DEFAULT_DEPENDENCY_TIMEOUT_MS})
|
|
796
|
+
|
|
797
|
+
if (timeoutMs < 1) {
|
|
798
|
+
issues.push({fix: `Set ${key} to a positive number of milliseconds, e.g. 30000, or omit it for the default.`, message: `${key} must be a positive number`})
|
|
799
|
+
|
|
800
|
+
return DEFAULT_DEPENDENCY_TIMEOUT_MS
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
return timeoutMs
|
|
804
|
+
}
|
|
805
|
+
|
|
768
806
|
/**
|
|
769
807
|
* @param {JsonValue} value - Raw output retention value.
|
|
770
808
|
* @param {string} key - Config key.
|
|
@@ -949,6 +987,57 @@ function validateProcessSet(processes, issues) {
|
|
|
949
987
|
|
|
950
988
|
issues.push({fix: `Add a port range to the proxied process "${processConfig.id || "(unnamed)"}", e.g. port: {from: 18000, to: 18099}.`, message: `Proxied process ${processConfig.id || "(unnamed)"} must define a port range`})
|
|
951
989
|
}
|
|
990
|
+
|
|
991
|
+
validateProcessDependencies(processes, issues)
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
/**
|
|
995
|
+
* Validates `dependsOn`: a dependency must name a ported process that Rollbridge starts
|
|
996
|
+
* before the dependent. Waiting is done on the dependency's allocated port, and an edge that
|
|
997
|
+
* points backwards in the start order can never be satisfied, which also rules out cycles.
|
|
998
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
999
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
1000
|
+
* @returns {void}
|
|
1001
|
+
*/
|
|
1002
|
+
function validateProcessDependencies(processes, issues) {
|
|
1003
|
+
const declared = new Map(processes.filter((processConfig) => processConfig.id).map((processConfig) => [processConfig.id, processConfig]))
|
|
1004
|
+
const startOrder = new Map(processStartOrder(processes, {includePersistentServices: true}).map((processConfig, index) => [processConfig.id, index]))
|
|
1005
|
+
|
|
1006
|
+
for (const processConfig of processes) {
|
|
1007
|
+
if (!processConfig.id || processConfig.dependsOn.length === 0) continue
|
|
1008
|
+
|
|
1009
|
+
const dependentIndex = startOrder.get(processConfig.id)
|
|
1010
|
+
|
|
1011
|
+
for (const dependencyId of processConfig.dependsOn) {
|
|
1012
|
+
if (dependencyId === processConfig.id) {
|
|
1013
|
+
issues.push({fix: `Remove "${processConfig.id}" from its own dependsOn.`, message: `Process "${processConfig.id}" cannot depend on itself`})
|
|
1014
|
+
continue
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
const dependency = declared.get(dependencyId)
|
|
1018
|
+
|
|
1019
|
+
if (!dependency) {
|
|
1020
|
+
issues.push({fix: `Name a configured process in "${processConfig.id}".dependsOn; "${dependencyId}" is not one of: ${[...declared.keys()].join(", ")}.`, message: `Process "${processConfig.id}" depends on unknown process "${dependencyId}"`})
|
|
1021
|
+
continue
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
if (dependency.policy === "singleton") {
|
|
1025
|
+
issues.push({fix: `Remove "${dependencyId}" from "${processConfig.id}".dependsOn; a singleton is not part of a release generation, so Rollbridge cannot gate a release start on it.`, message: `Process "${processConfig.id}" cannot depend on singleton "${dependencyId}"`})
|
|
1026
|
+
continue
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
if (!dependency.port) {
|
|
1030
|
+
issues.push({fix: `Give "${dependencyId}" a port, or remove it from "${processConfig.id}".dependsOn; Rollbridge waits for a dependency on its allocated port.`, message: `Process "${dependencyId}" has no port, so "${processConfig.id}" cannot wait for it`})
|
|
1031
|
+
continue
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
const dependencyIndex = startOrder.get(dependencyId)
|
|
1035
|
+
|
|
1036
|
+
if (dependencyIndex === undefined || dependentIndex === undefined || dependencyIndex > dependentIndex) {
|
|
1037
|
+
issues.push({fix: `Move "${dependencyId}" before "${processConfig.id}" in processes[]; Rollbridge starts persistent services, then handoff services, companions, and the proxied process.`, message: `Process "${processConfig.id}" depends on "${dependencyId}", which Rollbridge starts after it`})
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
952
1041
|
}
|
|
953
1042
|
|
|
954
1043
|
/**
|
package/src/health.js
CHANGED
|
@@ -1,35 +1,126 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
+
import net from "node:net"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @typedef {import("./config.js").HealthConfig} HealthConfig
|
|
7
|
+
* @typedef {{attempts: number, deadlineMs: number, elapsedMs: number, processId: string, url: string}} HealthCheckDiagnostics
|
|
8
|
+
* @typedef {Error & {healthCheck?: HealthCheckDiagnostics}} HealthCheckError
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* @param {number} ms - Milliseconds.
|
|
13
|
+
* @returns {Promise<void>} Resolves after the delay.
|
|
14
|
+
*/
|
|
15
|
+
function sleep(ms) {
|
|
16
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
17
|
+
}
|
|
18
|
+
|
|
3
19
|
/**
|
|
4
20
|
* Waits for a process HTTP health endpoint.
|
|
21
|
+
*
|
|
22
|
+
* The failure names its own budget: `Health check failed ...: fetch failed` cannot tell a
|
|
23
|
+
* slow boot apart from a health deadline that was simply too short.
|
|
5
24
|
* @param {object} args - Options.
|
|
6
25
|
* @param {string} args.host - Host.
|
|
7
26
|
* @param {number} args.port - Port.
|
|
8
|
-
* @param {
|
|
9
|
-
* @
|
|
27
|
+
* @param {HealthConfig} args.health - Health config.
|
|
28
|
+
* @param {string} [args.processId] - Process the probe belongs to, named in the failure.
|
|
29
|
+
* @returns {Promise<{elapsedMs: number}>} Time spent reaching the first healthy response.
|
|
10
30
|
*/
|
|
11
|
-
export async function waitForHealth({health, host, port}) {
|
|
31
|
+
export async function waitForHealth({health, host, port, processId = "proxied"}) {
|
|
32
|
+
const startedAtMs = Date.now()
|
|
33
|
+
|
|
12
34
|
if (health.startDelayMs > 0) {
|
|
13
|
-
await
|
|
35
|
+
await sleep(health.startDelayMs)
|
|
14
36
|
}
|
|
15
37
|
|
|
16
|
-
const
|
|
38
|
+
const deadlineMs = health.timeoutMs
|
|
39
|
+
const deadline = Date.now() + deadlineMs
|
|
17
40
|
const url = `http://${host}:${port}${health.path}`
|
|
41
|
+
let attempts = 0
|
|
18
42
|
let lastError = "no attempts"
|
|
19
43
|
|
|
20
44
|
while (Date.now() <= deadline) {
|
|
45
|
+
attempts += 1
|
|
46
|
+
|
|
21
47
|
try {
|
|
22
48
|
const response = await fetch(url)
|
|
23
49
|
|
|
24
|
-
if (response.ok) return
|
|
50
|
+
if (response.ok) return {elapsedMs: Date.now() - startedAtMs}
|
|
25
51
|
|
|
26
52
|
lastError = `HTTP ${response.status}`
|
|
27
53
|
} catch (error) {
|
|
28
54
|
lastError = error instanceof Error ? error.message : String(error)
|
|
29
55
|
}
|
|
30
56
|
|
|
31
|
-
await
|
|
57
|
+
await sleep(health.intervalMs)
|
|
32
58
|
}
|
|
33
59
|
|
|
34
|
-
|
|
60
|
+
const elapsedMs = Date.now() - startedAtMs
|
|
61
|
+
/** @type {HealthCheckError} */
|
|
62
|
+
const failure = new Error(`Health check failed for ${url}: ${lastError} (process ${processId}: ${attempts} probes, ${elapsedMs}ms elapsed of the ${deadlineMs}ms health deadline)`)
|
|
63
|
+
|
|
64
|
+
failure.healthCheck = {attempts, deadlineMs, elapsedMs, processId, url}
|
|
65
|
+
|
|
66
|
+
throw failure
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Waits until a TCP endpoint accepts a connection, on a budget of its own.
|
|
71
|
+
*
|
|
72
|
+
* This is the dependency leg of a release start: waiting for a process that must bind a
|
|
73
|
+
* port before another one can boot. Charged to the health deadline it measures the
|
|
74
|
+
* dependency instead of the app, so it is timed here on a separate budget.
|
|
75
|
+
* @param {object} args - Options.
|
|
76
|
+
* @param {string} args.host - Host to connect to.
|
|
77
|
+
* @param {number} args.intervalMs - Delay between connection attempts.
|
|
78
|
+
* @param {number} args.port - Port that must accept a connection.
|
|
79
|
+
* @param {number} args.timeoutMs - Budget for the wait.
|
|
80
|
+
* @returns {Promise<{attempts: number, elapsedMs: number}>} Time and attempts spent waiting.
|
|
81
|
+
*/
|
|
82
|
+
export async function waitForListeningPort({host, intervalMs, port, timeoutMs}) {
|
|
83
|
+
const startedAtMs = Date.now()
|
|
84
|
+
const deadline = startedAtMs + timeoutMs
|
|
85
|
+
let attempts = 0
|
|
86
|
+
|
|
87
|
+
while (Date.now() <= deadline) {
|
|
88
|
+
attempts += 1
|
|
89
|
+
|
|
90
|
+
if (await tryConnect({deadline, host, intervalMs, port})) {
|
|
91
|
+
return {attempts, elapsedMs: Date.now() - startedAtMs}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
await sleep(Math.max(1, Math.min(intervalMs, deadline - Date.now())))
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
throw new Error(`Dependency endpoint ${host}:${port} did not accept a connection within ${timeoutMs}ms (${attempts} attempts, ${Date.now() - startedAtMs}ms waited)`)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* @param {object} args - Attempt options.
|
|
102
|
+
* @param {number} args.deadline - Absolute deadline that bounds this attempt.
|
|
103
|
+
* @param {string} args.host - Host to connect to.
|
|
104
|
+
* @param {number} args.intervalMs - Delay between connection attempts.
|
|
105
|
+
* @param {number} args.port - Port to connect to.
|
|
106
|
+
* @returns {Promise<boolean>} True when the endpoint accepted the connection.
|
|
107
|
+
*/
|
|
108
|
+
function tryConnect({deadline, host, intervalMs, port}) {
|
|
109
|
+
return new Promise((resolve) => {
|
|
110
|
+
const socket = net.createConnection({host, port})
|
|
111
|
+
let settled = false
|
|
112
|
+
|
|
113
|
+
/** @param {boolean} connected - Whether the endpoint accepted the connection. */
|
|
114
|
+
const finish = (connected) => {
|
|
115
|
+
if (settled) return
|
|
116
|
+
settled = true
|
|
117
|
+
socket.destroy()
|
|
118
|
+
resolve(connected)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
socket.setTimeout(Math.max(1, Math.min(intervalMs, deadline - Date.now())))
|
|
122
|
+
socket.once("connect", () => finish(true))
|
|
123
|
+
socket.once("error", () => finish(false))
|
|
124
|
+
socket.once("timeout", () => finish(false))
|
|
125
|
+
})
|
|
35
126
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @typedef {import("./config.js").ProcessConfig} ProcessConfig
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const POLICY_START_RANK = /** @type {Record<ProcessConfig["policy"], number>} */ ({companion: 1, proxied: 2, service: 0, singleton: 3})
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Order in which Rollbridge starts processes, which is also the only order a `dependsOn`
|
|
11
|
+
* edge may point backwards in: daemon-wide persistent services are started before every
|
|
12
|
+
* release generation, then handoff services, companions, and the proxied process last.
|
|
13
|
+
* Config order is kept inside each group.
|
|
14
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
15
|
+
* @param {{includePersistentServices?: boolean}} [options] - Whether to include the
|
|
16
|
+
* daemon-wide persistent services that a release group never starts itself.
|
|
17
|
+
* @returns {ProcessConfig[]} Ordered process configs.
|
|
18
|
+
*/
|
|
19
|
+
export function processStartOrder(processes, {includePersistentServices = false} = {}) {
|
|
20
|
+
const persistentServices = includePersistentServices
|
|
21
|
+
? processes.filter((processConfig) => processConfig.policy === "service" && processConfig.deployStrategy === "persistent")
|
|
22
|
+
: []
|
|
23
|
+
const generationProcesses = processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
|
|
24
|
+
|
|
25
|
+
return [...persistentServices, ...generationProcesses.sort((left, right) => POLICY_START_RANK[left.policy] - POLICY_START_RANK[right.policy])]
|
|
26
|
+
}
|
package/src/release-group.js
CHANGED
|
@@ -4,8 +4,11 @@ import {spawn} from "node:child_process"
|
|
|
4
4
|
import {EventEmitter} from "node:events"
|
|
5
5
|
import ManagedProcess from "./managed-process.js"
|
|
6
6
|
import {findAvailablePort} from "./port-allocator.js"
|
|
7
|
+
import {processStartOrder} from "./process-order.js"
|
|
7
8
|
import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
|
|
8
|
-
import {waitForHealth} from "./health.js"
|
|
9
|
+
import {waitForHealth, waitForListeningPort} from "./health.js"
|
|
10
|
+
|
|
11
|
+
const DEPENDENCY_READY_INTERVAL_MS = 250
|
|
9
12
|
|
|
10
13
|
/**
|
|
11
14
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -13,6 +16,7 @@ import {waitForHealth} from "./health.js"
|
|
|
13
16
|
* @typedef {{http: number, websocket: number}} ReleaseConnections
|
|
14
17
|
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
15
18
|
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
|
|
19
|
+
* @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
|
|
16
20
|
*/
|
|
17
21
|
|
|
18
22
|
|
|
@@ -57,6 +61,14 @@ function envId(id) {
|
|
|
57
61
|
return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
|
|
58
62
|
}
|
|
59
63
|
|
|
64
|
+
/**
|
|
65
|
+
* @param {number | undefined} startedAtMs - When the measured phase began.
|
|
66
|
+
* @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
|
|
67
|
+
*/
|
|
68
|
+
function elapsedSince(startedAtMs) {
|
|
69
|
+
return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
|
|
70
|
+
}
|
|
71
|
+
|
|
60
72
|
/**
|
|
61
73
|
* @param {import("./config.js").ProcessConfig} processConfig - Process config.
|
|
62
74
|
* @param {number} index - Zero-based replica index.
|
|
@@ -107,16 +119,27 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
107
119
|
this.stoppedAt = /** @type {string | undefined} */ (undefined)
|
|
108
120
|
this.retirementError = /** @type {string | undefined} */ (undefined)
|
|
109
121
|
this.preserveConfigOnRetirement = false
|
|
122
|
+
this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
|
|
123
|
+
this.readyProcessIds = /** @type {Set<string>} */ (new Set())
|
|
124
|
+
this.startedProcessIds = /** @type {Set<string>} */ (new Set())
|
|
125
|
+
this.startupDependencyWaitMs = 0
|
|
126
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
|
|
127
|
+
this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
|
|
110
128
|
}
|
|
111
129
|
|
|
112
130
|
/** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
|
|
113
131
|
async start() {
|
|
114
132
|
this.state = "starting"
|
|
133
|
+
this.startupStartedAtMs = Date.now()
|
|
115
134
|
|
|
116
135
|
try {
|
|
117
136
|
await this.allocatePorts()
|
|
118
137
|
|
|
119
138
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
139
|
+
await this.waitForConfiguredDependencies(processConfig)
|
|
140
|
+
/** @type {string[]} */
|
|
141
|
+
const instanceIds = []
|
|
142
|
+
|
|
120
143
|
for (let index = 0; index < processConfig.replicas; index += 1) {
|
|
121
144
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
122
145
|
|
|
@@ -127,17 +150,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
127
150
|
if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
|
|
128
151
|
if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
|
|
129
152
|
await processInstance.start("deploy")
|
|
153
|
+
this.processLaunchAtMs.set(instanceId, Date.now())
|
|
154
|
+
this.startedProcessIds.add(instanceId)
|
|
155
|
+
instanceIds.push(instanceId)
|
|
130
156
|
}
|
|
131
157
|
|
|
132
158
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
133
|
-
await
|
|
134
|
-
health: processConfig.health,
|
|
135
|
-
host: this.config.proxy.upstreamHost,
|
|
136
|
-
port: this.ports[processConfig.id]
|
|
137
|
-
})
|
|
138
|
-
|
|
139
|
-
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
159
|
+
await this.waitForProxiedHealth(processConfig)
|
|
140
160
|
}
|
|
161
|
+
|
|
162
|
+
for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
|
|
163
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
141
164
|
}
|
|
142
165
|
} catch (error) {
|
|
143
166
|
this.state = "failed"
|
|
@@ -257,16 +280,132 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
257
280
|
}
|
|
258
281
|
|
|
259
282
|
/**
|
|
260
|
-
*
|
|
283
|
+
* Waits for the ports a process declares it depends on, on a budget that is not the
|
|
284
|
+
* health budget. The dependency leg of a start is not app boot, and charging it to the
|
|
285
|
+
* health deadline measures the dependency instead of the app.
|
|
286
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
|
|
287
|
+
* @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
|
|
288
|
+
*/
|
|
289
|
+
async waitForConfiguredDependencies(processConfig) {
|
|
290
|
+
if (processConfig.dependsOn.length === 0) return
|
|
291
|
+
|
|
292
|
+
for (const dependencyId of processConfig.dependsOn) {
|
|
293
|
+
const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
|
|
294
|
+
|
|
295
|
+
if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
|
|
296
|
+
|
|
297
|
+
this.logger("release process dependency wait", {
|
|
298
|
+
dependencyPort: port,
|
|
299
|
+
dependencyProcessId: dependencyId,
|
|
300
|
+
processId: processConfig.id,
|
|
301
|
+
releaseId: this.releaseId,
|
|
302
|
+
timeoutMs: processConfig.dependencyTimeoutMs
|
|
303
|
+
})
|
|
304
|
+
|
|
305
|
+
const waitStartedAtMs = Date.now()
|
|
306
|
+
|
|
307
|
+
try {
|
|
308
|
+
const {attempts, elapsedMs} = await waitForListeningPort({
|
|
309
|
+
host: this.config.proxy.upstreamHost,
|
|
310
|
+
intervalMs: DEPENDENCY_READY_INTERVAL_MS,
|
|
311
|
+
port,
|
|
312
|
+
timeoutMs: processConfig.dependencyTimeoutMs
|
|
313
|
+
})
|
|
314
|
+
|
|
315
|
+
this.startupDependencyWaitMs += elapsedMs
|
|
316
|
+
this.logger("release process dependency ready", {
|
|
317
|
+
attempts,
|
|
318
|
+
dependencyPort: port,
|
|
319
|
+
dependencyProcessId: dependencyId,
|
|
320
|
+
elapsedMs,
|
|
321
|
+
processId: processConfig.id,
|
|
322
|
+
releaseId: this.releaseId
|
|
323
|
+
})
|
|
324
|
+
} catch (error) {
|
|
325
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
326
|
+
deadlineMs: processConfig.dependencyTimeoutMs,
|
|
327
|
+
dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
|
|
328
|
+
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
329
|
+
failedProcessId: processConfig.id,
|
|
330
|
+
probes: undefined,
|
|
331
|
+
stage: "dependency"
|
|
332
|
+
})
|
|
333
|
+
throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Health checks the proxied process on its own boot budget and records what the boot cost.
|
|
342
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
|
|
343
|
+
* @returns {Promise<void>} Resolves once the process answers its health path.
|
|
344
|
+
*/
|
|
345
|
+
async waitForProxiedHealth(processConfig) {
|
|
346
|
+
const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
|
|
347
|
+
const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
|
|
348
|
+
|
|
349
|
+
try {
|
|
350
|
+
const {elapsedMs} = await waitForHealth({
|
|
351
|
+
health,
|
|
352
|
+
host: this.config.proxy.upstreamHost,
|
|
353
|
+
port: /** @type {number} */ (this.ports[processConfig.id]),
|
|
354
|
+
processId: processConfig.id
|
|
355
|
+
})
|
|
356
|
+
|
|
357
|
+
this.logger("release candidate health ready", {
|
|
358
|
+
bootMs: Date.now() - launchedAtMs,
|
|
359
|
+
deadlineMs: health.timeoutMs,
|
|
360
|
+
healthGateMs: elapsedMs,
|
|
361
|
+
path: health.path,
|
|
362
|
+
port: this.ports[processConfig.id],
|
|
363
|
+
processId: processConfig.id,
|
|
364
|
+
releaseId: this.releaseId,
|
|
365
|
+
waitedForDependenciesMs: this.startupDependencyWaitMs
|
|
366
|
+
})
|
|
367
|
+
} catch (error) {
|
|
368
|
+
const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
|
|
369
|
+
|
|
370
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
371
|
+
deadlineMs: health.timeoutMs,
|
|
372
|
+
dependencyWaitMs: this.startupDependencyWaitMs,
|
|
373
|
+
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
374
|
+
failedProcessId: processConfig.id,
|
|
375
|
+
probes: healthCheck?.attempts,
|
|
376
|
+
stage: "health"
|
|
377
|
+
})
|
|
378
|
+
throw error
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
|
|
261
384
|
* @param {Error | string} error - Startup failure.
|
|
262
385
|
* @param {{phase: string}} options - Diagnostic phase.
|
|
263
386
|
* @returns {void}
|
|
264
387
|
*/
|
|
265
388
|
logStartupFailure(error, {phase}) {
|
|
389
|
+
const diagnostics = this.startupFailureDiagnostics
|
|
390
|
+
const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
|
|
391
|
+
const lastLog = failedLogs?.at(-1)
|
|
392
|
+
const readyProcesses = [...this.readyProcessIds]
|
|
393
|
+
|
|
266
394
|
this.logger("release startup failed", {
|
|
395
|
+
deadlineMs: diagnostics?.deadlineMs ?? null,
|
|
396
|
+
dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
|
|
397
|
+
elapsedMs: diagnostics?.elapsedMs ?? null,
|
|
267
398
|
error: error instanceof Error ? error.message : error,
|
|
399
|
+
failedProcessId: diagnostics?.failedProcessId ?? null,
|
|
400
|
+
lastOutput: lastLog?.line ?? null,
|
|
401
|
+
lastOutputAt: lastLog?.at ?? null,
|
|
268
402
|
phase,
|
|
269
|
-
|
|
403
|
+
probes: diagnostics?.probes ?? null,
|
|
404
|
+
readyProcesses,
|
|
405
|
+
releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
406
|
+
releaseId: this.releaseId,
|
|
407
|
+
stage: diagnostics?.stage ?? null,
|
|
408
|
+
unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
|
|
270
409
|
})
|
|
271
410
|
|
|
272
411
|
for (const processInstance of this.processes.values()) {
|
|
@@ -291,12 +430,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
291
430
|
* @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
|
|
292
431
|
*/
|
|
293
432
|
releaseProcessStartOrder() {
|
|
294
|
-
|
|
295
|
-
const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
|
|
296
|
-
const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
|
|
297
|
-
const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
|
|
298
|
-
|
|
299
|
-
return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
|
|
433
|
+
return processStartOrder(this.config.processes)
|
|
300
434
|
}
|
|
301
435
|
|
|
302
436
|
/** @returns {void} Marks this release active. */
|
|
@@ -367,24 +501,25 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
367
501
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
368
502
|
this.assertCommittedGenerationRecoverable()
|
|
369
503
|
this.state = "starting"
|
|
504
|
+
this.startupStartedAtMs = Date.now()
|
|
370
505
|
try {
|
|
371
506
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
507
|
+
await this.waitForConfiguredDependencies(processConfig)
|
|
372
508
|
const instances = this.getProcesses(processConfig.id)
|
|
373
509
|
|
|
374
510
|
if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
|
|
375
|
-
for (const {process} of instances) {
|
|
511
|
+
for (const {id, process} of instances) {
|
|
376
512
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
377
513
|
await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
|
|
514
|
+
this.processLaunchAtMs.set(id, Date.now())
|
|
515
|
+
this.startedProcessIds.add(id)
|
|
378
516
|
}
|
|
379
517
|
|
|
380
518
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
381
|
-
await
|
|
382
|
-
health: processConfig.health,
|
|
383
|
-
host: this.config.proxy.upstreamHost,
|
|
384
|
-
port: this.ports[processConfig.id]
|
|
385
|
-
})
|
|
519
|
+
await this.waitForProxiedHealth(processConfig)
|
|
386
520
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
387
521
|
}
|
|
522
|
+
for (const {id} of instances) this.readyProcessIds.add(id)
|
|
388
523
|
}
|
|
389
524
|
} catch (error) {
|
|
390
525
|
await this.abortCommittedGenerationRestart()
|
|
@@ -587,6 +587,59 @@ test("validateConfig defaults health.startDelayMs to 0, accepts an override, and
|
|
|
587
587
|
expect(negative.issues.some((issue) => issue.message === "processes[0].health.startDelayMs must be a non-negative number")).toBeTruthy()
|
|
588
588
|
})
|
|
589
589
|
|
|
590
|
+
test("validateConfig defaults dependsOn and its own wait budget, and rejects dependencies it cannot wait on", () => {
|
|
591
|
+
/**
|
|
592
|
+
* @param {import("../src/json.js").JsonValue} processes - Processes under test.
|
|
593
|
+
* @returns {{config: import("../src/config.js").RollbridgeConfig, issues: import("../src/config.js").ConfigIssue[]}} Validation result.
|
|
594
|
+
*/
|
|
595
|
+
const validateProcesses = (processes) => validateConfig({
|
|
596
|
+
application: "demo",
|
|
597
|
+
control: {path: "/tmp/demo.sock"},
|
|
598
|
+
processes,
|
|
599
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
600
|
+
})
|
|
601
|
+
|
|
602
|
+
const base = [
|
|
603
|
+
{command: "run beacon", id: "beacon", policy: "service", port: 7330},
|
|
604
|
+
{command: "run jobs", dependsOn: ["beacon"], id: "jobs", policy: "service", port: {from: 7331, to: 7399}},
|
|
605
|
+
{command: "run web", dependsOn: ["jobs"], id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
|
|
606
|
+
]
|
|
607
|
+
const valid = validateProcesses(base)
|
|
608
|
+
|
|
609
|
+
expect(valid.issues).toEqual([])
|
|
610
|
+
expect(valid.config.processes[0].dependsOn).toEqual([])
|
|
611
|
+
expect(valid.config.processes[1].dependsOn).toEqual(["beacon"])
|
|
612
|
+
expect(valid.config.processes[1].dependencyTimeoutMs).toBe(30000)
|
|
613
|
+
|
|
614
|
+
const custom = validateProcesses([...base.slice(0, 2), {...base[2], dependencyTimeoutMs: 60000}])
|
|
615
|
+
|
|
616
|
+
expect(custom.issues).toEqual([])
|
|
617
|
+
expect(custom.config.processes[2].dependencyTimeoutMs).toBe(60000)
|
|
618
|
+
|
|
619
|
+
const invalid = validateProcesses([
|
|
620
|
+
{command: "run web", dependsOn: ["missing", "web", "jobs", "worker"], dependencyTimeoutMs: 0, id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
621
|
+
{command: "run jobs", id: "jobs", policy: "singleton"},
|
|
622
|
+
{command: "run worker", id: "worker", policy: "companion"}
|
|
623
|
+
])
|
|
624
|
+
const messages = invalid.issues.map((issue) => issue.message)
|
|
625
|
+
|
|
626
|
+
expect(messages.some((message) => message === 'Process "web" depends on unknown process "missing"')).toBeTruthy()
|
|
627
|
+
expect(messages.some((message) => message === 'Process "web" cannot depend on itself')).toBeTruthy()
|
|
628
|
+
expect(messages.some((message) => message === 'Process "web" cannot depend on singleton "jobs"')).toBeTruthy()
|
|
629
|
+
expect(messages.some((message) => message === 'Process "worker" has no port, so "web" cannot wait for it')).toBeTruthy()
|
|
630
|
+
expect(messages.some((message) => message === "processes[0].dependencyTimeoutMs must be a positive number")).toBeTruthy()
|
|
631
|
+
|
|
632
|
+
const backwards = validateProcesses([
|
|
633
|
+
{command: "run web", dependsOn: "web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
634
|
+
{command: "run worker", dependsOn: ["web"], id: "worker", policy: "companion", port: 7001}
|
|
635
|
+
])
|
|
636
|
+
const backwardsMessages = backwards.issues.map((issue) => issue.message)
|
|
637
|
+
|
|
638
|
+
expect(backwardsMessages.some((message) => message === "processes[0].dependsOn must be an array")).toBeTruthy()
|
|
639
|
+
expect(backwardsMessages.some((message) => message === 'Process "worker" depends on "web", which Rollbridge starts after it')).toBeTruthy()
|
|
640
|
+
expect(backwards.issues.every((issue) => typeof issue.fix === "string" && issue.fix.length > 0)).toBe(true)
|
|
641
|
+
})
|
|
642
|
+
|
|
590
643
|
test("validateConfig defaults releaseRetention, accepts overrides, and rejects bad values", () => {
|
|
591
644
|
/**
|
|
592
645
|
* @param {import("../src/json.js").JsonValue} releaseRetention - Retention config under test, or undefined.
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import http from "node:http"
|
|
4
|
+
import net from "node:net"
|
|
5
|
+
|
|
6
|
+
const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
|
|
7
|
+
const bootMs = Number(process.env.ROLLBRIDGE_BOOT_MS || "0")
|
|
8
|
+
const dependencyPort = process.env.ROLLBRIDGE_DEPENDENCY_PORT
|
|
9
|
+
const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
|
|
10
|
+
|
|
11
|
+
log(`starting (boot budget ${bootMs}ms)`)
|
|
12
|
+
|
|
13
|
+
if (dependencyPort) {
|
|
14
|
+
// The app performs its own dependency wait, exactly as the production web command does.
|
|
15
|
+
await waitForDependency(Number(dependencyPort))
|
|
16
|
+
log(`dependency ${dependencyPort} accepted a connection`)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
await sleep(bootMs)
|
|
20
|
+
|
|
21
|
+
const server = http.createServer((request, response) => {
|
|
22
|
+
if (request.url === "/ping") {
|
|
23
|
+
response.writeHead(200, {"Content-Type": "application/json"})
|
|
24
|
+
response.end(JSON.stringify({message: "Pong", releaseId}))
|
|
25
|
+
return
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
|
|
29
|
+
response.end(`${releaseId}\n`)
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
process.on("SIGTERM", () => {
|
|
33
|
+
server.close(() => process.exit(0))
|
|
34
|
+
setTimeout(() => process.exit(0), 10).unref()
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
server.listen(port, "127.0.0.1")
|
|
38
|
+
log(`listening on ${port}`)
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* @param {number} dependencyPortNumber - Port of the process this app depends on.
|
|
42
|
+
* @returns {Promise<void>} Resolves once the dependency accepts a connection.
|
|
43
|
+
*/
|
|
44
|
+
async function waitForDependency(dependencyPortNumber) {
|
|
45
|
+
const deadline = Date.now() + 10000
|
|
46
|
+
|
|
47
|
+
while (Date.now() < deadline) {
|
|
48
|
+
if (await tryConnect(dependencyPortNumber)) return
|
|
49
|
+
await sleep(50)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
log(`dependency ${dependencyPortNumber} never accepted a connection`)
|
|
53
|
+
process.exit(21)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @param {number} dependencyPortNumber - Port to connect to.
|
|
58
|
+
* @returns {Promise<boolean>} True when the dependency accepted the connection.
|
|
59
|
+
*/
|
|
60
|
+
function tryConnect(dependencyPortNumber) {
|
|
61
|
+
return new Promise((resolve) => {
|
|
62
|
+
const socket = net.createConnection({host: "127.0.0.1", port: dependencyPortNumber})
|
|
63
|
+
let settled = false
|
|
64
|
+
|
|
65
|
+
/** @param {boolean} connected - Whether the connection succeeded. */
|
|
66
|
+
const finish = (connected) => {
|
|
67
|
+
if (settled) return
|
|
68
|
+
settled = true
|
|
69
|
+
socket.destroy()
|
|
70
|
+
resolve(connected)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
socket.setTimeout(200)
|
|
74
|
+
socket.once("connect", () => finish(true))
|
|
75
|
+
socket.once("error", () => finish(false))
|
|
76
|
+
socket.once("timeout", () => finish(false))
|
|
77
|
+
})
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* @param {number} ms - Milliseconds.
|
|
82
|
+
* @returns {Promise<void>} Resolves after the delay.
|
|
83
|
+
*/
|
|
84
|
+
function sleep(ms) {
|
|
85
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* @param {string} line - Output line, so the failure record can name the app's last output.
|
|
90
|
+
* @returns {void}
|
|
91
|
+
*/
|
|
92
|
+
function log(line) {
|
|
93
|
+
console.log(`boot: ${process.pid} ${line}`)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* @param {string} key - Environment variable name.
|
|
98
|
+
* @returns {string} Environment variable value.
|
|
99
|
+
*/
|
|
100
|
+
function requiredEnv(key) {
|
|
101
|
+
const value = process.env[key]
|
|
102
|
+
|
|
103
|
+
if (!value) throw new Error(`${key} is required`)
|
|
104
|
+
|
|
105
|
+
return value
|
|
106
|
+
}
|
package/test/health.test.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import http from "node:http"
|
|
4
|
+
import net from "node:net"
|
|
4
5
|
import {describe, expect, test} from "@velocious/testing"
|
|
5
|
-
import {waitForHealth} from "../src/health.js"
|
|
6
|
+
import {waitForHealth, waitForListeningPort} from "../src/health.js"
|
|
6
7
|
|
|
7
8
|
describe("health", () => {
|
|
8
9
|
|
|
@@ -31,6 +32,18 @@ async function startHealthServer() {
|
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
|
|
35
|
+
/**
|
|
36
|
+
* @param {net.Server} server - Server to bind.
|
|
37
|
+
* @returns {Promise<number>} Ephemeral port it bound.
|
|
38
|
+
*/
|
|
39
|
+
async function bindPort(server) {
|
|
40
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(undefined)))
|
|
41
|
+
|
|
42
|
+
const address = server.address()
|
|
43
|
+
|
|
44
|
+
return address && typeof address === "object" ? address.port : 0
|
|
45
|
+
}
|
|
46
|
+
|
|
34
47
|
test("waitForHealth delays the first probe by startDelayMs", async () => {
|
|
35
48
|
const server = await startHealthServer()
|
|
36
49
|
|
|
@@ -62,4 +75,64 @@ test("waitForHealth probes immediately when startDelayMs is 0", async () => {
|
|
|
62
75
|
await server.close()
|
|
63
76
|
}
|
|
64
77
|
})
|
|
78
|
+
|
|
79
|
+
test("waitForHealth names the deadline and the time it spent when the app never becomes healthy", async () => {
|
|
80
|
+
const server = http.createServer((_request, response) => {
|
|
81
|
+
response.writeHead(500)
|
|
82
|
+
response.end("bad")
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
try {
|
|
86
|
+
const port = await bindPort(server)
|
|
87
|
+
|
|
88
|
+
/** @type {import("../src/health.js").HealthCheckError} */
|
|
89
|
+
const failure = /** @type {import("../src/health.js").HealthCheckError} */ (await waitForHealth({
|
|
90
|
+
health: {intervalMs: 25, path: "/ping", startDelayMs: 0, timeoutMs: 300},
|
|
91
|
+
host: "127.0.0.1",
|
|
92
|
+
port,
|
|
93
|
+
processId: "web"
|
|
94
|
+
}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
95
|
+
|
|
96
|
+
expect(String(failure.message)).toMatch(/process web: \d+ probes, \d+ms elapsed of the 300ms health deadline/)
|
|
97
|
+
expect(failure.healthCheck?.deadlineMs).toBe(300)
|
|
98
|
+
expect({value: Boolean(failure.healthCheck && failure.healthCheck.elapsedMs >= 300), context: `expected the health gate to have spent its whole budget, was ${failure.healthCheck?.elapsedMs}ms`}).toMatchObject({value: true})
|
|
99
|
+
expect({value: Boolean(failure.healthCheck && failure.healthCheck.attempts >= 1), context: `expected at least one probe, was ${failure.healthCheck?.attempts}`}).toMatchObject({value: true})
|
|
100
|
+
expect(failure.healthCheck?.processId).toBe("web")
|
|
101
|
+
} finally {
|
|
102
|
+
await server.close()
|
|
103
|
+
}
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
test("waitForListeningPort waits for a dependency port on a budget of its own", async () => {
|
|
107
|
+
const server = net.createServer()
|
|
108
|
+
const port = await bindPort(server)
|
|
109
|
+
|
|
110
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
111
|
+
|
|
112
|
+
const waiting = waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 2000})
|
|
113
|
+
|
|
114
|
+
await new Promise((resolve) => setTimeout(resolve, 300))
|
|
115
|
+
await new Promise((resolve) => server.listen(port, "127.0.0.1", () => resolve(undefined)))
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const result = await waiting
|
|
119
|
+
|
|
120
|
+
expect({value: Boolean(result.attempts >= 2), context: `expected more than one connection attempt, was ${result.attempts}`}).toMatchObject({value: true})
|
|
121
|
+
expect({value: Boolean(result.elapsedMs >= 250), context: `expected the dependency wait to take ~300ms, was ${result.elapsedMs}ms`}).toMatchObject({value: true})
|
|
122
|
+
} finally {
|
|
123
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
124
|
+
}
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
test("waitForListeningPort names the budget it ran out on when the dependency never binds", async () => {
|
|
128
|
+
const server = net.createServer()
|
|
129
|
+
const port = await bindPort(server)
|
|
130
|
+
|
|
131
|
+
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
132
|
+
|
|
133
|
+
/** @type {Error} */
|
|
134
|
+
const failure = /** @type {Error} */ (await waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 200}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
135
|
+
|
|
136
|
+
expect(String(failure.message)).toMatch(/did not accept a connection within 200ms/)
|
|
137
|
+
})
|
|
65
138
|
})
|
package/test/rollbridge.test.js
CHANGED
|
@@ -19,6 +19,7 @@ describe("rollbridge", () => {
|
|
|
19
19
|
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
|
20
20
|
const binPath = path.join(currentDir, "..", "bin", "rollbridge")
|
|
21
21
|
const dependentAppPath = path.join(currentDir, "fixtures", "dependent-app.js")
|
|
22
|
+
const deferredBootAppPath = path.join(currentDir, "fixtures", "deferred-boot-app.js")
|
|
22
23
|
const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
|
|
23
24
|
const memoryHogPath = path.join(currentDir, "fixtures", "memory-hog.js")
|
|
24
25
|
const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
|
|
@@ -243,6 +244,98 @@ test("failed release startup logs process output and cleanup status", async () =
|
|
|
243
244
|
}
|
|
244
245
|
})
|
|
245
246
|
|
|
247
|
+
test("the dependency wait is charged to its own budget, not to the proxied boot budget", async () => {
|
|
248
|
+
const fixture = await createFixture()
|
|
249
|
+
const config = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 1200, webBootMs: 600, webHealthTimeoutMs: 900})
|
|
250
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
251
|
+
const logs = []
|
|
252
|
+
const daemon = new RollbridgeDaemon({config, logger: (message, data = {}) => logs.push({data, message})})
|
|
253
|
+
|
|
254
|
+
await daemon.start()
|
|
255
|
+
|
|
256
|
+
try {
|
|
257
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
258
|
+
|
|
259
|
+
const dependencyLog = logs.find((entry) => entry.message === "release process dependency ready")
|
|
260
|
+
const healthLog = logs.find((entry) => entry.message === "release candidate health ready")
|
|
261
|
+
|
|
262
|
+
if (!dependencyLog?.data || !healthLog?.data) throw new Error("expected the dependency wait and the boot time to be logged")
|
|
263
|
+
// The dependency took ~1200ms and the app boot ~600ms. The boot budget is 900ms, so this
|
|
264
|
+
// deploy only succeeds because the dependency wait is charged to a budget of its own.
|
|
265
|
+
expect(Number(dependencyLog.data.elapsedMs)).toBeGreaterThanOrEqual(1000)
|
|
266
|
+
expect(Number(healthLog.data.bootMs)).toBeLessThan(900)
|
|
267
|
+
expect(Number(healthLog.data.waitedForDependenciesMs)).toBeGreaterThanOrEqual(1000)
|
|
268
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
269
|
+
} finally {
|
|
270
|
+
await daemon.shutdown()
|
|
271
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
272
|
+
}
|
|
273
|
+
})
|
|
274
|
+
|
|
275
|
+
test("a dependency that never opens fails the deploy on its own budget and keeps the incumbent serving", async () => {
|
|
276
|
+
const fixture = await createFixture()
|
|
277
|
+
const initialConfig = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 0, webHealthTimeoutMs: 900})
|
|
278
|
+
const configPath = await writeConfigFile(initialConfig, fixture.root)
|
|
279
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
280
|
+
const logs = []
|
|
281
|
+
const daemon = new RollbridgeDaemon({config: initialConfig, configPath, logger: (message, data = {}) => logs.push({data, message})})
|
|
282
|
+
|
|
283
|
+
await daemon.start()
|
|
284
|
+
|
|
285
|
+
try {
|
|
286
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
287
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
288
|
+
|
|
289
|
+
// v2's service never binds its port, so the dependency gate runs out before the app is started.
|
|
290
|
+
await writeConfigFile(deferredBootConfig(fixture, {dependencyTimeoutMs: 300, serviceBootMs: 60000, webBootMs: 0, webHealthTimeoutMs: 900}), fixture.root)
|
|
291
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/did not accept a connection within 300ms/)
|
|
292
|
+
|
|
293
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
294
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
295
|
+
|
|
296
|
+
const failure = logs.find((entry) => entry.message === "release startup failed" && entry.data?.phase === "before cleanup")
|
|
297
|
+
|
|
298
|
+
if (!failure?.data) throw new Error("expected the dependency failure to name its budget")
|
|
299
|
+
expect(failure.data.stage).toBe("dependency")
|
|
300
|
+
expect(failure.data.deadlineMs).toBe(300)
|
|
301
|
+
expect(Number(failure.data.dependencyWaitMs)).toBeGreaterThanOrEqual(300)
|
|
302
|
+
expect(failure.data.readyProcesses).toEqual(["beacon"])
|
|
303
|
+
expect(failure.data.unreadyProcesses).toEqual([])
|
|
304
|
+
} finally {
|
|
305
|
+
await daemon.shutdown()
|
|
306
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
307
|
+
}
|
|
308
|
+
})
|
|
309
|
+
|
|
310
|
+
test("a startup timeout names the deadline, the elapsed boot, the ready processes, and the last output", async () => {
|
|
311
|
+
const fixture = await createFixture()
|
|
312
|
+
const config = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 1200, webHealthTimeoutMs: 400})
|
|
313
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
314
|
+
const logs = []
|
|
315
|
+
const daemon = new RollbridgeDaemon({config, logger: (message, data = {}) => logs.push({data, message})})
|
|
316
|
+
|
|
317
|
+
await daemon.start()
|
|
318
|
+
|
|
319
|
+
try {
|
|
320
|
+
await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/health deadline/)
|
|
321
|
+
|
|
322
|
+
const failure = logs.find((entry) => entry.message === "release startup failed" && entry.data?.phase === "before cleanup")
|
|
323
|
+
|
|
324
|
+
if (!failure?.data) throw new Error("expected the startup failure to name its budget")
|
|
325
|
+
expect(failure.data.stage).toBe("health")
|
|
326
|
+
expect(failure.data.deadlineMs).toBe(400)
|
|
327
|
+
expect(Number(failure.data.elapsedMs)).toBeGreaterThanOrEqual(400)
|
|
328
|
+
expect(Number(failure.data.probes)).toBeGreaterThanOrEqual(1)
|
|
329
|
+
expect(failure.data.readyProcesses).toEqual(["beacon"])
|
|
330
|
+
expect(failure.data.unreadyProcesses).toEqual(["web"])
|
|
331
|
+
expect(String(failure.data.lastOutput)).toMatch(/accepted a connection/)
|
|
332
|
+
expect(String(failure.data.lastOutputAt)).toMatch(/^\d{4}-\d{2}-\d{2}T/)
|
|
333
|
+
} finally {
|
|
334
|
+
await daemon.shutdown()
|
|
335
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
336
|
+
}
|
|
337
|
+
})
|
|
338
|
+
|
|
246
339
|
test("singleton processes restart without overlap during deploy", async () => {
|
|
247
340
|
const fixture = await createFixture({includeSingleton: true})
|
|
248
341
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -2199,6 +2292,41 @@ async function createFixture(options = {}) {
|
|
|
2199
2292
|
return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
|
|
2200
2293
|
}
|
|
2201
2294
|
|
|
2295
|
+
/**
|
|
2296
|
+
* Config where both processes run the deferred-boot fixture, so a measured boot budget can be
|
|
2297
|
+
* compared against a dependency wait that must not be charged to it.
|
|
2298
|
+
* @param {{config: import("../src/config.js").RollbridgeConfig, root: string}} fixture - Fixture.
|
|
2299
|
+
* @param {{dependencyTimeoutMs: number, serviceBootMs: number, webBootMs: number, webHealthTimeoutMs: number}} options - Budgets.
|
|
2300
|
+
* @returns {import("../src/config.js").RollbridgeConfig} Config for the deferred-boot processes.
|
|
2301
|
+
*/
|
|
2302
|
+
function deferredBootConfig(fixture, options) {
|
|
2303
|
+
const appCommand = `${JSON.stringify(process.execPath)} ${JSON.stringify(deferredBootAppPath)}`
|
|
2304
|
+
|
|
2305
|
+
return normalizeConfig({
|
|
2306
|
+
...fixture.config,
|
|
2307
|
+
processes: [
|
|
2308
|
+
{
|
|
2309
|
+
command: appCommand,
|
|
2310
|
+
deployStrategy: "handoff",
|
|
2311
|
+
env: {ROLLBRIDGE_BOOT_MS: String(options.serviceBootMs)},
|
|
2312
|
+
id: "beacon",
|
|
2313
|
+
policy: "service",
|
|
2314
|
+
port: {from: 15000, to: 15099}
|
|
2315
|
+
},
|
|
2316
|
+
{
|
|
2317
|
+
command: appCommand,
|
|
2318
|
+
dependsOn: ["beacon"],
|
|
2319
|
+
dependencyTimeoutMs: options.dependencyTimeoutMs,
|
|
2320
|
+
env: {ROLLBRIDGE_BOOT_MS: String(options.webBootMs), ROLLBRIDGE_DEPENDENCY_PORT: "{{ports.beacon}}"},
|
|
2321
|
+
health: {intervalMs: 50, path: "/ping", timeoutMs: options.webHealthTimeoutMs},
|
|
2322
|
+
id: "web",
|
|
2323
|
+
policy: "proxied",
|
|
2324
|
+
port: {from: 0, to: 0}
|
|
2325
|
+
}
|
|
2326
|
+
]
|
|
2327
|
+
})
|
|
2328
|
+
}
|
|
2329
|
+
|
|
2202
2330
|
/**
|
|
2203
2331
|
* @param {string} lifecycleLogPath - Fixture lifecycle event file.
|
|
2204
2332
|
* @returns {Promise<string[]>} Ordered lifecycle events.
|