rollbridge 0.1.59 → 0.1.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/docs/config.md +42 -2
- package/docs/troubleshooting.md +22 -6
- package/docs/velocious.md +1 -1
- package/examples/tensorbuzz.com.js +1 -1
- package/package.json +1 -1
- package/src/config.js +104 -3
- package/src/health.js +99 -8
- package/src/process-order.js +26 -0
- package/src/release-group.js +157 -22
- package/tensorbuzz.yml +17 -0
- package/test/config-validation.test.js +102 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/health.test.js +74 -1
- package/test/owner-replacement.test.js +22 -2
- package/test/rollbridge.test.js +202 -14
- package/test/tensorbuzz-config.test.js +228 -0
package/README.md
CHANGED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
### Added
|
|
2
|
+
|
|
3
|
+
- `processes[].dependsOn` names the same-release processes a process needs, and
|
|
4
|
+
Rollbridge now waits for each dependency's allocated port **before** spawning
|
|
5
|
+
the dependent, on `processes[].dependencyTimeoutMs` (default 30 seconds) — a
|
|
6
|
+
budget of its own that is never charged to `health.timeoutMs`. Each wait is
|
|
7
|
+
logged as `release process dependency wait` and `release process dependency
|
|
8
|
+
ready` with `attempts` and `elapsedMs`. `validate` rejects a dependency on
|
|
9
|
+
itself, an unknown process, a `singleton`, a process with no `port`, and a
|
|
10
|
+
process Rollbridge starts after the dependent, which also rules out cycles.
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
|
|
14
|
+
- A release-scoped process no longer spends the proxied health deadline waiting
|
|
15
|
+
for a dependency. Measured on production 2026-11-06: `background-jobs-main`
|
|
16
|
+
took 11 817 ms to start listening (14:33:39.175 → 14:33:50.992) and the web
|
|
17
|
+
command's own `wait-port` hop then ate part of the same 30 000 ms that
|
|
18
|
+
`proxy.healthTimeoutMs` and `health.timeoutMs` allowed for the whole start
|
|
19
|
+
sequence, leaving the app about 16 seconds to boot before Rollbridge killed a
|
|
20
|
+
release that was still starting normally. The health gate now measures only
|
|
21
|
+
the proxied process's own boot and reports `bootMs`, `healthGateMs`, and
|
|
22
|
+
`waitedForDependenciesMs` in `release candidate health ready`.
|
|
23
|
+
- A failed release start is diagnostic. The health failure now reads
|
|
24
|
+
`Health check failed for <url>: <reason> (process <id>: 18 probes, 941ms
|
|
25
|
+
elapsed of the 900ms health deadline)` instead of naming only the connection
|
|
26
|
+
error, and `release startup failed` carries `deadlineMs`, `dependencyWaitMs`,
|
|
27
|
+
`elapsedMs`, `releaseElapsedMs`, `stage` (`dependency` or `health`), `probes`,
|
|
28
|
+
`failedProcessId`, `readyProcesses`, `unreadyProcesses`, and the failed
|
|
29
|
+
process's `lastOutput`/`lastOutputAt`. A dependency gate that expires fails
|
|
30
|
+
the deploy as `Process <id> could not start because dependency <id> was not
|
|
31
|
+
ready: …` on its own budget; the health gate is unchanged, so a genuinely
|
|
32
|
+
unhealthy release still fails and the incumbent keeps serving.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
### Added
|
|
2
|
+
|
|
3
|
+
- `tensorbuzz.yml` declares a `paths` filter for the CI lane: `"**"` with a quoted
|
|
4
|
+
`!**/*.md` negation, so a delivery that changes only Markdown is recorded as a
|
|
5
|
+
terminal `skipped` build instead of installing the dependency tree and running
|
|
6
|
+
`npm run all-checks`. 31 of the repository's 114 tracked paths are Markdown and
|
|
7
|
+
no gate input is among them — `test/tensorbuzz-config.test.js` asserts that
|
|
8
|
+
partition over every tracked path, that every negation is quoted, and that the
|
|
9
|
+
catch-all stays first so a later rule cannot re-admit what a negation carved out.
|
|
10
|
+
`tensorbuzz.yml` itself, an unavailable or capped changed-file list, and a manual
|
|
11
|
+
rebuild still run the lane.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
### Changed
|
|
2
|
+
|
|
3
|
+
- `proxy.healthTimeoutMs` now defaults to **120000** instead of 30000, so a release
|
|
4
|
+
gets a boot budget above the measured production startup without every project
|
|
5
|
+
having to configure one. Measured on TensorBuzz production (server3) 2026-11-06:
|
|
6
|
+
release `20261106142602` started its `web` process at 14:33:39.205, was still
|
|
7
|
+
printing `attachment storage root migration` lines at 14:34:08.842, and was
|
|
8
|
+
SIGTERMed at 14:34:09.492 without ever logging a listening line — the app's own
|
|
9
|
+
boot alone exceeded the 30 000 ms deadline, after the dependency wait had already
|
|
10
|
+
consumed about 11 817 ms of it. Release `20261106144454` was killed 15 373 ms after
|
|
11
|
+
its `web` process started, before reaching those lines. 120 000 ms is the budget the
|
|
12
|
+
same release has published on since `config/rollbridge.production.js` set it on
|
|
13
|
+
2026-11-06.
|
|
14
|
+
- A process that configures no `health` block inherits that budget from
|
|
15
|
+
`proxy.healthTimeoutMs`; setting either level still wins over the default. The
|
|
16
|
+
documentation examples now show 120 000 rather than the value that failed.
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- A release whose app needs more than 30 seconds to boot publishes while an
|
|
21
|
+
incumbent generation is live. `test/rollbridge.test.js` now deploys a candidate
|
|
22
|
+
that answers `/ping` 32 seconds after launch on the default budget and asserts
|
|
23
|
+
`release candidate health ready` records `bootMs`, `healthGateMs`, and
|
|
24
|
+
`deadlineMs: 120000`, with the incumbent still serving `/release` during the boot.
|
|
25
|
+
The health gate is not weakened: a candidate that never becomes healthy still
|
|
26
|
+
fails the deploy when its deadline expires and the incumbent keeps serving.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
### Fixed
|
|
2
|
+
|
|
3
|
+
- `test/owner-replacement.test.js` "failed partial upgrade resumes an incumbent retired release
|
|
4
|
+
drain" no longer fails on a busy runner. Its whole-fixture budget was `5000ms` while the scenario
|
|
5
|
+
spawns six child processes (daemon, a `worker` and a proxied `web` for each of two releases, plus a
|
|
6
|
+
partial owner-replacement guardian) and then waits for the retired drain to resume, so its cost
|
|
7
|
+
tracked host contention instead of the behaviour under test. Measured in the same invocation:
|
|
8
|
+
1.457s at load average 8.3 and 7.105s at load average 24.5, both with the drain completing — the
|
|
9
|
+
cap sat between the two, which is why the spec reported `Error: Timed out after 5000ms` naming no
|
|
10
|
+
stage while nothing in the drain stalled. The budget is now `PARTIAL_DRAIN_FIXTURE_TIMEOUT_MS`
|
|
11
|
+
(20000ms, 2.8x the measured worst case) with the measurement cited at its declaration.
|
|
12
|
+
- The same spec now asserts the resumed drain in the retired release's own status record
|
|
13
|
+
(`drainStartedAt` and `stoppedAt`, ordered) instead of only the stdout line its wait is keyed on,
|
|
14
|
+
so a release cleared without draining its processes cannot pass. No product behavior changed and no
|
|
15
|
+
health gate was weakened: the waits stay event-gated, so a release that never resumes its drain
|
|
16
|
+
still fails.
|
|
17
|
+
- `test/rollbridge.test.js` "candidate activation quiesces the old jobs generation before a blocked
|
|
18
|
+
singleton replacement completes" no longer depends on inotify to reach its assertions. It awaited an
|
|
19
|
+
`fs.watch` event with no initial read and no fallback, so in a runtime whose filesystem delivers no
|
|
20
|
+
events it hung to the runner's ambient 60s default and named no stage. Measured here: a minimal
|
|
21
|
+
`fs.watch` receives 0 events for a file written by an external process, on overlayfs `/tmp` and on
|
|
22
|
+
ext3 `/data`, with `max_user_watches` at 1048576 — the container, not the product. The wait is now on
|
|
23
|
+
the durable marker the old generation writes (`SINGLETON_QUIET_MARKER_TIMEOUT_MS = 20000`, spec
|
|
24
|
+
measured at 1.740s), naming the marker path if it ever expires. The three assertions are unchanged
|
|
25
|
+
and now execute everywhere. The suite's other `fs.watch` helpers (`waitForState`,
|
|
26
|
+
`waitForLifecycleEvent`) already read the file before watching, which is why only this spec was
|
|
27
|
+
affected.
|
package/docs/config.md
CHANGED
|
@@ -77,7 +77,7 @@ to let a deploy group talk to the daemon.
|
|
|
77
77
|
| `proxy.port` | number | `8182` | Stable port Nginx (or another front end) points at. |
|
|
78
78
|
| `proxy.upstreamHost` | string | `proxy.host`, or `"127.0.0.1"` when `proxy.host` is `0.0.0.0`/`::` | Host Rollbridge uses for release health checks and proxy targets. |
|
|
79
79
|
| `proxy.healthPath` | string | `"/ping"` | Default health-check path for proxied processes. |
|
|
80
|
-
| `proxy.healthTimeoutMs` | number | `
|
|
80
|
+
| `proxy.healthTimeoutMs` | number | `120000` | Default boot budget for proxied processes: how long a release may take to answer its health path for the first time. Measured on TensorBuzz production (server3, 2026-11-06): the web app was still initialising when a `30000` deadline expired, so two deploys were killed mid-boot; `120000` is the budget the same release publishes on today. Waiting for a `dependsOn` service is not charged to it. |
|
|
81
81
|
| `proxy.drainTimeoutMs` | number | `60000` | How long to drain HTTP/WebSocket connections before stopping the retired proxied process. Expiry never stops an independently draining jobs generation. |
|
|
82
82
|
| `proxy.forceStopTimeoutMs` | number | `10000` | Default per-process graceful-stop timeout (`SIGTERM`, then `SIGKILL`). |
|
|
83
83
|
|
|
@@ -233,6 +233,8 @@ legacyTakeover: {
|
|
|
233
233
|
| `env` | object of string → string | `{}` | Extra environment variables (values templated). Merged over the injected `ROLLBRIDGE_*` vars. |
|
|
234
234
|
| `port` | number or `{from, to}` | unset | Port (or range) allocated per release. **Required for the `proxied` process.** A plain number `n` means the fixed port `n` (`{from: n, to: n}`). |
|
|
235
235
|
| `health` | object or `false` | enabled with defaults | Health check for the `proxied` process; set `false` to disable (see below). |
|
|
236
|
+
| `dependsOn` | array of process ids | `[]` | Processes whose allocated port must accept a connection before Rollbridge starts this process (see [`processes[].dependsOn`](#processesdependson)). |
|
|
237
|
+
| `dependencyTimeoutMs` | positive number | `30000` | Budget for each `dependsOn` wait. Charged to its own clock, never to `health.timeoutMs`. |
|
|
236
238
|
| `stopSignal` | signal name (e.g. `"SIGTERM"`, `"SIGINT"`, `"SIGQUIT"`) | `"SIGTERM"` | Signal sent to gracefully stop the process; after `gracefulStopMs` it is `SIGKILL`ed. Use a worker's quit signal so it finishes in-flight work before exiting. |
|
|
237
239
|
| `nonBlockingDrain` | boolean | `false` | When a release is retired, drain this process **immediately** (in parallel with the proxied connection drain) instead of after it. Companion processes only — typically background workers (see below). |
|
|
238
240
|
| `lifecycle` | object | no hooks | Command hooks run when gracefully stopping the process (see below). |
|
|
@@ -443,10 +445,47 @@ release). Set `health: false` to disable it.
|
|
|
443
445
|
| Field | Type | Default | Description |
|
|
444
446
|
| --- | --- | --- | --- |
|
|
445
447
|
| `health.path` | string | `proxy.healthPath` | HTTP path probed on the process's port. |
|
|
446
|
-
| `health.timeoutMs` | number | `proxy.healthTimeoutMs` | Total time to wait for the first healthy response. |
|
|
448
|
+
| `health.timeoutMs` | number | `proxy.healthTimeoutMs` | Total time to wait for the first healthy response, measured from the process launch. Only the app's own boot is charged to it — a `dependsOn` wait runs on `dependencyTimeoutMs` before this window opens. |
|
|
447
449
|
| `health.intervalMs` | number | `250` | Delay between probes. |
|
|
448
450
|
| `health.startDelayMs` | non-negative number | `0` | Wait this long after the process starts before the first probe (runs before the `timeoutMs` window). |
|
|
449
451
|
|
|
452
|
+
### `processes[].dependsOn`
|
|
453
|
+
|
|
454
|
+
A release-scoped process that talks to a same-release service usually waits for
|
|
455
|
+
that service inside its own command (`wait-port … && exec <app>`). That wait
|
|
456
|
+
runs after Rollbridge has started the process, so it is charged against
|
|
457
|
+
`health.timeoutMs`: a dependency that takes 12 seconds to bind leaves the app 18
|
|
458
|
+
seconds of a 30-second health deadline, and the deploy fails while the app is
|
|
459
|
+
still booting normally.
|
|
460
|
+
|
|
461
|
+
Declare the dependency instead, and Rollbridge waits for each `dependsOn`
|
|
462
|
+
process's allocated port **before** spawning the dependent, on a budget of its
|
|
463
|
+
own (`dependencyTimeoutMs`):
|
|
464
|
+
|
|
465
|
+
```js
|
|
466
|
+
{
|
|
467
|
+
id: "web",
|
|
468
|
+
policy: "proxied",
|
|
469
|
+
dependsOn: ["background-jobs-main"],
|
|
470
|
+
dependencyTimeoutMs: 30000,
|
|
471
|
+
command: "exec node build/bin/velocious.js server --port {{port}}",
|
|
472
|
+
port: {from: 14500, to: 14599},
|
|
473
|
+
health: {path: "/ping", timeoutMs: 120000}
|
|
474
|
+
}
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
`health.timeoutMs` then measures only the app's own boot. Both waits are logged
|
|
478
|
+
separately: `release process dependency wait` / `release process dependency
|
|
479
|
+
ready` (with `attempts` and `elapsedMs`) and `release candidate health ready`
|
|
480
|
+
(with `bootMs`, `healthGateMs`, and `waitedForDependenciesMs`).
|
|
481
|
+
|
|
482
|
+
`dependsOn` accepts ids of processes Rollbridge starts before this one —
|
|
483
|
+
persistent services, handoff services, companions, and the proxied process. A
|
|
484
|
+
dependency must have a `port` (there is nothing to wait for otherwise), may not
|
|
485
|
+
be a `singleton` (singleton ownership is not per-release), and may not be this
|
|
486
|
+
process itself. Dependencies that Rollbridge would start after this process are
|
|
487
|
+
rejected, which also rules out cycles.
|
|
488
|
+
|
|
450
489
|
## `compose[]`
|
|
451
490
|
|
|
452
491
|
Optional Docker Compose services this app updates. Each entry is a unit Rollbridge
|
|
@@ -548,6 +587,7 @@ Rollbridge sets these in every managed process's environment (the process's own
|
|
|
548
587
|
- `replicas` must be a positive integer; `replicas > 1` is allowed only on a `companion` process without a `port`. Process ids must not contain `#` (reserved for replica instance ids).
|
|
549
588
|
- `lifecycle.activateCommand`/`quietCommand`/`drainCommand`/`stopCommand` must be non-empty strings when set, and `lifecycle.drainTimeoutMs` a non-negative number; `lifecycle.drainCommand` requires a positive `lifecycle.drainTimeoutMs`. `activateCommand` is allowed on at most one handoff service, requires that service's `quietCommand`, and requires `statePath` plus `ownerRecovery`. A `lifecycle.stopCommand` may not be combined with a custom `stopSignal` (the `stopCommand` runs instead of the signal, so the signal would be ignored).
|
|
550
589
|
- `nonBlockingDrain` must be a boolean, and is allowed only on a `companion` process.
|
|
590
|
+
- `dependsOn` must be an array of configured process ids: not the process itself, not a `singleton`, not a process without a `port`, and not a process Rollbridge starts after it. `dependencyTimeoutMs` must be a positive number.
|
|
551
591
|
- `statePath` must be a string when set.
|
|
552
592
|
- `compose` must be a non-empty array of unit mappings. Each unit needs `id`, `service`, `projectDirectory`, and a valid `image` reference (a tag, or a `@sha256:` digest with 64 hex characters; no whitespace, no leading `-`).
|
|
553
593
|
- Compose unit `id`s must be unique and must not collide with a process `id`.
|
package/docs/troubleshooting.md
CHANGED
|
@@ -54,14 +54,30 @@ Health check failed for http://127.0.0.1:18182/ping: HTTP 503
|
|
|
54
54
|
new release never went live; the previous release stays active.
|
|
55
55
|
|
|
56
56
|
**Diagnose.** The new release's `proxied` process didn't return a healthy
|
|
57
|
-
response in time.
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
57
|
+
response in time. The reason names what it spent that budget on —
|
|
58
|
+
`(process web: 18 probes, 941ms elapsed of the 900ms health deadline)` — and the
|
|
59
|
+
`release startup failed` event carries `deadlineMs`, `elapsedMs`,
|
|
60
|
+
`dependencyWaitMs`, `probes`, `stage` (`"dependency"` or `"health"`),
|
|
61
|
+
`failedProcessId`, `readyProcesses`/`unreadyProcesses`, and the process's
|
|
62
|
+
`lastOutput`. Check that process's output with `rollbridge logs --process <id>`
|
|
63
|
+
and its state/`exitCode` with `rollbridge status`. Common causes: the app doesn't
|
|
64
|
+
listen on the templated `{{port}}`, the `health.path` returns a non-2xx status, or
|
|
65
|
+
the app boots slower than `health.timeoutMs`.
|
|
66
|
+
|
|
67
|
+
If the boot itself is fine but the app waits for a same-release service inside
|
|
68
|
+
its own command, that wait is inside the health deadline. `stage: "dependency"`
|
|
69
|
+
plus a `release process dependency ready` event shows the wait Rollbridge did on
|
|
70
|
+
the dependency's own budget.
|
|
61
71
|
|
|
62
72
|
**Fix.** Make the proxied command bind `{{port}}` and serve `health.path` with a
|
|
63
|
-
2xx status.
|
|
64
|
-
|
|
73
|
+
2xx status. The boot budget is `proxy.healthTimeoutMs` (`120000` by default), which
|
|
74
|
+
every process inherits unless it sets `health.timeoutMs` itself, and it measures only
|
|
75
|
+
the app's own boot. For an app that needs longer than the default, raise
|
|
76
|
+
`health.timeoutMs` above the measured boot time (or set `health.startDelayMs` so
|
|
77
|
+
probing begins after the app is up). For a
|
|
78
|
+
same-release service dependency, declare `dependsOn` (see
|
|
79
|
+
[`processes[].dependsOn`](config.md#processesdependson)) so the wait runs on
|
|
80
|
+
`dependencyTimeoutMs` instead of eating the boot budget.
|
|
65
81
|
|
|
66
82
|
## Port conflicts / exhausted ranges
|
|
67
83
|
|
package/docs/velocious.md
CHANGED
package/package.json
CHANGED
package/src/config.js
CHANGED
|
@@ -6,6 +6,7 @@ import {createRequire} from "node:module"
|
|
|
6
6
|
import os from "node:os"
|
|
7
7
|
import path from "node:path"
|
|
8
8
|
import {pathToFileURL} from "node:url"
|
|
9
|
+
import {processStartOrder} from "./process-order.js"
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -17,7 +18,7 @@ import {pathToFileURL} from "node:url"
|
|
|
17
18
|
* @typedef {{activateCommand?: string, activateTimeoutMs?: number, drainCommand?: string, drainTimeoutMs: number, quietCommand?: string, reactivateCommand?: string, stopCommand?: string}} LifecycleConfig
|
|
18
19
|
* @typedef {number | "indefinite"} StopTimeoutMs
|
|
19
20
|
* @typedef {"persistent" | "handoff"} ServiceDeployStrategy
|
|
20
|
-
* @typedef {{cwd?: string, deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
|
|
21
|
+
* @typedef {{cwd?: string, dependencyTimeoutMs: number, dependsOn: string[], deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
|
|
21
22
|
* @typedef {{group?: number | string, mode?: number, owner?: number | string, path: string}} ControlConfig
|
|
22
23
|
* @typedef {{drainTimeoutMs: number, forceStopTimeoutMs: number, healthPath: string, healthTimeoutMs: number, host: string, port: number, upstreamHost: string}} ProxyConfig
|
|
23
24
|
* @typedef {{includes: string[], name: string}} LegacyTakeoverProcessConfig
|
|
@@ -35,6 +36,25 @@ import {pathToFileURL} from "node:url"
|
|
|
35
36
|
|
|
36
37
|
const PROCESS_POLICIES = new Set(["proxied", "companion", "singleton", "service"])
|
|
37
38
|
const DEFAULT_CONFIG_FILENAMES = ["rollbridge.js"]
|
|
39
|
+
// Budget for one process waiting for a dependency's port, separate from the health budget.
|
|
40
|
+
// Measured on TensorBuzz production (server3, 2026-11-06): the candidate's
|
|
41
|
+
// background-jobs-main took 11817ms (14:33:39.175 start -> 14:33:50.992 listening) before
|
|
42
|
+
// the web process could reach its own boot, and the production command chain already waited
|
|
43
|
+
// 30000ms per hop. 30000ms keeps that dependency budget, so the health deadline measures
|
|
44
|
+
// only the app boot.
|
|
45
|
+
const DEFAULT_DEPENDENCY_TIMEOUT_MS = 30000
|
|
46
|
+
// Budget for one process to answer its health path for the first time, measured from its
|
|
47
|
+
// launch. Measured on TensorBuzz production (server3, 2026-11-06): the 30000ms release
|
|
48
|
+
// health deadline killed two candidate web processes mid-boot. Release 20261106142602
|
|
49
|
+
// started web at 14:33:39.205, was still printing attachment storage root migration lines
|
|
50
|
+
// at 14:34:08.842, and got SIGTERM at 14:34:09.492 without ever logging a listening line,
|
|
51
|
+
// so the app's own boot alone exceeded 30000ms; release 20261106144454 was killed 15373ms
|
|
52
|
+
// after its web process started, before reaching those lines. 120000ms is the budget the
|
|
53
|
+
// same app has been published on since (config/rollbridge.production.js, merged
|
|
54
|
+
// 2026-11-06) and is made the default so a slow boot is not a trap every project has to
|
|
55
|
+
// know to configure. A candidate that never becomes healthy still fails the deploy at the
|
|
56
|
+
// deadline while the incumbent keeps serving; the budget only moves when the gate reports.
|
|
57
|
+
const DEFAULT_HEALTH_TIMEOUT_MS = 120000
|
|
38
58
|
const commonJsRequire = createRequire(import.meta.url)
|
|
39
59
|
|
|
40
60
|
/**
|
|
@@ -204,7 +224,7 @@ function normalizeProxy(source, issues) {
|
|
|
204
224
|
drainTimeoutMs: normalizeNumber(source.drainTimeoutMs, "proxy.drainTimeoutMs", issues, {default: 60000}),
|
|
205
225
|
forceStopTimeoutMs: normalizeNumber(source.forceStopTimeoutMs, "proxy.forceStopTimeoutMs", issues, {default: 10000}),
|
|
206
226
|
healthPath: normalizeString(source.healthPath, "proxy.healthPath", issues, {default: "/ping"}),
|
|
207
|
-
healthTimeoutMs: normalizeNumber(source.healthTimeoutMs, "proxy.healthTimeoutMs", issues, {default:
|
|
227
|
+
healthTimeoutMs: normalizeNumber(source.healthTimeoutMs, "proxy.healthTimeoutMs", issues, {default: DEFAULT_HEALTH_TIMEOUT_MS}),
|
|
208
228
|
host,
|
|
209
229
|
port: normalizeNumber(source.port, "proxy.port", issues, {default: 8182}),
|
|
210
230
|
upstreamHost: normalizeString(source.upstreamHost, "proxy.upstreamHost", issues, {default: defaultUpstreamHost(host)})
|
|
@@ -232,7 +252,7 @@ function normalizeProcess(value, index, proxy, issues) {
|
|
|
232
252
|
if (!isPlainObject(value)) {
|
|
233
253
|
issues.push({fix: `Define processes[${index}] as a mapping with id, policy, and command.`, message: `processes[${index}] must be an object`})
|
|
234
254
|
|
|
235
|
-
return {command: "", cwd: undefined, deployStrategy: "persistent", env: {}, gracefulStopMs: proxy.forceStopTimeoutMs, health: undefined, id: "", lifecycle: {drainTimeoutMs: 0}, memory: undefined, nonBlockingDrain: false, outputLines: 50, policy: "companion", port: undefined, replicas: 1, restart: defaultRestartConfig(), restartDelayMs: 1000, stopSignal: "SIGTERM"}
|
|
255
|
+
return {command: "", cwd: undefined, dependencyTimeoutMs: DEFAULT_DEPENDENCY_TIMEOUT_MS, dependsOn: [], deployStrategy: "persistent", env: {}, gracefulStopMs: proxy.forceStopTimeoutMs, health: undefined, id: "", lifecycle: {drainTimeoutMs: 0}, memory: undefined, nonBlockingDrain: false, outputLines: 50, policy: "companion", port: undefined, replicas: 1, restart: defaultRestartConfig(), restartDelayMs: 1000, stopSignal: "SIGTERM"}
|
|
236
256
|
}
|
|
237
257
|
|
|
238
258
|
const source = value
|
|
@@ -240,6 +260,8 @@ function normalizeProcess(value, index, proxy, issues) {
|
|
|
240
260
|
return {
|
|
241
261
|
command: normalizeString(source.command, `processes[${index}].command`, issues),
|
|
242
262
|
cwd: source.cwd === undefined ? undefined : normalizeString(source.cwd, `processes[${index}].cwd`, issues),
|
|
263
|
+
dependencyTimeoutMs: normalizeDependencyTimeoutMs(source.dependencyTimeoutMs, `processes[${index}].dependencyTimeoutMs`, issues),
|
|
264
|
+
dependsOn: normalizeDependsOn(source.dependsOn, `processes[${index}].dependsOn`, issues),
|
|
243
265
|
deployStrategy: normalizeDeployStrategy(source.deployStrategy, `processes[${index}].deployStrategy`, issues),
|
|
244
266
|
env: normalizeEnv(source.env, `processes[${index}].env`, issues),
|
|
245
267
|
gracefulStopMs: normalizeStopTimeout(source.gracefulStopMs, `processes[${index}].gracefulStopMs`, proxy.forceStopTimeoutMs, issues),
|
|
@@ -765,6 +787,34 @@ function normalizeDeployStrategy(value, key, issues) {
|
|
|
765
787
|
return "persistent"
|
|
766
788
|
}
|
|
767
789
|
|
|
790
|
+
/**
|
|
791
|
+
* @param {JsonValue} value - Raw dependency list.
|
|
792
|
+
* @param {string} key - Config key.
|
|
793
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
794
|
+
* @returns {string[]} Ids of the processes this process waits for before it starts.
|
|
795
|
+
*/
|
|
796
|
+
function normalizeDependsOn(value, key, issues) {
|
|
797
|
+
return normalizeStringList(value, key, issues)
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
/**
|
|
801
|
+
* @param {JsonValue} value - Raw dependency wait budget.
|
|
802
|
+
* @param {string} key - Config key.
|
|
803
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
804
|
+
* @returns {number} Milliseconds this process may spend waiting for its dependency ports.
|
|
805
|
+
*/
|
|
806
|
+
function normalizeDependencyTimeoutMs(value, key, issues) {
|
|
807
|
+
const timeoutMs = normalizeNumber(value, key, issues, {default: DEFAULT_DEPENDENCY_TIMEOUT_MS})
|
|
808
|
+
|
|
809
|
+
if (timeoutMs < 1) {
|
|
810
|
+
issues.push({fix: `Set ${key} to a positive number of milliseconds, e.g. 30000, or omit it for the default.`, message: `${key} must be a positive number`})
|
|
811
|
+
|
|
812
|
+
return DEFAULT_DEPENDENCY_TIMEOUT_MS
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
return timeoutMs
|
|
816
|
+
}
|
|
817
|
+
|
|
768
818
|
/**
|
|
769
819
|
* @param {JsonValue} value - Raw output retention value.
|
|
770
820
|
* @param {string} key - Config key.
|
|
@@ -949,6 +999,57 @@ function validateProcessSet(processes, issues) {
|
|
|
949
999
|
|
|
950
1000
|
issues.push({fix: `Add a port range to the proxied process "${processConfig.id || "(unnamed)"}", e.g. port: {from: 18000, to: 18099}.`, message: `Proxied process ${processConfig.id || "(unnamed)"} must define a port range`})
|
|
951
1001
|
}
|
|
1002
|
+
|
|
1003
|
+
validateProcessDependencies(processes, issues)
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
/**
|
|
1007
|
+
* Validates `dependsOn`: a dependency must name a ported process that Rollbridge starts
|
|
1008
|
+
* before the dependent. Waiting is done on the dependency's allocated port, and an edge that
|
|
1009
|
+
* points backwards in the start order can never be satisfied, which also rules out cycles.
|
|
1010
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
1011
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
1012
|
+
* @returns {void}
|
|
1013
|
+
*/
|
|
1014
|
+
function validateProcessDependencies(processes, issues) {
|
|
1015
|
+
const declared = new Map(processes.filter((processConfig) => processConfig.id).map((processConfig) => [processConfig.id, processConfig]))
|
|
1016
|
+
const startOrder = new Map(processStartOrder(processes, {includePersistentServices: true}).map((processConfig, index) => [processConfig.id, index]))
|
|
1017
|
+
|
|
1018
|
+
for (const processConfig of processes) {
|
|
1019
|
+
if (!processConfig.id || processConfig.dependsOn.length === 0) continue
|
|
1020
|
+
|
|
1021
|
+
const dependentIndex = startOrder.get(processConfig.id)
|
|
1022
|
+
|
|
1023
|
+
for (const dependencyId of processConfig.dependsOn) {
|
|
1024
|
+
if (dependencyId === processConfig.id) {
|
|
1025
|
+
issues.push({fix: `Remove "${processConfig.id}" from its own dependsOn.`, message: `Process "${processConfig.id}" cannot depend on itself`})
|
|
1026
|
+
continue
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
const dependency = declared.get(dependencyId)
|
|
1030
|
+
|
|
1031
|
+
if (!dependency) {
|
|
1032
|
+
issues.push({fix: `Name a configured process in "${processConfig.id}".dependsOn; "${dependencyId}" is not one of: ${[...declared.keys()].join(", ")}.`, message: `Process "${processConfig.id}" depends on unknown process "${dependencyId}"`})
|
|
1033
|
+
continue
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
if (dependency.policy === "singleton") {
|
|
1037
|
+
issues.push({fix: `Remove "${dependencyId}" from "${processConfig.id}".dependsOn; a singleton is not part of a release generation, so Rollbridge cannot gate a release start on it.`, message: `Process "${processConfig.id}" cannot depend on singleton "${dependencyId}"`})
|
|
1038
|
+
continue
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
if (!dependency.port) {
|
|
1042
|
+
issues.push({fix: `Give "${dependencyId}" a port, or remove it from "${processConfig.id}".dependsOn; Rollbridge waits for a dependency on its allocated port.`, message: `Process "${dependencyId}" has no port, so "${processConfig.id}" cannot wait for it`})
|
|
1043
|
+
continue
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
const dependencyIndex = startOrder.get(dependencyId)
|
|
1047
|
+
|
|
1048
|
+
if (dependencyIndex === undefined || dependentIndex === undefined || dependencyIndex > dependentIndex) {
|
|
1049
|
+
issues.push({fix: `Move "${dependencyId}" before "${processConfig.id}" in processes[]; Rollbridge starts persistent services, then handoff services, companions, and the proxied process.`, message: `Process "${processConfig.id}" depends on "${dependencyId}", which Rollbridge starts after it`})
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
952
1053
|
}
|
|
953
1054
|
|
|
954
1055
|
/**
|
package/src/health.js
CHANGED
|
@@ -1,35 +1,126 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
+
import net from "node:net"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @typedef {import("./config.js").HealthConfig} HealthConfig
|
|
7
|
+
* @typedef {{attempts: number, deadlineMs: number, elapsedMs: number, processId: string, url: string}} HealthCheckDiagnostics
|
|
8
|
+
* @typedef {Error & {healthCheck?: HealthCheckDiagnostics}} HealthCheckError
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* @param {number} ms - Milliseconds.
|
|
13
|
+
* @returns {Promise<void>} Resolves after the delay.
|
|
14
|
+
*/
|
|
15
|
+
function sleep(ms) {
|
|
16
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
17
|
+
}
|
|
18
|
+
|
|
3
19
|
/**
|
|
4
20
|
* Waits for a process HTTP health endpoint.
|
|
21
|
+
*
|
|
22
|
+
* The failure names its own budget: `Health check failed ...: fetch failed` cannot tell a
|
|
23
|
+
* slow boot apart from a health deadline that was simply too short.
|
|
5
24
|
* @param {object} args - Options.
|
|
6
25
|
* @param {string} args.host - Host.
|
|
7
26
|
* @param {number} args.port - Port.
|
|
8
|
-
* @param {
|
|
9
|
-
* @
|
|
27
|
+
* @param {HealthConfig} args.health - Health config.
|
|
28
|
+
* @param {string} [args.processId] - Process the probe belongs to, named in the failure.
|
|
29
|
+
* @returns {Promise<{elapsedMs: number}>} Time spent reaching the first healthy response.
|
|
10
30
|
*/
|
|
11
|
-
export async function waitForHealth({health, host, port}) {
|
|
31
|
+
export async function waitForHealth({health, host, port, processId = "proxied"}) {
|
|
32
|
+
const startedAtMs = Date.now()
|
|
33
|
+
|
|
12
34
|
if (health.startDelayMs > 0) {
|
|
13
|
-
await
|
|
35
|
+
await sleep(health.startDelayMs)
|
|
14
36
|
}
|
|
15
37
|
|
|
16
|
-
const
|
|
38
|
+
const deadlineMs = health.timeoutMs
|
|
39
|
+
const deadline = Date.now() + deadlineMs
|
|
17
40
|
const url = `http://${host}:${port}${health.path}`
|
|
41
|
+
let attempts = 0
|
|
18
42
|
let lastError = "no attempts"
|
|
19
43
|
|
|
20
44
|
while (Date.now() <= deadline) {
|
|
45
|
+
attempts += 1
|
|
46
|
+
|
|
21
47
|
try {
|
|
22
48
|
const response = await fetch(url)
|
|
23
49
|
|
|
24
|
-
if (response.ok) return
|
|
50
|
+
if (response.ok) return {elapsedMs: Date.now() - startedAtMs}
|
|
25
51
|
|
|
26
52
|
lastError = `HTTP ${response.status}`
|
|
27
53
|
} catch (error) {
|
|
28
54
|
lastError = error instanceof Error ? error.message : String(error)
|
|
29
55
|
}
|
|
30
56
|
|
|
31
|
-
await
|
|
57
|
+
await sleep(health.intervalMs)
|
|
32
58
|
}
|
|
33
59
|
|
|
34
|
-
|
|
60
|
+
const elapsedMs = Date.now() - startedAtMs
|
|
61
|
+
/** @type {HealthCheckError} */
|
|
62
|
+
const failure = new Error(`Health check failed for ${url}: ${lastError} (process ${processId}: ${attempts} probes, ${elapsedMs}ms elapsed of the ${deadlineMs}ms health deadline)`)
|
|
63
|
+
|
|
64
|
+
failure.healthCheck = {attempts, deadlineMs, elapsedMs, processId, url}
|
|
65
|
+
|
|
66
|
+
throw failure
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Waits until a TCP endpoint accepts a connection, on a budget of its own.
|
|
71
|
+
*
|
|
72
|
+
* This is the dependency leg of a release start: waiting for a process that must bind a
|
|
73
|
+
* port before another one can boot. Charged to the health deadline it measures the
|
|
74
|
+
* dependency instead of the app, so it is timed here on a separate budget.
|
|
75
|
+
* @param {object} args - Options.
|
|
76
|
+
* @param {string} args.host - Host to connect to.
|
|
77
|
+
* @param {number} args.intervalMs - Delay between connection attempts.
|
|
78
|
+
* @param {number} args.port - Port that must accept a connection.
|
|
79
|
+
* @param {number} args.timeoutMs - Budget for the wait.
|
|
80
|
+
* @returns {Promise<{attempts: number, elapsedMs: number}>} Time and attempts spent waiting.
|
|
81
|
+
*/
|
|
82
|
+
export async function waitForListeningPort({host, intervalMs, port, timeoutMs}) {
|
|
83
|
+
const startedAtMs = Date.now()
|
|
84
|
+
const deadline = startedAtMs + timeoutMs
|
|
85
|
+
let attempts = 0
|
|
86
|
+
|
|
87
|
+
while (Date.now() <= deadline) {
|
|
88
|
+
attempts += 1
|
|
89
|
+
|
|
90
|
+
if (await tryConnect({deadline, host, intervalMs, port})) {
|
|
91
|
+
return {attempts, elapsedMs: Date.now() - startedAtMs}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
await sleep(Math.max(1, Math.min(intervalMs, deadline - Date.now())))
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
throw new Error(`Dependency endpoint ${host}:${port} did not accept a connection within ${timeoutMs}ms (${attempts} attempts, ${Date.now() - startedAtMs}ms waited)`)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* @param {object} args - Attempt options.
|
|
102
|
+
* @param {number} args.deadline - Absolute deadline that bounds this attempt.
|
|
103
|
+
* @param {string} args.host - Host to connect to.
|
|
104
|
+
* @param {number} args.intervalMs - Delay between connection attempts.
|
|
105
|
+
* @param {number} args.port - Port to connect to.
|
|
106
|
+
* @returns {Promise<boolean>} True when the endpoint accepted the connection.
|
|
107
|
+
*/
|
|
108
|
+
function tryConnect({deadline, host, intervalMs, port}) {
|
|
109
|
+
return new Promise((resolve) => {
|
|
110
|
+
const socket = net.createConnection({host, port})
|
|
111
|
+
let settled = false
|
|
112
|
+
|
|
113
|
+
/** @param {boolean} connected - Whether the endpoint accepted the connection. */
|
|
114
|
+
const finish = (connected) => {
|
|
115
|
+
if (settled) return
|
|
116
|
+
settled = true
|
|
117
|
+
socket.destroy()
|
|
118
|
+
resolve(connected)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
socket.setTimeout(Math.max(1, Math.min(intervalMs, deadline - Date.now())))
|
|
122
|
+
socket.once("connect", () => finish(true))
|
|
123
|
+
socket.once("error", () => finish(false))
|
|
124
|
+
socket.once("timeout", () => finish(false))
|
|
125
|
+
})
|
|
35
126
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @typedef {import("./config.js").ProcessConfig} ProcessConfig
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const POLICY_START_RANK = /** @type {Record<ProcessConfig["policy"], number>} */ ({companion: 1, proxied: 2, service: 0, singleton: 3})
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Order in which Rollbridge starts processes, which is also the only order a `dependsOn`
|
|
11
|
+
* edge may point backwards in: daemon-wide persistent services are started before every
|
|
12
|
+
* release generation, then handoff services, companions, and the proxied process last.
|
|
13
|
+
* Config order is kept inside each group.
|
|
14
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
15
|
+
* @param {{includePersistentServices?: boolean}} [options] - Whether to include the
|
|
16
|
+
* daemon-wide persistent services that a release group never starts itself.
|
|
17
|
+
* @returns {ProcessConfig[]} Ordered process configs.
|
|
18
|
+
*/
|
|
19
|
+
export function processStartOrder(processes, {includePersistentServices = false} = {}) {
|
|
20
|
+
const persistentServices = includePersistentServices
|
|
21
|
+
? processes.filter((processConfig) => processConfig.policy === "service" && processConfig.deployStrategy === "persistent")
|
|
22
|
+
: []
|
|
23
|
+
const generationProcesses = processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
|
|
24
|
+
|
|
25
|
+
return [...persistentServices, ...generationProcesses.sort((left, right) => POLICY_START_RANK[left.policy] - POLICY_START_RANK[right.policy])]
|
|
26
|
+
}
|