rollbridge 0.1.73 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +68 -2
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/daemon.js +411 -78
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +68 -23
- package/src/health.js +99 -8
- package/src/managed-process.js +198 -45
- package/src/process-guardian.js +212 -37
- package/src/process-memory.js +58 -23
- package/src/process-order.js +26 -0
- package/src/release-group.js +438 -45
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +119 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -24
- package/test/owner-recovery.test.js +125 -21
- package/test/owner-replacement.test.js +175 -14
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +408 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/docs/config.md
CHANGED
|
@@ -44,6 +44,8 @@ restart.
|
|
|
44
44
|
| Field | Type | Default | Description |
|
|
45
45
|
| --- | --- | --- | --- |
|
|
46
46
|
| `application` | string | basename of the config file's directory | Names the app; used in the default control-socket path and the `ROLLBRIDGE_APPLICATION` env var. |
|
|
47
|
+
| `compose` | array | unset (no Compose units) | Docker Compose services Rollbridge rolls drain-first (see [`compose[]`](#compose)). |
|
|
48
|
+
| `composeStatePath` | string | `rollbridge-<application>.compose.state.json` next to the config file | File that retains each Compose unit's image generations; relative paths resolve from the config file directory. |
|
|
47
49
|
| `control` | object | — | Control-socket settings (see below). |
|
|
48
50
|
| `legacyTakeover` | object | unset | Optional matchers for `rollbridge predeploy-cleanup` to stop pre-Rollbridge supervisors during first handover (see below). |
|
|
49
51
|
| `ownerRecovery` | object | unset | Durable guardian recovery and atomic incompatible owner replacement; requires `statePath` (see below). |
|
|
@@ -75,8 +77,9 @@ to let a deploy group talk to the daemon.
|
|
|
75
77
|
| `proxy.port` | number | `8182` | Stable port Nginx (or another front end) points at. |
|
|
76
78
|
| `proxy.upstreamHost` | string | `proxy.host`, or `"127.0.0.1"` when `proxy.host` is `0.0.0.0`/`::` | Host Rollbridge uses for release health checks and proxy targets. |
|
|
77
79
|
| `proxy.healthPath` | string | `"/ping"` | Default health-check path for proxied processes. |
|
|
78
|
-
| `proxy.healthTimeoutMs` | number | `
|
|
79
|
-
| `proxy.drainTimeoutMs` | number | `60000` | How long to drain HTTP/WebSocket connections before stopping the retired proxied process. Expiry never stops an independently draining jobs generation. |
|
|
80
|
+
| `proxy.healthTimeoutMs` | number | `120000` | Default boot budget for proxied processes: how long a release may take to answer its health path for the first time. Measured on TensorBuzz production (server3, 2026-11-06): the web app was still initialising when a `30000` deadline expired, so two deploys were killed mid-boot; `120000` is the budget the same release publishes on today. Waiting for a `dependsOn` service is not charged to it. |
|
|
81
|
+
| `proxy.drainTimeoutMs` | number | `60000` | How long to drain HTTP/WebSocket connections before stopping the retired proxied process. Expiry never stops an independently draining jobs generation. It is also the budget a `drain` classification measures elapsed drain against: a generation that still owns connections past it reports `timedOut` rather than `progressing` (see [`status` in the CLI guide](cli.md#status)). |
|
|
82
|
+
| `proxy.nonBlockingDrainTimeoutMs` | number | `60000` | How long release retirement waits for an independently draining process before recording it as still draining and completing the release; `stop` and `rollback` hand the same bound to the processes they must not kill, so neither command can hold the owner-mutation fence for the whole drain. Expiry never sends `SIGKILL`; the process keeps its live work. |
|
|
80
83
|
| `proxy.forceStopTimeoutMs` | number | `10000` | Default per-process graceful-stop timeout (`SIGTERM`, then `SIGKILL`). |
|
|
81
84
|
|
|
82
85
|
## `releaseRetention`
|
|
@@ -85,10 +88,81 @@ to let a deploy group talk to the daemon.
|
|
|
85
88
|
| --- | --- | --- | --- |
|
|
86
89
|
| `releaseRetention.keep` | non-negative integer | `10` | Number of most-recent **stopped** releases the daemon keeps in memory and reports in `status`. |
|
|
87
90
|
| `releaseRetention.maxAgeMs` | non-negative number | `0` (disabled) | Also prune stopped releases older than this many milliseconds. |
|
|
91
|
+
| `releaseRetention.maxConcurrentDrainingReleaseGroups` | non-negative integer | `12` | How many release groups may drain at once. A deploy that would push the count past this bound is **refused before activation** (see below); `0` disables the check. |
|
|
92
|
+
| `releaseRetention.escalateWedgedGenerations` | boolean | `false` | Lets the generation supervisor `SIGKILL` a process group it has **proven** wedged and holding nothing. Off by default: the wedge is reported with its pids instead of terminated. See the escalation below. |
|
|
88
93
|
|
|
89
94
|
Active and draining releases are never pruned. This governs Rollbridge's own
|
|
90
95
|
release records; the deploy tool still owns on-disk release directories.
|
|
91
96
|
|
|
97
|
+
### The draining-generation bound
|
|
98
|
+
|
|
99
|
+
Activating a new generation retires the incumbent, so every deploy adds one draining
|
|
100
|
+
group. When a drain outlasts the deploy cadence, that backlog grows without limit: the
|
|
101
|
+
measured TensorBuzz case had 16 releases draining in 18–114 minutes (median ~37 minutes)
|
|
102
|
+
against deploys every 3–7 minutes, and each further deploy added another group that its own
|
|
103
|
+
post-activation check then reported as a failure.
|
|
104
|
+
|
|
105
|
+
`deploy` therefore counts the groups already draining plus the incumbent it is about to
|
|
106
|
+
retire, and refuses before starting any process of the candidate:
|
|
107
|
+
|
|
108
|
+
```text
|
|
109
|
+
Deploy refused before activation: 13 release groups already draining (20261115063805 wedged
|
|
110
|
+
(1 live process past its retirement budget with no runner child remaining); …). Activating
|
|
111
|
+
20261115073717 would make 14, above the releaseRetention.maxConcurrentDrainingReleaseGroups
|
|
112
|
+
bound of 12.
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
The refusal is a deploy failure like any other: the incumbent keeps serving, nothing of the
|
|
116
|
+
candidate is started, and the named reason says which groups are still holding the fleet and
|
|
117
|
+
whether each is `progressing`, `timedOut`, or `wedged` (see [`status` in the CLI
|
|
118
|
+
guide](cli.md#status)). A refused deploy is retried once a drain finishes; the bound is a
|
|
119
|
+
policy, not a queue.
|
|
120
|
+
|
|
121
|
+
### The wedged-generation escalation
|
|
122
|
+
|
|
123
|
+
A bounded retirement wait (see [`proxy.nonBlockingDrainTimeoutMs`](#proxy)) makes the release
|
|
124
|
+
finish, but it deliberately leaves the wedged process alive: `gracefulStopMs: "indefinite"`
|
|
125
|
+
exists precisely because a stop-timeout `SIGKILL` there aborts live build lanes (measured in
|
|
126
|
+
STOE-1077: 67 builds, 27.3 build-hours). The residue is what then blocks the fleet — the
|
|
127
|
+
process group still holds the generation, and a consumer that requires one live generation per
|
|
128
|
+
role refuses every later deploy until an operator signals the exact PIDs.
|
|
129
|
+
|
|
130
|
+
`releaseRetention.escalateWedgedGenerations: true` hands that decision back to the supervisor.
|
|
131
|
+
It is **off by default**, because only the operator knows whether their worker's live work sits
|
|
132
|
+
in forked runners (kill-safe to prove) or inside the process itself (not provable from procfs).
|
|
133
|
+
|
|
134
|
+
The escalation is not a timer, a poller, or a watchdog. It runs at two moments that already
|
|
135
|
+
exist: at the end of a retirement transition, bracketing the wait that transition already
|
|
136
|
+
performed, and at the start of a `deploy`, reusing the liveness observation the admission bound
|
|
137
|
+
already takes. A generation that stopped with a live process group keeps its observations, so
|
|
138
|
+
the next deploy can prove the residue without sampling anything new.
|
|
139
|
+
|
|
140
|
+
A `SIGKILL` is delivered only when **all four** proofs hold, each named in the log line:
|
|
141
|
+
|
|
142
|
+
1. The generation is classified `wedged` — it owns no live connection.
|
|
143
|
+
2. Every live process already expired its retirement budget.
|
|
144
|
+
3. **No runner child remains** in any tracked process group. This is the STOE-1077 boundary: a
|
|
145
|
+
live lane is never aborted, and a wedge that still holds one is reported instead of killed.
|
|
146
|
+
4. The group's CPU is **flat across a full 20 s sample window**, so a group that is merely slow
|
|
147
|
+
is never mistaken for a wedged one.
|
|
148
|
+
|
|
149
|
+
Anything else reports. A refused wedge is logged with the pids and process groups it is holding
|
|
150
|
+
— the information an operator needs in place of guessing — and the deploy proceeds to the
|
|
151
|
+
admission bound above:
|
|
152
|
+
|
|
153
|
+
```text
|
|
154
|
+
release wedged generation left alive {escalation: refused, pids: [4242], processIds: ["worker"],
|
|
155
|
+
reason: "no escalation: 1 runner child is still live in the generation; SIGKILL would abort
|
|
156
|
+
in-flight work", releaseId: "20261115164249", trigger: "deploy of 20261115202720"}
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
When the proofs do hold, the supervisor terminates the group and says so:
|
|
160
|
+
|
|
161
|
+
```text
|
|
162
|
+
process retirement escalated to SIGKILL {id: "worker", pid: 4242, reason: "release retirement: escalating: …"}
|
|
163
|
+
release wedged generation escalated {escalation: SIGKILL, processIds: ["worker"], releaseId: "20261115164249"}
|
|
164
|
+
```
|
|
165
|
+
|
|
92
166
|
## `statePath`
|
|
93
167
|
|
|
94
168
|
When set, the daemon persists a sanitized operational state snapshot — the active and
|
|
@@ -168,12 +242,16 @@ The first upgrade from an authenticated pre-replacement Rollbridge guardian and
|
|
|
168
242
|
daemon uses an explicitly disruptive compatibility bridge because that legacy
|
|
169
243
|
owner cannot transfer listeners on Node 20. A retained guardian that supports
|
|
170
244
|
replacement prepare/stage but lacks the retired-owner commit command is explicitly
|
|
171
|
-
classified and uses the same bridge
|
|
245
|
+
classified and uses the same bridge, and so is a retained guardian whose rollbridge
|
|
246
|
+
version differs from the daemon runtime: the candidate starts a coordinator of its
|
|
247
|
+
own version, the retained guardian keeps the processes it already owns, and nothing
|
|
248
|
+
waits for those to drain. Rollbridge attests the exact guardian and
|
|
172
249
|
daemon processes, sockets, runtime/config authority, and durable process
|
|
173
250
|
registrations before retiring the legacy listeners. Managed process PIDs and
|
|
174
251
|
release state remain supervised, but live proxy/control connections may close.
|
|
175
252
|
The resulting status includes `ownerTransition: {disruptive: true, mode:
|
|
176
|
-
"legacy-first-upgrade", ...}
|
|
253
|
+
"legacy-first-upgrade", ...}` for a protocol gap or `{disruptive: true, mode:
|
|
254
|
+
"guardian-version-handoff", ...}` for a version drift. The bridge only accepts the incumbent config
|
|
177
255
|
identity; retry config or socket changes after the protocol upgrade, when the
|
|
178
256
|
normal atomic handoff applies. Other guardian/auth/transport/identity failures
|
|
179
257
|
remain fail-closed.
|
|
@@ -231,6 +309,8 @@ legacyTakeover: {
|
|
|
231
309
|
| `env` | object of string → string | `{}` | Extra environment variables (values templated). Merged over the injected `ROLLBRIDGE_*` vars. |
|
|
232
310
|
| `port` | number or `{from, to}` | unset | Port (or range) allocated per release. **Required for the `proxied` process.** A plain number `n` means the fixed port `n` (`{from: n, to: n}`). |
|
|
233
311
|
| `health` | object or `false` | enabled with defaults | Health check for the `proxied` process; set `false` to disable (see below). |
|
|
312
|
+
| `dependsOn` | array of process ids | `[]` | Processes whose allocated port must accept a connection before Rollbridge starts this process (see [`processes[].dependsOn`](#processesdependson)). |
|
|
313
|
+
| `dependencyTimeoutMs` | positive number | `30000` | Budget for each `dependsOn` wait. Charged to its own clock, never to `health.timeoutMs`. |
|
|
234
314
|
| `stopSignal` | signal name (e.g. `"SIGTERM"`, `"SIGINT"`, `"SIGQUIT"`) | `"SIGTERM"` | Signal sent to gracefully stop the process; after `gracefulStopMs` it is `SIGKILL`ed. Use a worker's quit signal so it finishes in-flight work before exiting. |
|
|
235
315
|
| `nonBlockingDrain` | boolean | `false` | When a release is retired, drain this process **immediately** (in parallel with the proxied connection drain) instead of after it. Companion processes only — typically background workers (see below). |
|
|
236
316
|
| `lifecycle` | object | no hooks | Command hooks run when gracefully stopping the process (see below). |
|
|
@@ -441,10 +521,97 @@ release). Set `health: false` to disable it.
|
|
|
441
521
|
| Field | Type | Default | Description |
|
|
442
522
|
| --- | --- | --- | --- |
|
|
443
523
|
| `health.path` | string | `proxy.healthPath` | HTTP path probed on the process's port. |
|
|
444
|
-
| `health.timeoutMs` | number | `proxy.healthTimeoutMs` | Total time to wait for the first healthy response. |
|
|
524
|
+
| `health.timeoutMs` | number | `proxy.healthTimeoutMs` | Total time to wait for the first healthy response, measured from the process launch. Only the app's own boot is charged to it — a `dependsOn` wait runs on `dependencyTimeoutMs` before this window opens. |
|
|
445
525
|
| `health.intervalMs` | number | `250` | Delay between probes. |
|
|
446
526
|
| `health.startDelayMs` | non-negative number | `0` | Wait this long after the process starts before the first probe (runs before the `timeoutMs` window). |
|
|
447
527
|
|
|
528
|
+
### `processes[].dependsOn`
|
|
529
|
+
|
|
530
|
+
A release-scoped process that talks to a same-release service usually waits for
|
|
531
|
+
that service inside its own command (`wait-port … && exec <app>`). That wait
|
|
532
|
+
runs after Rollbridge has started the process, so it is charged against
|
|
533
|
+
`health.timeoutMs`: a dependency that takes 12 seconds to bind leaves the app 18
|
|
534
|
+
seconds of a 30-second health deadline, and the deploy fails while the app is
|
|
535
|
+
still booting normally.
|
|
536
|
+
|
|
537
|
+
Declare the dependency instead, and Rollbridge waits for each `dependsOn`
|
|
538
|
+
process's allocated port **before** spawning the dependent, on a budget of its
|
|
539
|
+
own (`dependencyTimeoutMs`):
|
|
540
|
+
|
|
541
|
+
```js
|
|
542
|
+
{
|
|
543
|
+
id: "web",
|
|
544
|
+
policy: "proxied",
|
|
545
|
+
dependsOn: ["background-jobs-main"],
|
|
546
|
+
dependencyTimeoutMs: 30000,
|
|
547
|
+
command: "exec node build/bin/velocious.js server --port {{port}}",
|
|
548
|
+
port: {from: 14500, to: 14599},
|
|
549
|
+
health: {path: "/ping", timeoutMs: 120000}
|
|
550
|
+
}
|
|
551
|
+
```
|
|
552
|
+
|
|
553
|
+
`health.timeoutMs` then measures only the app's own boot. Both waits are logged
|
|
554
|
+
separately: `release process dependency wait` / `release process dependency
|
|
555
|
+
ready` (with `attempts` and `elapsedMs`) and `release candidate health ready`
|
|
556
|
+
(with `bootMs`, `healthGateMs`, and `waitedForDependenciesMs`).
|
|
557
|
+
|
|
558
|
+
`dependsOn` accepts ids of processes Rollbridge starts before this one —
|
|
559
|
+
persistent services, handoff services, companions, and the proxied process. A
|
|
560
|
+
dependency must have a `port` (there is nothing to wait for otherwise), may not
|
|
561
|
+
be a `singleton` (singleton ownership is not per-release), and may not be this
|
|
562
|
+
process itself. Dependencies that Rollbridge would start after this process are
|
|
563
|
+
rejected, which also rules out cycles.
|
|
564
|
+
|
|
565
|
+
## `compose[]`
|
|
566
|
+
|
|
567
|
+
Optional Docker Compose services this app updates. Each entry is a unit Rollbridge
|
|
568
|
+
rolls drain-first — quiesce, wait for in-flight work, recreate with the target
|
|
569
|
+
image, gate on activation, roll back on failure — instead of a raw
|
|
570
|
+
`docker compose up -d --force-recreate`. See
|
|
571
|
+
[`docs/compose-services.md`](compose-services.md) for the stage-by-stage behavior
|
|
572
|
+
and the operator commands.
|
|
573
|
+
|
|
574
|
+
| Field | Type | Default | Description |
|
|
575
|
+
| --- | --- | --- | --- |
|
|
576
|
+
| `compose[].id` | string | **required** | Names the unit for `compose-roll`/`compose-rollback`/`compose-status`. Unique across units and not equal to any process id. |
|
|
577
|
+
| `compose[].service` | string | **required** | Service name inside the Compose file. |
|
|
578
|
+
| `compose[].projectDirectory` | string | **required** | Directory holding the Compose file; every Compose command runs here. |
|
|
579
|
+
| `compose[].image` | string | **required** | Target image reference for `compose-roll` (a tag or a `@sha256:` digest; never a bare image name). |
|
|
580
|
+
| `compose[].composeFile` | string | `"compose.yml"` | Compose file relative to `projectDirectory`. |
|
|
581
|
+
| `compose[].projectName` | string | unset (Compose default) | Passed as `--project-name`, so a project whose services come from several files stays addressable. |
|
|
582
|
+
| `compose[].envFile` | string | unset | Passed as `--env-file`, relative to `projectDirectory`. |
|
|
583
|
+
| `compose[].stopGracePeriodMs` | non-negative number | `10000` | Written into the roll's override as the service's `stop_grace_period`: how long Compose waits for the old container to exit. |
|
|
584
|
+
| `compose[].recreateTimeoutMs` | non-negative number | `120000` | Timeout for the `compose up -d --no-deps --force-recreate` the roll runs. |
|
|
585
|
+
| `compose[].drain.command` | string | unset | Hook that stops the unit admitting new work. Runs through a shell; unset means the unit has no quiescence hook and the roll proceeds to the in-flight gate. |
|
|
586
|
+
| `compose[].drain.timeoutMs` | positive number | `60000` | Required when `drain.command` is set. Expiry fails the roll at `stage: "drain"` without recreating the container. |
|
|
587
|
+
| `compose[].inFlight.command` | string | unset | Gate that exits 0 only when the work the unit already accepted has settled. Polled until it exits 0 or `inFlight.timeoutMs` expires. |
|
|
588
|
+
| `compose[].inFlight.intervalMs` | positive number | `1000` | Delay between gate attempts. |
|
|
589
|
+
| `compose[].inFlight.timeoutMs` | positive number | `0` (no gate) | Required when `inFlight.command` is set. Expiry fails the roll at `stage: "in_flight"` without recreating the container. |
|
|
590
|
+
| `compose[].activation.healthcheck` | boolean | `true` | Require the recreated container's Compose healthcheck to report `healthy` (containers without a `HEALTHCHECK` pass). |
|
|
591
|
+
| `compose[].activation.readyCommand` | string | unset | Additional readiness probe run through a shell; must exit 0. |
|
|
592
|
+
| `compose[].activation.readyUrl` | string | unset | Additional readiness probe fetched over HTTP; any 2xx passes. |
|
|
593
|
+
| `compose[].activation.startDelayMs` | non-negative number | `0` | Wait before the first probe (runs before the `timeoutMs` window). |
|
|
594
|
+
| `compose[].activation.intervalMs` | positive number | `1000` | Delay between probes. |
|
|
595
|
+
| `compose[].activation.timeoutMs` | positive number | `60000` | How long the activation gate may keep probing before the roll rolls back. |
|
|
596
|
+
| `compose[].rollback.enabled` | boolean | `true` | Roll back to the retained previous image when activation fails. When `false`, a failed activation is reported as a failed roll. |
|
|
597
|
+
|
|
598
|
+
`composeStatePath` (top-level) is where each unit's generations are retained, so
|
|
599
|
+
`compose-rollback` has a previous tag to select. It defaults to
|
|
600
|
+
`rollbridge-<application>.compose.state.json` next to the config file.
|
|
601
|
+
|
|
602
|
+
Drain, in-flight, and readiness-command hooks support `{{...}}` placeholders:
|
|
603
|
+
|
|
604
|
+
| Placeholder | Value |
|
|
605
|
+
| --- | --- |
|
|
606
|
+
| `{{application}}` | `application` |
|
|
607
|
+
| `{{unitId}}` | This unit's `id`. |
|
|
608
|
+
| `{{service}}` | This unit's `service`. |
|
|
609
|
+
| `{{composeArgs}}` | The Compose global arguments for this unit (`docker compose`, its `--project-name`, `-f <file>`, `--env-file`), so a hook can run `{{composeArgs}} exec -T <service> ...`. |
|
|
610
|
+
| `{{composeFile}}` | This unit's `composeFile`. |
|
|
611
|
+
| `{{projectDirectory}}` | This unit's `projectDirectory`. |
|
|
612
|
+
| `{{releaseId}}` | This roll's release id. |
|
|
613
|
+
| `{{env.<NAME>}}` | A variable from the Rollbridge process environment. |
|
|
614
|
+
|
|
448
615
|
## Template variables
|
|
449
616
|
|
|
450
617
|
`command`, `cwd`, and `env` values support `{{...}}` placeholders, rendered when
|
|
@@ -490,10 +657,14 @@ Rollbridge sets these in every managed process's environment (the process's own
|
|
|
490
657
|
- `port` must be a positive port number or an ascending `{from, to}` range.
|
|
491
658
|
- `control.mode` must be an octal mode between `0` and `0o777`.
|
|
492
659
|
- `control.owner` and `control.group` must each be a non-negative integer id or a non-empty name (resolved at daemon start).
|
|
493
|
-
- `outputLines` and `releaseRetention.keep` must be positive/non-negative integers; `health.startDelayMs` and `releaseRetention.
|
|
660
|
+
- `outputLines` and `releaseRetention.keep` must be positive/non-negative integers; `health.startDelayMs`, `releaseRetention.maxAgeMs`, and `releaseRetention.maxConcurrentDrainingReleaseGroups` must be non-negative numbers.
|
|
494
661
|
- `restart.maxRestarts` must be a non-negative integer (omit it for unlimited restarts); `restart.backoffFactor` must be a number ≥ 1; `restart.windowMs` and `restart.maxDelayMs` must be non-negative numbers.
|
|
495
662
|
- When `memory` is set, `memory.limitBytes` must be a positive integer, `memory.warnBytes` a non-negative integer, and `memory.checkIntervalMs` a positive number.
|
|
496
663
|
- `replicas` must be a positive integer; `replicas > 1` is allowed only on a `companion` process without a `port`. Process ids must not contain `#` (reserved for replica instance ids).
|
|
497
664
|
- `lifecycle.activateCommand`/`quietCommand`/`drainCommand`/`stopCommand` must be non-empty strings when set, and `lifecycle.drainTimeoutMs` a non-negative number; `lifecycle.drainCommand` requires a positive `lifecycle.drainTimeoutMs`. `activateCommand` is allowed on at most one handoff service, requires that service's `quietCommand`, and requires `statePath` plus `ownerRecovery`. A `lifecycle.stopCommand` may not be combined with a custom `stopSignal` (the `stopCommand` runs instead of the signal, so the signal would be ignored).
|
|
498
665
|
- `nonBlockingDrain` must be a boolean, and is allowed only on a `companion` process.
|
|
666
|
+
- `dependsOn` must be an array of configured process ids: not the process itself, not a `singleton`, not a process without a `port`, and not a process Rollbridge starts after it. `dependencyTimeoutMs` must be a positive number.
|
|
499
667
|
- `statePath` must be a string when set.
|
|
668
|
+
- `compose` must be a non-empty array of unit mappings. Each unit needs `id`, `service`, `projectDirectory`, and a valid `image` reference (a tag, or a `@sha256:` digest with 64 hex characters; no whitespace, no leading `-`).
|
|
669
|
+
- Compose unit `id`s must be unique and must not collide with a process `id`.
|
|
670
|
+
- `compose[].drain.command` requires a positive `compose[].drain.timeoutMs`; `compose[].inFlight.command` requires a positive `compose[].inFlight.timeoutMs`.
|
package/docs/troubleshooting.md
CHANGED
|
@@ -27,6 +27,33 @@ preparation itself fails, check permissions for
|
|
|
27
27
|
`/tmp/rollbridge-<user-id>-<application-hash>-runtime`) before retrying. The directory
|
|
28
28
|
must be private to the invoking user.
|
|
29
29
|
|
|
30
|
+
## Retained guardian on an older rollbridge version
|
|
31
|
+
|
|
32
|
+
**Symptom.** `ensure-daemon` reports `The persistent Rollbridge guardian is version <x> while
|
|
33
|
+
this daemon runs rollbridge <y>` and confirms that the deploy was not sent.
|
|
34
|
+
|
|
35
|
+
**Cause.** The daemon reuses `recovery.guardian` from the state file on every start, so a
|
|
36
|
+
supervisor left running from an earlier release outlives the package it came from. A version
|
|
37
|
+
mismatch is normally reconciled without an operator: `ensure-daemon` starts a coordinator of its
|
|
38
|
+
own version over the retained guardian, the retained guardian keeps supervising the processes it
|
|
39
|
+
already owns, and nothing waits for them to drain. `status.ownerTransition` then reports
|
|
40
|
+
`mode: "guardian-version-handoff"` with `disruptive: true`; the incumbent daemon is retired while
|
|
41
|
+
its supervised application processes keep their exact PIDs.
|
|
42
|
+
|
|
43
|
+
**Fix.** The refusal is the last resort and names both versions plus the exact commands. Run them
|
|
44
|
+
in that order:
|
|
45
|
+
|
|
46
|
+
```sh
|
|
47
|
+
rollbridge shutdown --config /etc/rollbridge/app.cjs
|
|
48
|
+
rollbridge ensure-daemon --config /etc/rollbridge/app.cjs
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
It fires only when no handoff can be authorised: no incumbent daemon to hand guardian ownership
|
|
52
|
+
from, no incumbent PID reported by `ensure-daemon`, a guardian version that cannot be attested, an
|
|
53
|
+
incumbent PID that is not the attested owner, or a config change in the same invocation — the
|
|
54
|
+
bridge keeps the incumbent config identity unchanged, so apply a config or socket change in a
|
|
55
|
+
second invocation once the supervisor is current.
|
|
56
|
+
|
|
30
57
|
If startup instead reports `Rollbridge daemon candidate <pid> exited before
|
|
31
58
|
readiness`, the ensuring CLI observed that exact child exit before it could
|
|
32
59
|
attest status. The diagnostic includes the exit code or signal and spawned
|
|
@@ -54,14 +81,30 @@ Health check failed for http://127.0.0.1:18182/ping: HTTP 503
|
|
|
54
81
|
new release never went live; the previous release stays active.
|
|
55
82
|
|
|
56
83
|
**Diagnose.** The new release's `proxied` process didn't return a healthy
|
|
57
|
-
response in time.
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
84
|
+
response in time. The reason names what it spent that budget on —
|
|
85
|
+
`(process web: 18 probes, 941ms elapsed of the 900ms health deadline)` — and the
|
|
86
|
+
`release startup failed` event carries `deadlineMs`, `elapsedMs`,
|
|
87
|
+
`dependencyWaitMs`, `probes`, `stage` (`"dependency"` or `"health"`),
|
|
88
|
+
`failedProcessId`, `readyProcesses`/`unreadyProcesses`, and the process's
|
|
89
|
+
`lastOutput`. Check that process's output with `rollbridge logs --process <id>`
|
|
90
|
+
and its state/`exitCode` with `rollbridge status`. Common causes: the app doesn't
|
|
91
|
+
listen on the templated `{{port}}`, the `health.path` returns a non-2xx status, or
|
|
92
|
+
the app boots slower than `health.timeoutMs`.
|
|
93
|
+
|
|
94
|
+
If the boot itself is fine but the app waits for a same-release service inside
|
|
95
|
+
its own command, that wait is inside the health deadline. `stage: "dependency"`
|
|
96
|
+
plus a `release process dependency ready` event shows the wait Rollbridge did on
|
|
97
|
+
the dependency's own budget.
|
|
61
98
|
|
|
62
99
|
**Fix.** Make the proxied command bind `{{port}}` and serve `health.path` with a
|
|
63
|
-
2xx status.
|
|
64
|
-
|
|
100
|
+
2xx status. The boot budget is `proxy.healthTimeoutMs` (`120000` by default), which
|
|
101
|
+
every process inherits unless it sets `health.timeoutMs` itself, and it measures only
|
|
102
|
+
the app's own boot. For an app that needs longer than the default, raise
|
|
103
|
+
`health.timeoutMs` above the measured boot time (or set `health.startDelayMs` so
|
|
104
|
+
probing begins after the app is up). For a
|
|
105
|
+
same-release service dependency, declare `dependsOn` (see
|
|
106
|
+
[`processes[].dependsOn`](config.md#processesdependson)) so the wait runs on
|
|
107
|
+
`dependencyTimeoutMs` instead of eating the boot budget.
|
|
65
108
|
|
|
66
109
|
## Port conflicts / exhausted ranges
|
|
67
110
|
|
|
@@ -148,3 +191,51 @@ separate destructive `--takeover-owner` path. If
|
|
|
148
191
|
`retirementError` is set,
|
|
149
192
|
inspect the quiet-hook events. Rollbridge deliberately leaves that generation
|
|
150
193
|
alive rather than signaling arbitrary PIDs or continuing its stop sequence.
|
|
194
|
+
|
|
195
|
+
**A drain recorded but a process group is still alive.** `release drained`
|
|
196
|
+
appears, the release reports `state: "stopped"`, yet `processes[].pid` or
|
|
197
|
+
`processes[].heldProcessGroup` is still reported and later deploys keep naming
|
|
198
|
+
this generation `wedged`. That is a wedged non-blocking drain: its retirement
|
|
199
|
+
budget expired, the release finished retiring, and the process group it left
|
|
200
|
+
behind still holds the generation.
|
|
201
|
+
|
|
202
|
+
**Diagnose.** Read `drain.reason` for the classification and the evidence behind
|
|
203
|
+
it, and `heldProcessGroup: {liveMemberCount, pgid}` for the group that outlived
|
|
204
|
+
its tracked process. `escalation: refused` in a
|
|
205
|
+
`release wedged generation left alive` line names why the supervisor declined to
|
|
206
|
+
terminate it — usually a runner child that is still live, which is exactly the
|
|
207
|
+
case a `SIGKILL` must never reach.
|
|
208
|
+
|
|
209
|
+
**Fix.** With `releaseRetention.escalateWedgedGenerations: true`, the next
|
|
210
|
+
deploy terminates the group when all four proofs hold (no live connection, every
|
|
211
|
+
retirement budget expired, no runner child remaining, CPU flat across a full
|
|
212
|
+
20 000 ms window) and logs `release wedged generation escalated`. It does nothing
|
|
213
|
+
when any proof is missing, so enabling it does not turn a slow drain into a kill.
|
|
214
|
+
Without it, the log line is the deliverable: it names the pids and process groups
|
|
215
|
+
to signal by hand. A wedge that persists across many deploys with live runner
|
|
216
|
+
children is a drain that cannot finish, not a supervisor fault — see
|
|
217
|
+
[`docs/config.md`](config.md#the-wedged-generation-escalation).
|
|
218
|
+
|
|
219
|
+
## A command is refused with "Another owner mutation is already in progress"
|
|
220
|
+
|
|
221
|
+
**Symptom.** `deploy`, `stop`, `restart`, `rollback` or
|
|
222
|
+
`recover-generation-transition` fails with
|
|
223
|
+
`Another owner mutation is already in progress: owner mutation "<operation>";
|
|
224
|
+
mutation <id>; held by process <pid>; since <ISO-8601>; <n>s elapsed; waiting
|
|
225
|
+
for <blocker>; refused <operation>`.
|
|
226
|
+
|
|
227
|
+
**Cause.** The committed daemon holds the guardian's owner-mutation fence for
|
|
228
|
+
one control mutation at a time, and a second mutation arrived while it was still
|
|
229
|
+
held.
|
|
230
|
+
|
|
231
|
+
**Fix.** Read the named holder before touching anything: the message states
|
|
232
|
+
which mutation holds the fence, its lease id, the PID of the process that
|
|
233
|
+
started it, when it started, how long it has run, and the blocker that holder
|
|
234
|
+
last reported (`waitingFor`, which a running mutation can refine through
|
|
235
|
+
`reportOwnerMutationWait`). That is the command to inspect — not the one that
|
|
236
|
+
was refused. A mutation that outruns its lease (15 minutes by default,
|
|
237
|
+
overridable per command with `mutationTimeoutMs`) is logged as
|
|
238
|
+
`OWNER_MUTATION_LEASE_EXPIRED`, reported to the holder, and released, so the
|
|
239
|
+
next mutation proceeds and the stuck one fails loudly when it next ends its
|
|
240
|
+
lease. A retained jobs generation is not an owner mutation and may still drain
|
|
241
|
+
for hours; only the fence is bounded.
|
package/docs/velocious.md
CHANGED
package/docs/workers.md
CHANGED
|
@@ -26,6 +26,7 @@ fixed port such as `7330`.
|
|
|
26
26
|
{
|
|
27
27
|
id: "background-jobs-worker",
|
|
28
28
|
policy: "companion",
|
|
29
|
+
dependsOn: ["background-jobs-main"],
|
|
29
30
|
env: {VELOCIOUS_BACKGROUND_JOBS_PORT: "{{ports.background-jobs-main}}"},
|
|
30
31
|
command: "npx velocious background-jobs-worker",
|
|
31
32
|
replicas: 4,
|
|
@@ -106,5 +107,15 @@ Returned work may then run in the active generation. Normal retirement does not
|
|
|
106
107
|
simulate a disconnect and does not make a new jobs-main adopt the old worker's
|
|
107
108
|
handoffs.
|
|
108
109
|
|
|
110
|
+
The supervisor's answer to a drain that cannot finish is separate from these
|
|
111
|
+
timeouts. `releaseRetention.escalateWedgedGenerations` (off by default) lets the
|
|
112
|
+
generation supervisor terminate a process group it has proven wedged and holding
|
|
113
|
+
nothing — no live connection, every retirement budget expired, no runner child
|
|
114
|
+
remaining, CPU flat across a full sample window — at the end of a retirement
|
|
115
|
+
transition or at the next deploy. It never waits for a drain and never kills a
|
|
116
|
+
process that still holds a live runner; a wedge it may not terminate is reported
|
|
117
|
+
with its pids instead. See
|
|
118
|
+
[`docs/config.md`](config.md#the-wedged-generation-escalation).
|
|
119
|
+
|
|
109
120
|
See [`docs/velocious.md`](velocious.md) for the complete topology and
|
|
110
121
|
[`docs/config.md`](config.md#processesdeploystrategy) for handoff-service fields.
|
|
@@ -20,7 +20,7 @@ export default {
|
|
|
20
20
|
host: "127.0.0.1",
|
|
21
21
|
port: 4500,
|
|
22
22
|
healthPath: "/ping",
|
|
23
|
-
healthTimeoutMs:
|
|
23
|
+
healthTimeoutMs: 120000,
|
|
24
24
|
drainTimeoutMs: 60000,
|
|
25
25
|
forceStopTimeoutMs: 10000
|
|
26
26
|
},
|
|
@@ -60,6 +60,7 @@ export default {
|
|
|
60
60
|
{
|
|
61
61
|
id: "background-jobs-worker",
|
|
62
62
|
policy: "companion",
|
|
63
|
+
dependsOn: ["background-jobs-main"],
|
|
63
64
|
cwd: "{{releasePath}}/backend",
|
|
64
65
|
env: {
|
|
65
66
|
NODE_ENV: "production",
|
|
@@ -67,7 +68,7 @@ export default {
|
|
|
67
68
|
VELOCIOUS_BEACON_PORT: "{{ports.beacon}}",
|
|
68
69
|
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{ports.background-jobs-main}}"
|
|
69
70
|
},
|
|
70
|
-
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict --
|
|
71
|
+
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- npx velocious background-jobs-worker",
|
|
71
72
|
lifecycle: {
|
|
72
73
|
quietCommand: "appctl jobs-worker-retire --pid $ROLLBRIDGE_PID",
|
|
73
74
|
reactivateCommand: "appctl jobs-worker-reactivate --pid $ROLLBRIDGE_PID"
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -10,6 +10,7 @@ import RollbridgeDaemon, {ownerConfigDigest} from "./daemon.js"
|
|
|
10
10
|
import {loadDaemonRuntimeIdentity, prepareDaemonRuntime} from "./daemon-runtime.js"
|
|
11
11
|
import {loadConfig, parseConfigFile, resolveConfigPath, validateConfig} from "./config.js"
|
|
12
12
|
import {runEnvironmentChecks, runReleaseChecks} from "./doctor.js"
|
|
13
|
+
import {composeUnitStatus, rollComposeUnit, rollbackComposeUnit} from "./compose-unit.js"
|
|
13
14
|
import {predeployCleanup} from "./predeploy-cleanup.js"
|
|
14
15
|
import {recoverOrphans} from "./recover.js"
|
|
15
16
|
import {sendControlCommand} from "./control-client.js"
|
|
@@ -247,6 +248,49 @@ export async function runCli(argv) {
|
|
|
247
248
|
console.log(JSON.stringify(response, null, 2))
|
|
248
249
|
})
|
|
249
250
|
|
|
251
|
+
program
|
|
252
|
+
.command("compose-roll")
|
|
253
|
+
.description("Drain-first roll of one Docker Compose service: quiesce it, wait for in-flight work, recreate it with the target image, gate on activation, and roll back on failure.")
|
|
254
|
+
.option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
|
|
255
|
+
.requiredOption("--unit <id>", "Compose unit id declared in the config")
|
|
256
|
+
.option("--image <reference>", "Target image reference (defaults to the unit's configured image)")
|
|
257
|
+
.action(async (options) => {
|
|
258
|
+
const configPath = await resolveConfigPath(options.config)
|
|
259
|
+
const config = await loadConfig(configPath)
|
|
260
|
+
const result = await rollComposeUnit({config, image: options.image, unit: requireComposeUnit(config, options.unit, configPath)})
|
|
261
|
+
|
|
262
|
+
console.log(JSON.stringify(result, null, 2))
|
|
263
|
+
if (result.status === "failed") process.exitCode = 1
|
|
264
|
+
})
|
|
265
|
+
|
|
266
|
+
program
|
|
267
|
+
.command("compose-rollback")
|
|
268
|
+
.description("Roll one Compose unit back to its retained previous image through the same drain-first path.")
|
|
269
|
+
.option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
|
|
270
|
+
.requiredOption("--unit <id>", "Compose unit id declared in the config")
|
|
271
|
+
.option("--image <reference>", "Earlier image reference (defaults to the retained previous one)")
|
|
272
|
+
.action(async (options) => {
|
|
273
|
+
const configPath = await resolveConfigPath(options.config)
|
|
274
|
+
const config = await loadConfig(configPath)
|
|
275
|
+
const result = await rollbackComposeUnit({config, image: options.image, unit: requireComposeUnit(config, options.unit, configPath)})
|
|
276
|
+
|
|
277
|
+
console.log(JSON.stringify(result, null, 2))
|
|
278
|
+
if (result.status === "failed") process.exitCode = 1
|
|
279
|
+
})
|
|
280
|
+
|
|
281
|
+
program
|
|
282
|
+
.command("compose-status")
|
|
283
|
+
.description("Report a Compose unit's retained image generations and its live container state.")
|
|
284
|
+
.option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
|
|
285
|
+
.requiredOption("--unit <id>", "Compose unit id declared in the config")
|
|
286
|
+
.action(async (options) => {
|
|
287
|
+
const configPath = await resolveConfigPath(options.config)
|
|
288
|
+
const config = await loadConfig(configPath)
|
|
289
|
+
const status = await composeUnitStatus({config, unit: requireComposeUnit(config, options.unit, configPath)})
|
|
290
|
+
|
|
291
|
+
console.log(JSON.stringify(status, null, 2))
|
|
292
|
+
})
|
|
293
|
+
|
|
250
294
|
program
|
|
251
295
|
.command("ensure-daemon")
|
|
252
296
|
.description("Start the daemon if the control socket is not already accepting commands.")
|
|
@@ -367,7 +411,7 @@ export async function runCli(argv) {
|
|
|
367
411
|
const valid = issues.length === 0
|
|
368
412
|
|
|
369
413
|
if (options.json) {
|
|
370
|
-
const summary = valid ? {application: config.application, processes: config.processes.length, proxy: {host: config.proxy.host, port: config.proxy.port}} : null
|
|
414
|
+
const summary = valid ? {application: config.application, composeUnits: config.compose?.length ?? 0, processes: config.processes.length, proxy: {host: config.proxy.host, port: config.proxy.port}} : null
|
|
371
415
|
|
|
372
416
|
console.log(JSON.stringify({config: summary, issues, path: configPath, valid}, null, 2))
|
|
373
417
|
if (!valid) process.exitCode = 1
|
|
@@ -376,8 +420,10 @@ export async function runCli(argv) {
|
|
|
376
420
|
|
|
377
421
|
if (valid) {
|
|
378
422
|
const processCount = config.processes.length
|
|
423
|
+
const composeCount = config.compose?.length ?? 0
|
|
424
|
+
const composeSummary = composeCount > 0 ? `, ${composeCount} Compose unit${composeCount === 1 ? "" : "s"}` : ""
|
|
379
425
|
|
|
380
|
-
console.log(`${configPath} is valid: ${processCount} ${processCount === 1 ? "process" : "processes"}, proxy on ${config.proxy.host}:${config.proxy.port}.`)
|
|
426
|
+
console.log(`${configPath} is valid: ${processCount} ${processCount === 1 ? "process" : "processes"}${composeSummary}, proxy on ${config.proxy.host}:${config.proxy.port}.`)
|
|
381
427
|
return
|
|
382
428
|
}
|
|
383
429
|
|
|
@@ -559,6 +605,26 @@ export async function runCli(argv) {
|
|
|
559
605
|
await program.parseAsync(argv)
|
|
560
606
|
}
|
|
561
607
|
|
|
608
|
+
/**
|
|
609
|
+
* Resolves a Compose unit by id, naming the configured units when the id is unknown.
|
|
610
|
+
* @param {import("./config.js").RollbridgeConfig} config - Loaded config.
|
|
611
|
+
* @param {string} unitId - Requested Compose unit id.
|
|
612
|
+
* @param {string} configPath - Config path shown in the error.
|
|
613
|
+
* @returns {import("./config.js").ComposeUnitConfig} The configured unit.
|
|
614
|
+
*/
|
|
615
|
+
function requireComposeUnit(config, unitId, configPath) {
|
|
616
|
+
const units = config.compose ?? []
|
|
617
|
+
const unit = units.find((candidate) => candidate.id === unitId)
|
|
618
|
+
|
|
619
|
+
if (unit) return unit
|
|
620
|
+
|
|
621
|
+
if (units.length === 0) {
|
|
622
|
+
throw new Error(`${configPath} declares no compose units. Add a compose: [{id: "${unitId}", ...}] entry to roll it drain-first.`)
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
throw new Error(`No compose unit "${unitId}" in ${configPath}. Configured units: ${units.map((candidate) => candidate.id).join(", ")}.`)
|
|
626
|
+
}
|
|
627
|
+
|
|
562
628
|
/**
|
|
563
629
|
* @param {import("./predeploy-cleanup.js").PredeployCleanupResult} result - Cleanup result.
|
|
564
630
|
* @returns {string} Human-readable summary.
|