rollbridge 0.1.39 → 0.1.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/changelog.d/20260830-guardian-retired-replacement-process-key.md +12 -1
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +16 -2
- package/changelog.d/20260830055159-guardian-daemon-restart.md +2 -0
- package/docs/cli.md +23 -9
- package/docs/config.md +37 -13
- package/docs/logging.md +4 -3
- package/docs/troubleshooting.md +7 -5
- package/examples/tensorbuzz.com.js +5 -2
- package/package.json +1 -1
- package/src/cli.js +118 -24
- package/src/config.js +35 -8
- package/src/daemon.js +822 -152
- package/src/guardian-client.js +121 -16
- package/src/managed-process.js +117 -15
- package/src/process-guardian.js +734 -43
- package/src/release-group.js +45 -7
- package/test/completion.test.js +4 -2
- package/test/config-examples.test.js +1 -0
- package/test/config-validation.test.js +22 -0
- package/test/fixtures/guardian-recovery-owner.js +86 -0
- package/test/fixtures/pre-split3-process-guardian.js +14 -0
- package/test/guardian-client.test.js +1420 -62
- package/test/managed-process.test.js +163 -7
- package/test/owner-recovery.test.js +575 -73
- package/test/owner-replacement.test.js +525 -28
- package/test/release-group.test.js +19 -2
- package/test/release-runtime-retention.test.js +121 -4
- package/test/rollbridge.test.js +263 -13
- package/test/support/process.js +41 -0
package/README.md
CHANGED
|
@@ -227,7 +227,13 @@ then runs a private local process guardian which remains the OS supervisor for
|
|
|
227
227
|
managed processes if the control daemon exits unexpectedly. A replacement using
|
|
228
228
|
the exact same normalized config/runtime reconnects within `reconnectGraceMs`,
|
|
229
229
|
reconstructs active and draining generations and their ports, and fences
|
|
230
|
-
concurrent replacements.
|
|
230
|
+
concurrent replacements. If no replacement reconnects during that grace, the
|
|
231
|
+
guardian restarts the exact accepted daemon command and environment itself; the
|
|
232
|
+
recovery definition is kept only in the guardian's private authenticated state
|
|
233
|
+
and is refreshed atomically during a package/runtime replacement. A restart
|
|
234
|
+
attempt which cannot claim ownership and publish ready listeners within the
|
|
235
|
+
accepted startup timeout is terminated with its process group and retried with a
|
|
236
|
+
nonzero backoff. `ensure-daemon` can also prepare a requested
|
|
231
237
|
config/control-socket/package/runtime owner, prove it healthy, and atomically
|
|
232
238
|
transfer guardian authority while every retained generation keeps its exact
|
|
233
239
|
release reference and drains asynchronously. The old `statePath` is the durable
|
|
@@ -237,6 +243,15 @@ Prepared transactions fence owner mutations and compare a monotonic guardian
|
|
|
237
243
|
state revision at staging. Existing HTTP/WebSocket connections remain owned by
|
|
238
244
|
the retired listener process, while their counts transfer to the new daemon so
|
|
239
245
|
later deploys continue to honor the original drain boundary.
|
|
246
|
+
If that new daemon itself exits while a retired listener still owns connections,
|
|
247
|
+
recovery conservatively retains the last authenticated transferred count until
|
|
248
|
+
the configured drain timeout; it never guesses that the older sockets closed.
|
|
249
|
+
An intermediate guardian which supports atomic owner replacement but predates
|
|
250
|
+
daemon recovery cannot be hot-upgraded because it is the existing processes' OS
|
|
251
|
+
supervisor. The upgrade fails before handoff and requires one explicit clean
|
|
252
|
+
`shutdown` followed by `ensure-daemon`; subsequent package/runtime replacements
|
|
253
|
+
remain atomic. A genuinely pre-replacement guardian still uses the separately
|
|
254
|
+
documented one-time disruptive compatibility bridge below.
|
|
240
255
|
|
|
241
256
|
There is one explicit compatibility boundary: the first upgrade from a genuine
|
|
242
257
|
pre-owner-replacement Rollbridge guardian and daemon cannot share its listeners
|
|
@@ -425,10 +440,13 @@ If the new release fails to start or health-check, the previous release stays
|
|
|
425
440
|
active and any service started during this deploy is rolled back.
|
|
426
441
|
|
|
427
442
|
`status.releaseReferences` lists the id and path of every active or draining
|
|
428
|
-
release
|
|
443
|
+
release, plus a stopped release that still owns a persistent service definition
|
|
444
|
+
or singleton, or remains part of an unresolved generation transition. A
|
|
445
|
+
reference disappears only when that release has stopped and no longer owns
|
|
446
|
+
runtime state. With
|
|
429
447
|
`ownerRecovery`, those references and generations survive both same-authority
|
|
430
|
-
daemon recovery and guardian-fenced incompatible
|
|
431
|
-
runtime replacement through `ensure-daemon`.
|
|
448
|
+
daemon recovery and guardian-fenced incompatible
|
|
449
|
+
config/control-socket/package/runtime replacement through `ensure-daemon`.
|
|
432
450
|
|
|
433
451
|
## Commands
|
|
434
452
|
|
|
@@ -7,10 +7,21 @@
|
|
|
7
7
|
- Fail closed when an older retained guardian cannot commit that replacement
|
|
8
8
|
atomically after the incumbent control socket disappears, preserving the
|
|
9
9
|
incumbent owner, retained connections, and guardian-managed release processes.
|
|
10
|
+
- Retire the committed incumbent listener after a control-socket-absent handoff
|
|
11
|
+
only after relaying source-identified live connection counts through successive
|
|
12
|
+
owner replacements. Existing WebSocket connections can finish draining without
|
|
13
|
+
one retired listener clearing another listener's counts, and socket-path cleanup
|
|
14
|
+
remains fenced by the listener identity that was actually bound.
|
|
15
|
+
- Keep incumbent authority while it yields a fixed proxy, commit only after the
|
|
16
|
+
candidate receives complete listener state and binds successfully, and resume
|
|
17
|
+
the incumbent if that bind fails. Pending retirement survives candidate recovery,
|
|
18
|
+
concurrent replacements remain fenced, and crashed local sources publish zero
|
|
19
|
+
tombstones while stopped zero-count releases remain omitted.
|
|
10
20
|
- Explicitly classify retained guardian replacement capabilities before preparing
|
|
11
21
|
a transaction. Guardians with prepare/stage support but no retired-owner commit
|
|
12
22
|
command use the fully attested one-time disruptive legacy upgrade bridge;
|
|
13
23
|
malformed, stale, or ambiguous protocol responses continue to fail closed. The
|
|
14
24
|
partial guardian's prepared transaction remains the mutation fence through
|
|
15
25
|
candidate reconstruction and boundary revalidation, and failed preparation
|
|
16
|
-
notifies the incumbent to resume paused release drains.
|
|
26
|
+
notifies the incumbent to resume paused release drains. The bridge coordinator
|
|
27
|
+
publishes its own authenticated recovery identity before the candidate proceeds.
|
|
@@ -5,5 +5,19 @@
|
|
|
5
5
|
candidate-activate acknowledgement, and synchronous active/proxy commit with
|
|
6
6
|
exact transition recovery and resume. Exact release definitions remain private
|
|
7
7
|
to guardian recovery, post-commit singleton work is resumable, unresolved
|
|
8
|
-
control mutations are fenced,
|
|
9
|
-
|
|
8
|
+
control and terminal owner mutations are fenced, recovery uses a complete
|
|
9
|
+
monotonic journal and publishes listener/PID readiness before replaying long
|
|
10
|
+
hooks, signals shut down cleanly after replay, and an active coordinator
|
|
11
|
+
restores its role after restart. Public recovery state advances only after its
|
|
12
|
+
private guardian authority, persistent service definitions retain their exact
|
|
13
|
+
release across recovery, manual restart reaches the active handoff coordinator,
|
|
14
|
+
unresolved transitions retain both release definitions and reject
|
|
15
|
+
config-authority-changing owner replacements, and live activation-mode changes
|
|
16
|
+
require a daemon restart. Hook-free configs keep their existing deploy behavior.
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- Serialize active-role restoration with retirement, reject empty lifecycle
|
|
21
|
+
commands, bound guardian process-log forwarding for stalled clients, and keep
|
|
22
|
+
a claimed recovery owner alive until replacement listener retirement. Journal
|
|
23
|
+
hook-free post-switch retirement and singleton work for exact crash recovery.
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
- Let an `ownerRecovery` process guardian restart the exact accepted Rollbridge daemon command and environment after the reconnection grace when no external replacement claims ownership. Active and draining managed processes keep their existing PIDs, foreground or configured daemon log destinations remain current, the guardian atomically publishes PID files only for ready owners, startup fencing lasts through listener readiness, failed or superseded process groups are terminated, failed starts use a nonzero retry backoff, diagnostics exclude private recovery values, relative control/state paths resolve from the durable config directory, ensured daemons recover from that directory rather than a removable release cwd, and atomic package/runtime replacement refreshes the guardian's private recovery definition.
|
|
2
|
+
- Require one explicit clean shutdown and restart when upgrading an intermediate persistent guardian that supports owner replacement but predates daemon recovery, because the existing OS supervisor cannot gain daemon-recovery code through an ordinary daemon-only handoff. Genuinely pre-replacement guardians retain their one-time disruptive compatibility bridge.
|
package/docs/cli.md
CHANGED
|
@@ -99,8 +99,11 @@ rollbridge ensure-daemon [--config <path>]
|
|
|
99
99
|
```
|
|
100
100
|
|
|
101
101
|
Starts the daemon as a detached process **only if** the control socket is not
|
|
102
|
-
already accepting commands, waits until it responds
|
|
103
|
-
status JSON. Idempotent — safe to call
|
|
102
|
+
already accepting commands, waits until it responds and its guardian accepts
|
|
103
|
+
the ready owner, then prints the daemon status JSON. Idempotent — safe to call
|
|
104
|
+
before every deploy. The detached daemon uses the config file's directory as its
|
|
105
|
+
working directory, rather than the invoking release, so release retention cannot
|
|
106
|
+
remove the accepted recovery cwd.
|
|
104
107
|
|
|
105
108
|
Before starting a detached daemon, Rollbridge atomically copies its runtime code
|
|
106
109
|
and production dependency closure into a content-addressed directory outside
|
|
@@ -128,8 +131,8 @@ and authority failures do not qualify and fail before any deploy is sent.
|
|
|
128
131
|
[`logging.md`](logging.md) for the log format and rotation guidance.
|
|
129
132
|
- `--daemon-pid-path <path>` — file the detached daemon's PID is written to.
|
|
130
133
|
Default: `/tmp/rollbridge-<application>.pid`. During replacement, the file
|
|
131
|
-
continues to name the incumbent until the
|
|
132
|
-
publishes
|
|
134
|
+
continues to name the incumbent until the authenticated guardian atomically
|
|
135
|
+
publishes the ready winner's exact `daemonPid`.
|
|
133
136
|
- `--daemon-runtime-path <path>` — parent directory for content-addressed daemon
|
|
134
137
|
runtime snapshots. Default:
|
|
135
138
|
`/tmp/rollbridge-<user-id>-<application-hash>-runtime`. The directory must be owned
|
|
@@ -167,10 +170,16 @@ An unresolved failure blocks different deploys; only the exact same release,
|
|
|
167
170
|
path, revision, and config authority may explicitly resume its incomplete
|
|
168
171
|
idempotent phase. A durable `committed_pending` phase keeps exact retry from
|
|
169
172
|
reporting success until singleton replacement finishes. Stop, restart, and
|
|
170
|
-
rollback mutations are rejected while the transition is unresolved.
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
173
|
+
rollback mutations are rejected while the transition is unresolved. Automatic
|
|
174
|
+
replay also fences `shutdown` and `retire-owner` until the replay attempt settles.
|
|
175
|
+
An owner replacement may change runtime/package identity during an unresolved
|
|
176
|
+
transition, but it cannot change config authority until that transition commits;
|
|
177
|
+
the incumbent owner remains in place when such a replacement is rejected.
|
|
178
|
+
Hook-free configs retain the existing post-activation quiet behavior and
|
|
179
|
+
retirement result. `status.releaseReferences` lists `{releaseId, releasePath}`
|
|
180
|
+
for every active or draining release and for a stopped release that still owns a
|
|
181
|
+
persistent service definition, pending singleton, or unresolved generation
|
|
182
|
+
transition; unrelated fully stopped history is excluded.
|
|
174
183
|
|
|
175
184
|
For hook-free configs, after candidate activation `Daemon.deploy()` begins old-generation retirement
|
|
176
185
|
and asynchronous drain before awaiting singleton replacement. A singleton
|
|
@@ -244,6 +253,9 @@ Memory-supervised processes also report `rssBytes`, `memoryRestarts`,
|
|
|
244
253
|
`daemonRuntime` identifies the immutable Rollbridge runtime serving the proxy:
|
|
245
254
|
its runtime `format`, package `version`, content `digest`, and absolute `path`.
|
|
246
255
|
`ensure-daemon` uses this attestation before reusing a responsive daemon.
|
|
256
|
+
With `ownerRecovery`, `ownerRecovery.ready` becomes `true` only after the
|
|
257
|
+
guardian has accepted that daemon's listener readiness and atomically published
|
|
258
|
+
its configured PID file; `ensure-daemon` does not return a pre-ready status.
|
|
247
259
|
|
|
248
260
|
A foreground known-release daemon also reports the exact CLI bootstrap identity:
|
|
249
261
|
|
|
@@ -302,7 +314,9 @@ Targeting it (by id or `--policy proxied`) is an error; use `rollbridge deploy`
|
|
|
302
314
|
for a zero-downtime replacement. `--process <id>` with an id that is not a
|
|
303
315
|
managed process (unknown, or a companion with no active release) is also an
|
|
304
316
|
error. Restarting a `service` bounces a shared broker (for example Velocious
|
|
305
|
-
Beacon), which briefly disrupts every process that depends on it.
|
|
317
|
+
Beacon), which briefly disrupts every process that depends on it. For a handoff
|
|
318
|
+
service, restart targets only the active release's instance and restores its
|
|
319
|
+
active lifecycle role before reporting success.
|
|
306
320
|
|
|
307
321
|
## `predeploy-cleanup`
|
|
308
322
|
|
package/docs/config.md
CHANGED
|
@@ -50,13 +50,13 @@ restart.
|
|
|
50
50
|
| `proxy` | object | **required** | Proxy listener and shared defaults (see below). |
|
|
51
51
|
| `processes` | array | **required** | Managed processes (see below). Exactly one must be `proxied`. |
|
|
52
52
|
| `releaseRetention` | object | — | How many stopped releases the daemon retains (see below). |
|
|
53
|
-
| `statePath` | string | unset (no persistence) | File the daemon persists its state to, enabling orphaned-process detection on the next startup (see [`statePath`](#statepath)). |
|
|
53
|
+
| `statePath` | string | unset (no persistence) | File the daemon persists its state to, enabling orphaned-process detection on the next startup; relative paths resolve from the config file directory (see [`statePath`](#statepath)). |
|
|
54
54
|
|
|
55
55
|
## `control`
|
|
56
56
|
|
|
57
57
|
| Field | Type | Default | Description |
|
|
58
58
|
| --- | --- | --- | --- |
|
|
59
|
-
| `control.path` | string | `/tmp/rollbridge-<application>.sock` | Unix domain socket the CLI uses to talk to the daemon. |
|
|
59
|
+
| `control.path` | string | `/tmp/rollbridge-<application>.sock` | Unix domain socket the CLI uses to talk to the daemon; relative paths resolve from the config file directory. |
|
|
60
60
|
| `control.mode` | octal string (e.g. `"660"`) or octal number (`0o660`) | unset | `chmod` applied to the socket after it binds, to share it with a deploy group. When unset, the daemon umask applies. |
|
|
61
61
|
| `control.owner` | non-negative integer uid or user name | unset | `chown` owner applied to the socket after it binds. |
|
|
62
62
|
| `control.group` | non-negative integer gid or group name | unset | `chown` group applied to the socket after it binds, so a shared deploy group can use it. |
|
|
@@ -126,15 +126,30 @@ ownerRecovery: {reconnectGraceMs: 30000}
|
|
|
126
126
|
The private guardian socket is derived from `statePath`; the atomic state file is
|
|
127
127
|
written mode `0600` and contains its authentication capability. The guardian
|
|
128
128
|
owns managed child processes, restart policy, lifecycle hooks, and exit events.
|
|
129
|
-
After an unexpected daemon exit, an exact config/runtime replacement
|
|
130
|
-
guardian during `reconnectGraceMs
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
129
|
+
After an unexpected daemon exit, an exact config/runtime replacement may claim
|
|
130
|
+
the guardian during `reconnectGraceMs`. If none does, the guardian restarts the
|
|
131
|
+
exact accepted daemon command and environment itself, retains startup fencing
|
|
132
|
+
until that daemon publishes its ready listeners and PID file, and uses a nonzero
|
|
133
|
+
retry backoff after failed starts. The replacement restores active and draining
|
|
134
|
+
releases with their allocated ports and resumes proxy/control ownership.
|
|
135
|
+
An unfailed durable generation transition resumes under the guardian's mutation
|
|
136
|
+
fence after those listeners and the recovered PID are ready, so a valid long
|
|
137
|
+
lifecycle hook does not consume the daemon startup deadline. Status remains
|
|
138
|
+
available during replay while competing mutations and terminal owner operations
|
|
139
|
+
stay fenced. `SIGINT` and `SIGTERM` wait for that startup replay before beginning
|
|
140
|
+
clean shutdown. Transition snapshots carry a monotonic journal revision; the
|
|
141
|
+
public snapshot is considered only when it is strictly newer and contains every
|
|
142
|
+
retained transition and singleton-owning release, while private guardian state
|
|
143
|
+
remains authoritative on ties and for older released snapshots without a
|
|
144
|
+
journal revision.
|
|
145
|
+
Concurrent matching starts are fenced: one claims ownership and losers attest
|
|
146
|
+
that winner. The authenticated guardian's private committed state is
|
|
147
|
+
authoritative when the public snapshot is stale or partially written; missing or
|
|
148
|
+
corrupt guardian identity, authentication, or authority still fails closed
|
|
149
|
+
without rewriting the snapshot. Owner disconnection alone never reclaims
|
|
150
|
+
accepted work or transfers workers: guardian-owned processes and their
|
|
151
|
+
generation-local connections continue during the grace, so the replacement
|
|
152
|
+
reconnects to supervision rather than duplicating execution.
|
|
138
153
|
|
|
139
154
|
For a responsive incompatible owner, `ensure-daemon` prepares the requested
|
|
140
155
|
durable runtime, restores exact active and draining generation definitions from
|
|
@@ -163,6 +178,13 @@ identity; retry config or socket changes after the protocol upgrade, when the
|
|
|
163
178
|
normal atomic handoff applies. Other guardian/auth/transport/identity failures
|
|
164
179
|
remain fail-closed.
|
|
165
180
|
|
|
181
|
+
An intermediate guardian which already supports atomic owner replacement but
|
|
182
|
+
predates guardian-owned daemon recovery cannot be upgraded through that bridge.
|
|
183
|
+
The candidate aborts before listener handoff, the incumbent resumes any paused
|
|
184
|
+
drains and remains serving, and the command requests one explicit clean
|
|
185
|
+
`shutdown` followed by `ensure-daemon` so the new guardian can become the OS
|
|
186
|
+
supervisor.
|
|
187
|
+
|
|
166
188
|
Without `ownerRecovery`, `statePath` retains the advisory orphan behavior above.
|
|
167
189
|
|
|
168
190
|
## `legacyTakeover`
|
|
@@ -293,7 +315,9 @@ release id, path, revision, and config may resume only its incomplete idempotent
|
|
|
293
315
|
phase. Once the health-ready candidate is journaled, its exact config becomes the
|
|
294
316
|
transition authority even if a later hook fails. A recorded failed hook is not
|
|
295
317
|
retried merely because daemon ownership changes. Omit `activateCommand` to retain
|
|
296
|
-
the existing activate-then-retire behavior.
|
|
318
|
+
the existing activate-then-retire behavior. Adding, removing, or moving
|
|
319
|
+
`activateCommand` changes the daemon's generation coordinator and therefore
|
|
320
|
+
requires a daemon restart before the next deploy.
|
|
297
321
|
|
|
298
322
|
The synchronous traffic assignment is persisted as `committed_pending` before
|
|
299
323
|
Rollbridge awaits singleton replacement. Exact retry or unambiguous owner recovery
|
|
@@ -469,6 +493,6 @@ Rollbridge sets these in every managed process's environment (the process's own
|
|
|
469
493
|
- `restart.maxRestarts` must be a non-negative integer (omit it for unlimited restarts); `restart.backoffFactor` must be a number ≥ 1; `restart.windowMs` and `restart.maxDelayMs` must be non-negative numbers.
|
|
470
494
|
- When `memory` is set, `memory.limitBytes` must be a positive integer, `memory.warnBytes` a non-negative integer, and `memory.checkIntervalMs` a positive number.
|
|
471
495
|
- `replicas` must be a positive integer; `replicas > 1` is allowed only on a `companion` process without a `port`. Process ids must not contain `#` (reserved for replica instance ids).
|
|
472
|
-
- `lifecycle.activateCommand`/`quietCommand`/`drainCommand`/`stopCommand` must be strings when set, and `lifecycle.drainTimeoutMs` a non-negative number; `lifecycle.drainCommand` requires a positive `lifecycle.drainTimeoutMs`. `activateCommand` is allowed on at most one handoff service, requires that service's `quietCommand`, and requires `statePath` plus `ownerRecovery`. A `lifecycle.stopCommand` may not be combined with a custom `stopSignal` (the `stopCommand` runs instead of the signal, so the signal would be ignored).
|
|
496
|
+
- `lifecycle.activateCommand`/`quietCommand`/`drainCommand`/`stopCommand` must be non-empty strings when set, and `lifecycle.drainTimeoutMs` a non-negative number; `lifecycle.drainCommand` requires a positive `lifecycle.drainTimeoutMs`. `activateCommand` is allowed on at most one handoff service, requires that service's `quietCommand`, and requires `statePath` plus `ownerRecovery`. A `lifecycle.stopCommand` may not be combined with a custom `stopSignal` (the `stopCommand` runs instead of the signal, so the signal would be ignored).
|
|
473
497
|
- `nonBlockingDrain` must be a boolean, and is allowed only on a `companion` process.
|
|
474
498
|
- `statePath` must be a string when set.
|
package/docs/logging.md
CHANGED
|
@@ -15,7 +15,7 @@ daemon was started.
|
|
|
15
15
|
|
|
16
16
|
| How the daemon runs | Destination |
|
|
17
17
|
| --- | --- |
|
|
18
|
-
| `rollbridge daemon` (foreground) | stdout — redirect it (`rollbridge daemon … >> /var/log/rollbridge/app.log 2>&1`) or let your service manager capture it. |
|
|
18
|
+
| `rollbridge daemon` (foreground) | stdout — redirect it (`rollbridge daemon … >> /var/log/rollbridge/app.log 2>&1`) or let your service manager capture it. With `ownerRecovery`, guardian restarts inherit that same accepted destination. |
|
|
19
19
|
| systemd (`examples/rollbridge.service`) | the journal — `journalctl -u rollbridge`. journald rotates on its own. |
|
|
20
20
|
| `rollbridge ensure-daemon` / `rollbridge deploy --ensure-daemon` | the **daemon log file**: `--daemon-log-path <path>`, default `/tmp/rollbridge-<application>.log`. The detached daemon's stdout and stderr are appended there. |
|
|
21
21
|
|
|
@@ -46,8 +46,9 @@ protected accordingly.
|
|
|
46
46
|
|
|
47
47
|
Without `ownerRecovery`, both in-memory views clear when the daemon restarts.
|
|
48
48
|
With it, guardian-held process output remains available after reconnection while
|
|
49
|
-
the replacement daemon begins a new event history.
|
|
50
|
-
|
|
49
|
+
the replacement daemon begins a new event history. A detached ensured daemon
|
|
50
|
+
keeps appending to its configured log file, while a recovered foreground daemon
|
|
51
|
+
inherits its original stdout/stderr destination.
|
|
51
52
|
|
|
52
53
|
## Rotation
|
|
53
54
|
|
package/docs/troubleshooting.md
CHANGED
|
@@ -138,11 +138,13 @@ end close idle WebSockets on deploy). In the documented compliant jobs topology,
|
|
|
138
138
|
jobs companions use `nonBlockingDrain: true`, so timeout expiry affects only the
|
|
139
139
|
web side and must not stop a still-draining jobs generation.
|
|
140
140
|
|
|
141
|
-
`status.releaseReferences` reports active and draining releases
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
141
|
+
`status.releaseReferences` reports active and draining releases, plus a stopped
|
|
142
|
+
release that still owns a persistent service definition, pending singleton, or
|
|
143
|
+
unresolved generation transition, until all runtime ownership ends; Rampway
|
|
144
|
+
still owns enforcement against on-disk cleanup. With `ownerRecovery`, references
|
|
145
|
+
reconstruct across same-authority daemon process replacement and transfer across
|
|
146
|
+
an incompatible `ensure-daemon` owner handoff. They do not transfer through the
|
|
147
|
+
separate destructive `--takeover-owner` path. If
|
|
146
148
|
`retirementError` is set,
|
|
147
149
|
inspect the quiet-hook events. Rollbridge deliberately leaves that generation
|
|
148
150
|
alive rather than signaling arbitrary PIDs or continuing its stop sequence.
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// binds that stable HTTP port, forwards to the active release's internal web
|
|
5
5
|
// port and keeps Beacon daemon-wide. The lifecycle socket path must match the
|
|
6
6
|
// reviewed release-local Velocious jobs-main configuration; the worker appctl
|
|
7
|
-
//
|
|
7
|
+
// commands remain illustrative application-specific quiescence/reactivation controls.
|
|
8
8
|
|
|
9
9
|
export default {
|
|
10
10
|
application: "tensorbuzz",
|
|
@@ -68,7 +68,10 @@ export default {
|
|
|
68
68
|
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{ports.background-jobs-main}}"
|
|
69
69
|
},
|
|
70
70
|
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- wait-for-it 127.0.0.1:{{ports.background-jobs-main}} --strict -- npx velocious background-jobs-worker",
|
|
71
|
-
lifecycle: {
|
|
71
|
+
lifecycle: {
|
|
72
|
+
quietCommand: "appctl jobs-worker-retire --pid $ROLLBRIDGE_PID",
|
|
73
|
+
reactivateCommand: "appctl jobs-worker-reactivate --pid $ROLLBRIDGE_PID"
|
|
74
|
+
},
|
|
72
75
|
nonBlockingDrain: true,
|
|
73
76
|
gracefulStopMs: "indefinite"
|
|
74
77
|
},
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -41,37 +41,71 @@ export async function runCli(argv) {
|
|
|
41
41
|
.option("--boot-attestation <digest>", "Opaque bootstrap ownership attestation (requires the complete bootstrap release tuple)")
|
|
42
42
|
.option("--takeover-owner", "Boot and health-check before retiring the current external owner")
|
|
43
43
|
.option("--replace-owner", "Resume a prepared durable owner replacement")
|
|
44
|
+
.addOption(new Option("--guardian-daemon-log-path <path>").hideHelp())
|
|
45
|
+
.addOption(new Option("--guardian-daemon-pid-path <path>").hideHelp())
|
|
46
|
+
.addOption(new Option("--guardian-daemon-start-timeout-ms <ms>").hideHelp())
|
|
44
47
|
.addOption(new Option("--legacy-incumbent-pid <pid>").hideHelp())
|
|
45
48
|
.action(async (options) => {
|
|
46
49
|
const bootstrap = await validateDaemonBootstrapOptions(options)
|
|
47
50
|
const configPath = await resolveConfigPath(options.config)
|
|
48
51
|
const config = await loadConfig(configPath)
|
|
49
52
|
const runtime = await loadDaemonRuntimeIdentity(process.env.ROLLBRIDGE_DAEMON_RUNTIME_MANIFEST)
|
|
53
|
+
const recoveryEnvironment = /** @type {Record<string, string>} */ (Object.fromEntries(Object.entries(process.env).filter((entry) => entry[1] !== undefined)))
|
|
54
|
+
const recoveryLogPath = options.guardianDaemonLogPath ? path.resolve(options.guardianDaemonLogPath) : undefined
|
|
55
|
+
const recoveryPidPath = options.guardianDaemonPidPath ? path.resolve(options.guardianDaemonPidPath) : undefined
|
|
56
|
+
const recoveryStartupTimeoutMs = normalizeTimeoutMs(options.guardianDaemonStartTimeoutMs)
|
|
50
57
|
const daemon = new RollbridgeDaemon({
|
|
51
58
|
bootstrap,
|
|
52
59
|
config,
|
|
53
60
|
configPath,
|
|
54
61
|
legacyIncumbentPid: positiveIntegerOrUndefined(options.legacyIncumbentPid, "legacy incumbent pid"),
|
|
62
|
+
recoveryCommand: {
|
|
63
|
+
args: [
|
|
64
|
+
path.resolve(argv[1]), "daemon", "--config", configPath,
|
|
65
|
+
...(recoveryLogPath ? ["--guardian-daemon-log-path", recoveryLogPath] : []),
|
|
66
|
+
...(recoveryPidPath ? ["--guardian-daemon-pid-path", recoveryPidPath] : []),
|
|
67
|
+
"--guardian-daemon-start-timeout-ms", String(recoveryStartupTimeoutMs)
|
|
68
|
+
],
|
|
69
|
+
cwd: process.cwd(),
|
|
70
|
+
env: recoveryEnvironment,
|
|
71
|
+
executable: process.execPath,
|
|
72
|
+
logPath: recoveryLogPath,
|
|
73
|
+
pidPath: recoveryPidPath,
|
|
74
|
+
startupTimeoutMs: recoveryStartupTimeoutMs
|
|
75
|
+
},
|
|
55
76
|
runtime
|
|
56
77
|
})
|
|
78
|
+
let startupPromise = Promise.resolve()
|
|
79
|
+
let shutdownRequested = false
|
|
80
|
+
const shutdown = async () => {
|
|
81
|
+
shutdownRequested = true
|
|
82
|
+
await startupPromise.catch(() => undefined)
|
|
83
|
+
await daemon.shutdown()
|
|
84
|
+
process.exit(0)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
process.once("SIGINT", () => { void shutdown() })
|
|
88
|
+
process.once("SIGTERM", () => { void shutdown() })
|
|
57
89
|
|
|
58
90
|
if (options.takeoverOwner && (!bootstrap || !bootstrap.attestation)) throw new Error("Daemon --takeover-owner requires the complete bootstrap release tuple and --boot-attestation.")
|
|
59
91
|
|
|
60
92
|
if (options.replaceOwner) {
|
|
61
93
|
if (bootstrap || options.takeoverOwner) throw new Error("Daemon --replace-owner cannot be combined with bootstrap takeover options.")
|
|
62
|
-
|
|
94
|
+
startupPromise = daemon.replaceIncompatibleOwner()
|
|
95
|
+
await startupPromise
|
|
63
96
|
}
|
|
64
97
|
|
|
65
98
|
if (!options.takeoverOwner && !options.replaceOwner) {
|
|
66
99
|
try {
|
|
67
|
-
|
|
100
|
+
startupPromise = daemon.start({exposeControl: !bootstrap})
|
|
101
|
+
await startupPromise
|
|
68
102
|
} catch (error) {
|
|
69
103
|
if (!config.ownerRecovery) throw error
|
|
70
104
|
|
|
71
105
|
const winner = await sendControlCommand({command: {command: "status"}, path: config.control.path}).catch(() => undefined)
|
|
72
106
|
const matchingWinner = winner?.application === config.application &&
|
|
73
107
|
winner.ownerRecovery && typeof winner.ownerRecovery === "object" && !Array.isArray(winner.ownerRecovery) &&
|
|
74
|
-
winner.ownerRecovery.configDigest === daemon.ownerRecoveryConfigDigest() &&
|
|
108
|
+
winner.ownerRecovery.configDigest === daemon.ownerRecoveryConfigDigest() && winner.ownerRecovery.ready === true &&
|
|
75
109
|
((!runtime && !winner.daemonRuntime) || (runtime && winner.daemonRuntime && typeof winner.daemonRuntime === "object" && !Array.isArray(winner.daemonRuntime) && winner.daemonRuntime.digest === runtime.digest))
|
|
76
110
|
|
|
77
111
|
if (!matchingWinner) throw error
|
|
@@ -79,14 +113,7 @@ export async function runCli(argv) {
|
|
|
79
113
|
return
|
|
80
114
|
}
|
|
81
115
|
}
|
|
82
|
-
|
|
83
|
-
const shutdown = async () => {
|
|
84
|
-
await daemon.shutdown()
|
|
85
|
-
process.exit(0)
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
process.once("SIGINT", () => { void shutdown() })
|
|
89
|
-
process.once("SIGTERM", () => { void shutdown() })
|
|
116
|
+
if (shutdownRequested) return
|
|
90
117
|
|
|
91
118
|
if (bootstrap) {
|
|
92
119
|
try {
|
|
@@ -105,6 +132,7 @@ export async function runCli(argv) {
|
|
|
105
132
|
if (config.ownerRecovery && daemon.releases.size > 0) {
|
|
106
133
|
if (daemon.activeRelease) {
|
|
107
134
|
await daemon.exposeControl()
|
|
135
|
+
await publishDaemonReadiness(daemon, recoveryPidPath)
|
|
108
136
|
return
|
|
109
137
|
}
|
|
110
138
|
await daemon.abandonOwnerRecoveryAttempt()
|
|
@@ -124,6 +152,7 @@ export async function runCli(argv) {
|
|
|
124
152
|
return
|
|
125
153
|
}
|
|
126
154
|
}
|
|
155
|
+
if (!shutdownRequested) await publishDaemonReadiness(daemon, recoveryPidPath)
|
|
127
156
|
})
|
|
128
157
|
|
|
129
158
|
program
|
|
@@ -184,6 +213,31 @@ export async function runCli(argv) {
|
|
|
184
213
|
console.log(JSON.stringify(response, null, 2))
|
|
185
214
|
})
|
|
186
215
|
|
|
216
|
+
program
|
|
217
|
+
.command("recover-generation-transition")
|
|
218
|
+
.description("Restore the exact incumbent and retire a failed pre-commit generation candidate.")
|
|
219
|
+
.option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
|
|
220
|
+
.requiredOption("--release-path <path>", "Failed candidate release path")
|
|
221
|
+
.requiredOption("--release-id <id>", "Failed candidate release id")
|
|
222
|
+
.requiredOption("--revision <sha>", "Failed candidate revision")
|
|
223
|
+
.requiredOption("--previous-release-id <id>", "Expected authoritative incumbent release id")
|
|
224
|
+
.action(async (options) => {
|
|
225
|
+
const configPath = await resolveConfigPath(options.config)
|
|
226
|
+
const config = await loadConfig(configPath)
|
|
227
|
+
const response = await sendControlCommand({
|
|
228
|
+
command: {
|
|
229
|
+
command: "recover-generation-transition",
|
|
230
|
+
previousReleaseId: options.previousReleaseId,
|
|
231
|
+
releaseId: options.releaseId,
|
|
232
|
+
releasePath: options.releasePath,
|
|
233
|
+
revision: options.revision
|
|
234
|
+
},
|
|
235
|
+
path: config.control.path
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
console.log(JSON.stringify(response, null, 2))
|
|
239
|
+
})
|
|
240
|
+
|
|
187
241
|
program
|
|
188
242
|
.command("ensure-daemon")
|
|
189
243
|
.description("Start the daemon if the control socket is not already accepting commands.")
|
|
@@ -839,6 +893,8 @@ async function validateDaemonBootstrapOptions(options) {
|
|
|
839
893
|
*/
|
|
840
894
|
async function ensureDaemonRunning({config, configPath, logPath, pidPath, runtimePath, timeoutMs}) {
|
|
841
895
|
const runtime = await prepareDaemonRuntime(runtimePath || defaultDaemonRuntimePath(config))
|
|
896
|
+
const resolvedConfigPath = path.resolve(configPath)
|
|
897
|
+
const resolvedLogPath = path.resolve(logPath || defaultDaemonLogPath(config))
|
|
842
898
|
const existingStatus = await daemonStatus(config)
|
|
843
899
|
const expectedConfigDigest = ownerConfigDigest(config)
|
|
844
900
|
|
|
@@ -848,26 +904,30 @@ async function ensureDaemonRunning({config, configPath, logPath, pidPath, runtim
|
|
|
848
904
|
existingStatus.ownerRecovery.configDigest === expectedConfigDigest
|
|
849
905
|
)
|
|
850
906
|
|
|
851
|
-
if (compatibleDaemonRuntime(existingStatus, runtime) && matchingRecoveryAuthority)
|
|
907
|
+
if (compatibleDaemonRuntime(existingStatus, runtime) && matchingRecoveryAuthority) {
|
|
908
|
+
if (!config.ownerRecovery || (existingStatus.ownerRecovery && typeof existingStatus.ownerRecovery === "object" && !Array.isArray(existingStatus.ownerRecovery) && existingStatus.ownerRecovery.ready === true)) return existingStatus
|
|
909
|
+
return await waitForDaemonStatus(config, timeoutMs, {configDigest: expectedConfigDigest, runtime})
|
|
910
|
+
}
|
|
852
911
|
if (!config.ownerRecovery) assertCompatibleDaemonRuntime(existingStatus, runtime)
|
|
853
912
|
}
|
|
854
913
|
|
|
855
914
|
const persistedState = config.ownerRecovery && config.statePath ? await readState(config.statePath) : undefined
|
|
856
915
|
const persistedOwner = persistedState && typeof persistedState === "object" && !Array.isArray(persistedState) ? persistedState : undefined
|
|
857
916
|
const persistedRecovery = persistedOwner?.recovery
|
|
858
|
-
const replacement = Boolean(config.ownerRecovery && (existingStatus || (persistedRecovery && typeof persistedRecovery === "object" && !Array.isArray(persistedRecovery)
|
|
859
|
-
|
|
860
|
-
))))
|
|
861
|
-
const resolvedPidPath = pidPath || defaultDaemonPidPath(config)
|
|
917
|
+
const replacement = Boolean(config.ownerRecovery && (existingStatus || (persistedRecovery && typeof persistedRecovery === "object" && !Array.isArray(persistedRecovery))))
|
|
918
|
+
const resolvedPidPath = path.resolve(pidPath || defaultDaemonPidPath(config))
|
|
862
919
|
const legacyIncumbentPid = replacement ? await readDaemonPid(resolvedPidPath) : undefined
|
|
863
920
|
|
|
864
921
|
await fsPromises.mkdir(path.dirname(resolvedPidPath), {recursive: true})
|
|
865
922
|
const candidate = await startDaemonProcess({
|
|
866
|
-
configPath,
|
|
867
|
-
|
|
923
|
+
configPath: resolvedConfigPath,
|
|
924
|
+
cwd: path.dirname(resolvedConfigPath),
|
|
925
|
+
logPath: resolvedLogPath,
|
|
926
|
+
pidPath: resolvedPidPath,
|
|
868
927
|
replacement,
|
|
869
928
|
legacyIncumbentPid,
|
|
870
|
-
runtime
|
|
929
|
+
runtime,
|
|
930
|
+
timeoutMs
|
|
871
931
|
})
|
|
872
932
|
let startedStatus
|
|
873
933
|
|
|
@@ -885,8 +945,11 @@ async function ensureDaemonRunning({config, configPath, logPath, pidPath, runtim
|
|
|
885
945
|
)
|
|
886
946
|
|
|
887
947
|
if (!startedPid) throw new Error("Started Rollbridge daemon did not report its exact PID")
|
|
888
|
-
await fsPromises.writeFile(resolvedPidPath, `${startedPid}\n`)
|
|
948
|
+
if (!config.ownerRecovery) await fsPromises.writeFile(resolvedPidPath, `${startedPid}\n`)
|
|
889
949
|
return startedStatus
|
|
950
|
+
} catch (error) {
|
|
951
|
+
killDetachedProcessGroup(candidate)
|
|
952
|
+
throw error
|
|
890
953
|
} finally {
|
|
891
954
|
candidate.unref()
|
|
892
955
|
}
|
|
@@ -917,13 +980,16 @@ async function daemonStatus(config) {
|
|
|
917
980
|
* Starts the foreground daemon command as a detached child.
|
|
918
981
|
* @param {object} args - Options.
|
|
919
982
|
* @param {string} args.configPath - Config path.
|
|
983
|
+
* @param {string} args.cwd - Durable daemon working directory.
|
|
920
984
|
* @param {string} args.logPath - Log file path.
|
|
985
|
+
* @param {string} args.pidPath - PID file path.
|
|
921
986
|
* @param {boolean} args.replacement - Whether to run the incompatible replacement transaction.
|
|
922
987
|
* @param {number | undefined} args.legacyIncumbentPid - Exact incumbent recorded before candidate spawn.
|
|
923
988
|
* @param {import("./daemon-runtime.js").DaemonRuntimeIdentity} args.runtime - Prepared runtime.
|
|
989
|
+
* @param {number} args.timeoutMs - Guardian recovery startup timeout.
|
|
924
990
|
* @returns {Promise<import("node:child_process").ChildProcess>} Referenced child after exact spawn completion.
|
|
925
991
|
*/
|
|
926
|
-
async function startDaemonProcess({configPath, legacyIncumbentPid, logPath, replacement = false, runtime}) {
|
|
992
|
+
async function startDaemonProcess({configPath, cwd, legacyIncumbentPid, logPath, pidPath, replacement = false, runtime, timeoutMs}) {
|
|
927
993
|
await fsPromises.mkdir(path.dirname(logPath), {recursive: true})
|
|
928
994
|
|
|
929
995
|
const stdoutFd = fs.openSync(logPath, "a")
|
|
@@ -932,9 +998,13 @@ async function startDaemonProcess({configPath, legacyIncumbentPid, logPath, repl
|
|
|
932
998
|
try {
|
|
933
999
|
const child = spawn(process.execPath, [
|
|
934
1000
|
path.join(runtime.path, "bin", "rollbridge"), "daemon", "--config", configPath,
|
|
1001
|
+
"--guardian-daemon-log-path", logPath,
|
|
1002
|
+
"--guardian-daemon-pid-path", pidPath,
|
|
1003
|
+
"--guardian-daemon-start-timeout-ms", String(timeoutMs),
|
|
935
1004
|
...(replacement ? ["--replace-owner"] : []),
|
|
936
1005
|
...(legacyIncumbentPid ? ["--legacy-incumbent-pid", String(legacyIncumbentPid)] : [])
|
|
937
1006
|
], {
|
|
1007
|
+
cwd,
|
|
938
1008
|
detached: true,
|
|
939
1009
|
env: {...process.env, ROLLBRIDGE_DAEMON_RUNTIME_MANIFEST: path.join(runtime.path, "runtime.json")},
|
|
940
1010
|
stdio: ["ignore", stdoutFd, stderrFd]
|
|
@@ -1026,11 +1096,10 @@ async function waitForDaemonStatus(config, timeoutMs, expected = {}) {
|
|
|
1026
1096
|
|
|
1027
1097
|
if (status) {
|
|
1028
1098
|
const statusPid = typeof status.daemonPid === "number" ? status.daemonPid : undefined
|
|
1029
|
-
const candidateResolved = !expected.candidate || candidateExit || statusPid === expected.candidate.pid
|
|
1030
|
-
|
|
1031
1099
|
if (expected.runtime && !compatibleDaemonRuntime(status, expected.runtime)) {
|
|
1032
1100
|
if (!expected.configDigest) assertCompatibleDaemonRuntime(status, expected.runtime)
|
|
1033
|
-
} else if (
|
|
1101
|
+
} else if (!expected.configDigest || (status.ownerRecovery && typeof status.ownerRecovery === "object" && !Array.isArray(status.ownerRecovery) && status.ownerRecovery.configDigest === expected.configDigest && status.ownerRecovery.ready === true)) {
|
|
1102
|
+
if (expected.candidate && !candidateExit && statusPid !== expected.candidate.pid) killDetachedProcessGroup(expected.candidate)
|
|
1034
1103
|
return status
|
|
1035
1104
|
}
|
|
1036
1105
|
}
|
|
@@ -1058,6 +1127,31 @@ async function waitForDaemonStatus(config, timeoutMs, expected = {}) {
|
|
|
1058
1127
|
}
|
|
1059
1128
|
}
|
|
1060
1129
|
|
|
1130
|
+
/**
|
|
1131
|
+
* Publishes the exact daemon PID before confirming completed startup to its guardian.
|
|
1132
|
+
* @param {RollbridgeDaemon} daemon - Started daemon.
|
|
1133
|
+
* @param {string | undefined} pidPath - Optional daemon PID file.
|
|
1134
|
+
*/
|
|
1135
|
+
async function publishDaemonReadiness(daemon, pidPath) {
|
|
1136
|
+
if (daemon.guardian) {
|
|
1137
|
+
await daemon.markOwnerReady()
|
|
1138
|
+
} else if (pidPath) {
|
|
1139
|
+
await fsPromises.mkdir(path.dirname(pidPath), {recursive: true})
|
|
1140
|
+
await fsPromises.writeFile(pidPath, `${process.pid}\n`)
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
/** @param {import("node:child_process").ChildProcess} child - Exact detached process-group leader. */
|
|
1145
|
+
function killDetachedProcessGroup(child) {
|
|
1146
|
+
if (!child.pid) return
|
|
1147
|
+
try {
|
|
1148
|
+
process.kill(-child.pid, "SIGKILL")
|
|
1149
|
+
} catch (error) {
|
|
1150
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") return
|
|
1151
|
+
throw error
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1061
1155
|
/**
|
|
1062
1156
|
* @param {import("node:child_process").ChildProcess} candidate - Exact spawned daemon candidate.
|
|
1063
1157
|
* @param {{code: number | null, signal: DaemonExitSignal}} exit - Exact exit status.
|