rollbridge 0.1.30 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +11 -6
- package/README.md +28 -12
- package/changelog.d/20260830-release-generation-activation-lifecycle.md +9 -0
- package/docs/cli.md +10 -7
- package/docs/config.md +41 -13
- package/docs/tensorbuzz-runbook.md +4 -2
- package/docs/velocious.md +19 -7
- package/docs/workers.md +18 -9
- package/examples/tensorbuzz.com.js +9 -5
- package/package.json +1 -1
- package/src/config.js +32 -2
- package/src/daemon.js +305 -42
- package/src/guardian-client.js +46 -3
- package/src/managed-process.js +85 -13
- package/src/process-guardian.js +45 -1
- package/src/release-group.js +53 -14
- package/test/config-validation.test.js +44 -0
- package/test/guardian-client.test.js +70 -0
- package/test/managed-process.test.js +33 -0
- package/test/owner-recovery.test.js +396 -6
- package/test/owner-replacement.test.js +153 -3
- package/test/rollbridge.test.js +333 -6
package/AGENTS.md
CHANGED
|
@@ -9,8 +9,10 @@ process lifecycle, retained generations, ports, and recovery state.
|
|
|
9
9
|
|
|
10
10
|
- One runtime generation is one release-scoped `background-jobs-main` plus its
|
|
11
11
|
worker pool. Start the complete candidate generation before activation.
|
|
12
|
-
-
|
|
13
|
-
|
|
12
|
+
- For an explicit candidate lifecycle, retire the old generation as one unit,
|
|
13
|
+
wait for its acknowledgement, then activate the candidate and commit traffic
|
|
14
|
+
synchronously. The old main stops schedules, new dispatch, and new ordinary
|
|
15
|
+
worker handoffs; its workers stop
|
|
14
16
|
accepting handoffs. The old main remains running with those workers and owns
|
|
15
17
|
their connections, lease fencing, report acceptance and acknowledgement, and
|
|
16
18
|
durable store transitions. The worker/reporting side durably retries terminal
|
|
@@ -40,10 +42,13 @@ implemented. Do not claim production compliance when source/config still uses a
|
|
|
40
42
|
fixed jobs-main, worker adoption by a new main, destructive orphan recovery, or
|
|
41
43
|
synchronous cleanup.
|
|
42
44
|
|
|
43
|
-
Current same-authority behavior
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
Current same-authority behavior preserves activate-then-retire ordering when no
|
|
46
|
+
activation hook is configured. An opt-in handoff-service `activateCommand`
|
|
47
|
+
durably journals old-retire/new-activate ordering, retains concurrent generations,
|
|
48
|
+
reports live release references, journals post-commit singleton completion, and
|
|
49
|
+
restores an active coordinator's generation-scoped role after process restart.
|
|
50
|
+
It requires `ownerRecovery` so a durable guardian preserves exact per-release
|
|
51
|
+
definitions and reconstructs active/draining generations after daemon process
|
|
47
52
|
exit. With `ownerRecovery` and the same `statePath` transaction anchor,
|
|
48
53
|
`ensure-daemon` also replaces incompatible config, control-socket, package, and
|
|
49
54
|
runtime owners through a guardian-fenced candidate-first handoff while retaining
|
package/README.md
CHANGED
|
@@ -49,6 +49,9 @@ export default {
|
|
|
49
49
|
path: "/tmp/rollbridge-ticket-server.sock"
|
|
50
50
|
},
|
|
51
51
|
|
|
52
|
+
statePath: "/var/lib/rollbridge/ticket-server.state.json",
|
|
53
|
+
ownerRecovery: {reconnectGraceMs: 30000},
|
|
54
|
+
|
|
52
55
|
proxy: {
|
|
53
56
|
host: "127.0.0.1",
|
|
54
57
|
port: 8182,
|
|
@@ -82,8 +85,12 @@ export default {
|
|
|
82
85
|
policy: "service",
|
|
83
86
|
deployStrategy: "handoff",
|
|
84
87
|
cwd: "{{releasePath}}",
|
|
88
|
+
env: {VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET: "{{releasePath}}/tmp/background-jobs-main.sock"},
|
|
85
89
|
command: "env VELOCIOUS_BACKGROUND_JOBS_PORT={{port}} npx velocious background-jobs-main",
|
|
86
|
-
lifecycle: {
|
|
90
|
+
lifecycle: {
|
|
91
|
+
activateCommand: 'npx velocious background-jobs:activate --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"',
|
|
92
|
+
quietCommand: 'npx velocious background-jobs:retire --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"'
|
|
93
|
+
},
|
|
87
94
|
port: {from: 7331, to: 7399}
|
|
88
95
|
},
|
|
89
96
|
{
|
|
@@ -98,8 +105,8 @@ export default {
|
|
|
98
105
|
}
|
|
99
106
|
```
|
|
100
107
|
|
|
101
|
-
|
|
102
|
-
|
|
108
|
+
The lifecycle socket path is illustrative; set it to the reviewed release-local
|
|
109
|
+
Velocious socket used by jobs-main.
|
|
103
110
|
|
|
104
111
|
Each process retains its most recent stdout/stderr lines and reports them in
|
|
105
112
|
`status`. Set `outputLines` (a positive integer, default 50) per process to keep
|
|
@@ -173,12 +180,16 @@ handoffs as soon as its release retires, independently of the proxied connection
|
|
|
173
180
|
drain. Its release-scoped handoff jobs-main remains running to supervise existing
|
|
174
181
|
handoffs and exits only after the worker pool has drained.
|
|
175
182
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
183
|
+
For a coordinator that starts quiescent, give the handoff jobs-main paired
|
|
184
|
+
`lifecycle.activateCommand` and `quietCommand` hooks. After candidate health,
|
|
185
|
+
Rollbridge waits for old retirement, waits for candidate activation, and commits
|
|
186
|
+
the active proxy target without an awaited boundary. The durable transition is
|
|
187
|
+
generation-scoped and resumable; failures remain visible and block unrelated
|
|
188
|
+
deploys. Post-commit singleton replacement is also journaled and must complete
|
|
189
|
+
before an exact retry reports success. If the active coordinator restarts,
|
|
190
|
+
Rollbridge restores its active role with the same bounded, generation-scoped
|
|
191
|
+
activation command before reporting it running. Omit `activateCommand` to
|
|
192
|
+
preserve the existing hook-free ordering.
|
|
182
193
|
|
|
183
194
|
See [`docs/workers.md`](docs/workers.md) for the full release-generation
|
|
184
195
|
deployment pattern: a handoff `background-jobs-main`, its companion worker pool,
|
|
@@ -384,7 +395,10 @@ candidate coordinator.
|
|
|
384
395
|
deployStrategy: "handoff",
|
|
385
396
|
cwd: "{{releasePath}}",
|
|
386
397
|
command: "npx velocious background-jobs-main",
|
|
387
|
-
lifecycle: {
|
|
398
|
+
lifecycle: {
|
|
399
|
+
activateCommand: 'npx velocious background-jobs:activate --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"',
|
|
400
|
+
quietCommand: 'npx velocious background-jobs:retire --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"'
|
|
401
|
+
},
|
|
388
402
|
port: {from: 7331, to: 7399}
|
|
389
403
|
}
|
|
390
404
|
```
|
|
@@ -396,8 +410,10 @@ On `rollbridge deploy`, the required ordering is:
|
|
|
396
410
|
1. starts any missing persistent service and the candidate's handoff services;
|
|
397
411
|
2. starts the new release's `companion`s, then its `proxied` process, and
|
|
398
412
|
health-checks the proxied process;
|
|
399
|
-
3.
|
|
400
|
-
|
|
413
|
+
3. when `activateCommand` is configured, retires and acknowledges the previous
|
|
414
|
+
jobs generation, then activates and acknowledges the candidate;
|
|
415
|
+
4. synchronously switches new traffic to the new release (hook-free configs keep
|
|
416
|
+
the existing switch-then-retire behavior);
|
|
401
417
|
5. replaces `singleton`s (stops the old one, then starts the new one);
|
|
402
418
|
6. returns success without waiting for the previous generation or its independent
|
|
403
419
|
HTTP/WebSocket drain; Rollbridge supervises all retained drains in the
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
### Added
|
|
2
|
+
|
|
3
|
+
- Add an opt-in, durable `lifecycle.activateCommand` for one release-scoped
|
|
4
|
+
handoff service. Rollbridge now supports strict old-retire acknowledgement,
|
|
5
|
+
candidate-activate acknowledgement, and synchronous active/proxy commit with
|
|
6
|
+
exact transition recovery and resume. Exact release definitions remain private
|
|
7
|
+
to guardian recovery, post-commit singleton work is resumable, unresolved
|
|
8
|
+
control mutations are fenced, and an active coordinator restores its role after
|
|
9
|
+
restart. Hook-free configs keep their existing deploy behavior.
|
package/docs/cli.md
CHANGED
|
@@ -149,16 +149,19 @@ for old workers, jobs, or HTTP/WebSocket connections to finish. Prints
|
|
|
149
149
|
If the new release fails to start or health-check, the previous release stays
|
|
150
150
|
active and the command errors.
|
|
151
151
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
152
|
+
With an opt-in handoff-service `lifecycle.activateCommand`, deploy journals the
|
|
153
|
+
exact transition, waits for old retirement acknowledgement, waits for candidate
|
|
154
|
+
activation acknowledgement, and synchronously commits the active proxy target.
|
|
155
|
+
An unresolved failure blocks different deploys; only the exact same release,
|
|
156
|
+
path, revision, and config authority may explicitly resume its incomplete
|
|
157
|
+
idempotent phase. A durable `committed_pending` phase keeps exact retry from
|
|
158
|
+
reporting success until singleton replacement finishes. Stop, restart, and
|
|
159
|
+
rollback mutations are rejected while the transition is unresolved. Hook-free
|
|
160
|
+
configs retain the existing post-activation quiet behavior and retirement result.
|
|
158
161
|
`status.releaseReferences` lists `{releaseId, releasePath}` for every active or
|
|
159
162
|
draining release and excludes fully stopped history.
|
|
160
163
|
|
|
161
|
-
|
|
164
|
+
For hook-free configs, after candidate activation `Daemon.deploy()` begins old-generation retirement
|
|
162
165
|
and asynchronous drain before awaiting singleton replacement. A singleton
|
|
163
166
|
replacement failure can therefore return non-zero while the candidate remains
|
|
164
167
|
active, but it cannot leave the old jobs generation dispatching.
|
package/docs/config.md
CHANGED
|
@@ -255,13 +255,16 @@ range** so old and new instances can run at the same time:
|
|
|
255
255
|
policy: "service",
|
|
256
256
|
deployStrategy: "handoff",
|
|
257
257
|
command: "npx velocious background-jobs-main",
|
|
258
|
-
lifecycle: {
|
|
258
|
+
lifecycle: {
|
|
259
|
+
activateCommand: 'npx velocious background-jobs:activate --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"',
|
|
260
|
+
quietCommand: 'npx velocious background-jobs:retire --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"'
|
|
261
|
+
},
|
|
259
262
|
port: {from: 7331, to: 7399}
|
|
260
263
|
}
|
|
261
264
|
```
|
|
262
265
|
|
|
263
|
-
The
|
|
264
|
-
|
|
266
|
+
The lifecycle socket path is illustrative; set it to the reviewed release-local
|
|
267
|
+
Velocious socket used by jobs-main.
|
|
265
268
|
|
|
266
269
|
Reference it from same-release processes with `{{ports.background-jobs-main}}`.
|
|
267
270
|
During a deploy, old workers keep the old port and new workers get the new port.
|
|
@@ -271,14 +274,38 @@ retirement must quiesce the old jobs-main's scheduling, dispatch, and new worker
|
|
|
271
274
|
handoffs while keeping it with its workers until their accepted work settles.
|
|
272
275
|
Workers are not adopted by the new service.
|
|
273
276
|
|
|
274
|
-
Configure `lifecycle.quietCommand` on
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
`
|
|
277
|
+
Configure `lifecycle.activateCommand` and `lifecycle.quietCommand` on one handoff
|
|
278
|
+
service when the service starts as a quiescent candidate and requires an explicit
|
|
279
|
+
generation transition. Rollbridge starts and health-checks the complete candidate,
|
|
280
|
+
waits for the old generation's strict retirement acknowledgement, waits for the
|
|
281
|
+
candidate's strict activation acknowledgement, then commits the active release and
|
|
282
|
+
proxy target synchronously. Activation is always bounded to 30 seconds;
|
|
283
|
+
retirement uses the process's `gracefulStopMs` bound (or 30 seconds when that
|
|
284
|
+
window is `"indefinite"`). Both run with the process environment plus
|
|
285
|
+
`ROLLBRIDGE_PID`.
|
|
286
|
+
|
|
287
|
+
This opt-in mode requires `statePath` and `ownerRecovery`. Rollbridge journals the
|
|
288
|
+
exact candidate, previous release, config authority, phase, and failure. An
|
|
289
|
+
unresolved transition blocks a different deploy; an explicit deploy with the same
|
|
290
|
+
release id, path, revision, and config may resume only its incomplete idempotent
|
|
291
|
+
phase. Once the health-ready candidate is journaled, its exact config becomes the
|
|
292
|
+
transition authority even if a later hook fails. A recorded failed hook is not
|
|
293
|
+
retried merely because daemon ownership changes. Omit `activateCommand` to retain
|
|
294
|
+
the existing activate-then-retire behavior.
|
|
295
|
+
|
|
296
|
+
The synchronous traffic assignment is persisted as `committed_pending` before
|
|
297
|
+
Rollbridge awaits singleton replacement. Exact retry or unambiguous owner recovery
|
|
298
|
+
finishes that idempotent post-commit work before the transition becomes
|
|
299
|
+
`committed` and deploy reports success. Stop, restart, and rollback operations are
|
|
300
|
+
rejected while any transition remains unresolved. Exact per-release definitions
|
|
301
|
+
stay in the guardian's authenticated private recovery state; `statePath` remains a
|
|
302
|
+
secret-safe status and transaction anchor.
|
|
303
|
+
|
|
304
|
+
An activation-owning handoff service also carries a durable desired role. If its
|
|
305
|
+
active process automatically or manually restarts, Rollbridge runs the exact
|
|
306
|
+
generation-scoped `activateCommand` once before marking it running. A failed role
|
|
307
|
+
restoration is surfaced and the process remains failed rather than being treated
|
|
308
|
+
as active. Retired generations remain fenced and are not restarted.
|
|
282
309
|
|
|
283
310
|
### `processes[].lifecycle`
|
|
284
311
|
|
|
@@ -294,7 +321,8 @@ legitimate hours-long generation drains are valid.
|
|
|
294
321
|
|
|
295
322
|
| Field | Type | Default | Description |
|
|
296
323
|
| --- | --- | --- | --- |
|
|
297
|
-
| `lifecycle.
|
|
324
|
+
| `lifecycle.activateCommand` | string | unset | For one handoff service, acknowledge activation of its already-started candidate generation after the previous generation has acknowledged retirement. Requires `quietCommand`, `statePath`, and `ownerRecovery`; bounded to 30 seconds. |
|
|
325
|
+
| `lifecycle.quietCommand` | string | unset | Run first to tell the process to stop accepting new work. Bounded by `gracefulStopMs`, or 30 seconds when that window is `"indefinite"`. |
|
|
298
326
|
| `lifecycle.drainCommand` | string | unset | Run after quieting to wait until the process has drained (it blocks until done). When unset, Rollbridge instead waits up to `drainTimeoutMs` for the process to exit on its own. Requires a positive `drainTimeoutMs` (which bounds it). |
|
|
299
327
|
| `lifecycle.drainTimeoutMs` | non-negative number | `0` | Bounds the drain step. `0` **skips the drain step entirely** (no `drainCommand`, no wait). |
|
|
300
328
|
| `lifecycle.stopCommand` | string | unset | Run to stop the process instead of sending `stopSignal`, if it is still running after draining. |
|
|
@@ -439,6 +467,6 @@ Rollbridge sets these in every managed process's environment (the process's own
|
|
|
439
467
|
- `restart.maxRestarts` must be a non-negative integer (omit it for unlimited restarts); `restart.backoffFactor` must be a number ≥ 1; `restart.windowMs` and `restart.maxDelayMs` must be non-negative numbers.
|
|
440
468
|
- When `memory` is set, `memory.limitBytes` must be a positive integer, `memory.warnBytes` a non-negative integer, and `memory.checkIntervalMs` a positive number.
|
|
441
469
|
- `replicas` must be a positive integer; `replicas > 1` is allowed only on a `companion` process without a `port`. Process ids must not contain `#` (reserved for replica instance ids).
|
|
442
|
-
- `lifecycle.quietCommand`/`drainCommand`/`stopCommand` must be strings when set, and `lifecycle.drainTimeoutMs` a non-negative number; `lifecycle.drainCommand` requires a positive `lifecycle.drainTimeoutMs`. A `lifecycle.stopCommand` may not be combined with a custom `stopSignal` (the `stopCommand` runs instead of the signal, so the signal would be ignored).
|
|
470
|
+
- `lifecycle.activateCommand`/`quietCommand`/`drainCommand`/`stopCommand` must be strings when set, and `lifecycle.drainTimeoutMs` a non-negative number; `lifecycle.drainCommand` requires a positive `lifecycle.drainTimeoutMs`. `activateCommand` is allowed on at most one handoff service, requires that service's `quietCommand`, and requires `statePath` plus `ownerRecovery`. A `lifecycle.stopCommand` may not be combined with a custom `stopSignal` (the `stopCommand` runs instead of the signal, so the signal would be ignored).
|
|
443
471
|
- `nonBlockingDrain` must be a boolean, and is allowed only on a `companion` process.
|
|
444
472
|
- `statePath` must be a string when set.
|
|
@@ -24,10 +24,12 @@ while a newer generation is active.
|
|
|
24
24
|
1. Prepare the candidate release and run backwards-compatible migrations.
|
|
25
25
|
2. Start the candidate jobs-main on a new port, then its worker pool and web
|
|
26
26
|
process. Health-check web before activation.
|
|
27
|
-
3.
|
|
28
|
-
|
|
27
|
+
3. Retire the previous jobs-main and workers as one generation and wait for its
|
|
28
|
+
exact lifecycle-socket acknowledgement. Jobs-main stops
|
|
29
29
|
schedule ownership, new dispatch, and new handoffs; workers stop accepting
|
|
30
30
|
handoffs.
|
|
31
|
+
4. Activate the candidate with its exact generation and lifecycle socket, then
|
|
32
|
+
synchronously switch new traffic.
|
|
31
33
|
5. Return deploy success and release the deploy lock. Do not wait for old jobs,
|
|
32
34
|
workers, jobs-main, HTTP/WebSocket connections, or other retained services.
|
|
33
35
|
|
package/docs/velocious.md
CHANGED
|
@@ -54,10 +54,14 @@ export default {
|
|
|
54
54
|
env: {
|
|
55
55
|
NODE_ENV: "production",
|
|
56
56
|
VELOCIOUS_BEACON_PORT: "{{ports.beacon}}",
|
|
57
|
-
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{port}}"
|
|
57
|
+
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{port}}",
|
|
58
|
+
VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET: "{{releasePath}}/tmp/background-jobs-main.sock"
|
|
58
59
|
},
|
|
59
60
|
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- npx velocious background-jobs-main",
|
|
60
|
-
lifecycle: {
|
|
61
|
+
lifecycle: {
|
|
62
|
+
activateCommand: 'npx velocious background-jobs:activate --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"',
|
|
63
|
+
quietCommand: 'npx velocious background-jobs:retire --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"'
|
|
64
|
+
},
|
|
61
65
|
port: {from: 7331, to: 7399}
|
|
62
66
|
},
|
|
63
67
|
{
|
|
@@ -91,8 +95,9 @@ export default {
|
|
|
91
95
|
}
|
|
92
96
|
```
|
|
93
97
|
|
|
94
|
-
|
|
95
|
-
|
|
98
|
+
The lifecycle socket path must match the release's reviewed Velocious generation
|
|
99
|
+
configuration. Rollbridge invokes the exact generation-scoped commands without
|
|
100
|
+
polling, PID discovery, or an application-specific wrapper.
|
|
96
101
|
|
|
97
102
|
Beacon keeps its fixed port because it is intentionally shared. Jobs-main uses a
|
|
98
103
|
range because every release gets its own coordinator. Same-release
|
|
@@ -104,9 +109,16 @@ old endpoint while candidate workers use the candidate endpoint.
|
|
|
104
109
|
Run backwards-compatible migrations before activation, then invoke
|
|
105
110
|
`rollbridge deploy` with the prepared release. Rollbridge starts the candidate
|
|
106
111
|
jobs-main, its complete worker pool, and the web process before health gating and
|
|
107
|
-
activation.
|
|
108
|
-
|
|
109
|
-
|
|
112
|
+
activation. Velocious generation mode starts jobs-main quiescent; the exact
|
|
113
|
+
`background-jobs:activate --generation … --socket …` acknowledgement is what
|
|
114
|
+
makes that generation active. A candidate startup or health failure leaves the
|
|
115
|
+
previous release active.
|
|
116
|
+
|
|
117
|
+
For the opt-in lifecycle above, Rollbridge first waits for the old generation's
|
|
118
|
+
retirement acknowledgement, then for candidate activation, and commits the active
|
|
119
|
+
proxy target synchronously. On the first deploy there is no retirement step.
|
|
120
|
+
If an active jobs-main process restarts, Rollbridge repeats that same exact,
|
|
121
|
+
idempotent activation command once before reporting the process running.
|
|
110
122
|
After successful activation, the deploy returns without waiting for any retired
|
|
111
123
|
generation or HTTP/WebSocket connection to finish. The old and new release code
|
|
112
124
|
may therefore overlap for hours. Keep schema, queue payloads, and external side
|
package/docs/workers.md
CHANGED
|
@@ -17,7 +17,10 @@ fixed port such as `7330`.
|
|
|
17
17
|
policy: "service",
|
|
18
18
|
deployStrategy: "handoff",
|
|
19
19
|
command: "npx velocious background-jobs-main",
|
|
20
|
-
lifecycle: {
|
|
20
|
+
lifecycle: {
|
|
21
|
+
activateCommand: 'npx velocious background-jobs:activate --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"',
|
|
22
|
+
quietCommand: 'npx velocious background-jobs:retire --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"'
|
|
23
|
+
},
|
|
21
24
|
port: {from: 7331, to: 7399}
|
|
22
25
|
},
|
|
23
26
|
{
|
|
@@ -31,8 +34,8 @@ fixed port such as `7330`.
|
|
|
31
34
|
}
|
|
32
35
|
```
|
|
33
36
|
|
|
34
|
-
|
|
35
|
-
reviewed
|
|
37
|
+
Set `VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET` in the service environment to
|
|
38
|
+
the release's reviewed lifecycle socket path.
|
|
36
39
|
|
|
37
40
|
Each worker receives its generation's jobs-main port. Old workers keep that port
|
|
38
41
|
for their entire lifetime; normal deploy draining never hands them to, or lets
|
|
@@ -43,23 +46,29 @@ them reconnect to, the new jobs-main. `replicas` scales the pool as
|
|
|
43
46
|
|
|
44
47
|
1. Before activation, Rollbridge starts the candidate release's jobs-main and
|
|
45
48
|
complete worker pool, then starts and health-checks the candidate web process.
|
|
46
|
-
2.
|
|
47
|
-
|
|
48
|
-
3. The previous jobs generation retires as one unit. Its jobs-main stops schedule
|
|
49
|
+
2. The previous jobs generation retires as one unit and acknowledges the exact
|
|
50
|
+
generation-scoped retirement command. Its jobs-main stops schedule
|
|
49
51
|
ownership, new queue dispatch, and new ordinary worker handoffs. Its workers
|
|
50
52
|
stop accepting handoffs.
|
|
51
|
-
|
|
53
|
+
3. The candidate acknowledges its exact generation-scoped activation command.
|
|
54
|
+
4. Rollbridge synchronously commits the active release and proxy target.
|
|
55
|
+
5. The old jobs-main stays running with its old workers. It continues owning
|
|
52
56
|
their connections and heartbeats, lease fencing, terminal-report acceptance
|
|
53
57
|
and acknowledgement, and durable store transitions. The old worker/reporting
|
|
54
58
|
side durably retries terminal reports, tracks outstanding report promises,
|
|
55
59
|
enforces per-job execution timeouts, and owns and reaps child runners.
|
|
56
|
-
|
|
60
|
+
6. Work returned or retried to the shared queue becomes eligible for the new
|
|
57
61
|
active generation. The retired main never dispatches it again.
|
|
58
|
-
|
|
62
|
+
7. The old main and workers remain one release generation until every accepted
|
|
59
63
|
handoff settles. Only then, after every old worker drains and exits, may the
|
|
60
64
|
old jobs-main exit. Rollbridge then reaps the generation and reports that its
|
|
61
65
|
release reference ended so Rampway can release the retention pin.
|
|
62
66
|
|
|
67
|
+
If the active jobs-main crashes or is manually restarted, Rollbridge restores its
|
|
68
|
+
active role with the exact generation-scoped activation command before reporting
|
|
69
|
+
the restarted process running. A retired generation remains fenced and is not
|
|
70
|
+
auto-restarted.
|
|
71
|
+
|
|
63
72
|
Old and new generations may overlap for hours, each running its own release code
|
|
64
73
|
and jobs-main endpoint. Multiple retired generations may drain concurrently.
|
|
65
74
|
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
//
|
|
3
3
|
// Nginx should keep proxying the backend host to 127.0.0.1:4500. Rollbridge
|
|
4
4
|
// binds that stable HTTP port, forwards to the active release's internal web
|
|
5
|
-
// port and keeps Beacon daemon-wide. The
|
|
6
|
-
//
|
|
7
|
-
//
|
|
5
|
+
// port and keeps Beacon daemon-wide. The lifecycle socket path must match the
|
|
6
|
+
// reviewed release-local Velocious jobs-main configuration; the worker appctl
|
|
7
|
+
// command remains an illustrative application-specific quiescence control.
|
|
8
8
|
|
|
9
9
|
export default {
|
|
10
10
|
application: "tensorbuzz",
|
|
@@ -47,10 +47,14 @@ export default {
|
|
|
47
47
|
NODE_ENV: "production",
|
|
48
48
|
VELOCIOUS_ENV: "production",
|
|
49
49
|
VELOCIOUS_BEACON_PORT: "{{ports.beacon}}",
|
|
50
|
-
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{port}}"
|
|
50
|
+
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{port}}",
|
|
51
|
+
VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET: "{{releasePath}}/tmp/background-jobs-main.sock"
|
|
51
52
|
},
|
|
52
53
|
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- npx velocious background-jobs-main",
|
|
53
|
-
lifecycle: {
|
|
54
|
+
lifecycle: {
|
|
55
|
+
activateCommand: 'npx velocious background-jobs:activate --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"',
|
|
56
|
+
quietCommand: 'npx velocious background-jobs:retire --generation "$ROLLBRIDGE_RELEASE_ID" --socket "$VELOCIOUS_BACKGROUND_JOBS_LIFECYCLE_SOCKET"'
|
|
57
|
+
},
|
|
54
58
|
port: {from: 7331, to: 7399}
|
|
55
59
|
},
|
|
56
60
|
{
|
package/package.json
CHANGED
package/src/config.js
CHANGED
|
@@ -14,7 +14,7 @@ import {pathToFileURL} from "node:url"
|
|
|
14
14
|
* @typedef {"proxied" | "companion" | "singleton" | "service"} ProcessPolicy
|
|
15
15
|
* @typedef {{backoffFactor: number, maxDelayMs: number, maxRestarts: number | undefined, windowMs: number}} RestartConfig
|
|
16
16
|
* @typedef {{checkIntervalMs: number, limitBytes: number, warnBytes: number}} MemoryConfig
|
|
17
|
-
* @typedef {{drainCommand?: string, drainTimeoutMs: number, quietCommand?: string, stopCommand?: string}} LifecycleConfig
|
|
17
|
+
* @typedef {{activateCommand?: string, drainCommand?: string, drainTimeoutMs: number, quietCommand?: string, stopCommand?: string}} LifecycleConfig
|
|
18
18
|
* @typedef {number | "indefinite"} StopTimeoutMs
|
|
19
19
|
* @typedef {"persistent" | "handoff"} ServiceDeployStrategy
|
|
20
20
|
* @typedef {{cwd?: string, deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
|
|
@@ -158,6 +158,7 @@ export function validateConfig(rawConfig, configPath = process.cwd()) {
|
|
|
158
158
|
if (ownerRecovery && !statePath) issues.push({fix: "Configure statePath when ownerRecovery is enabled.", message: "ownerRecovery requires statePath"})
|
|
159
159
|
|
|
160
160
|
validateProcessSet(processes, issues)
|
|
161
|
+
validateActivationLifecycle(processes, ownerRecovery, statePath, issues)
|
|
161
162
|
|
|
162
163
|
return {config: {application, control, legacyTakeover, ownerRecovery, processes, proxy, releaseRetention, statePath}, issues}
|
|
163
164
|
}
|
|
@@ -341,7 +342,7 @@ function normalizeLifecycle(value, key, issues) {
|
|
|
341
342
|
if (value === undefined || value === null) return {drainTimeoutMs: 0}
|
|
342
343
|
|
|
343
344
|
if (!isPlainObject(value)) {
|
|
344
|
-
issues.push({fix: `Set ${key} to a mapping with optional quietCommand, drainCommand, stopCommand, and drainTimeoutMs.`, message: `${key} must be an object`})
|
|
345
|
+
issues.push({fix: `Set ${key} to a mapping with optional activateCommand, quietCommand, drainCommand, stopCommand, and drainTimeoutMs.`, message: `${key} must be an object`})
|
|
345
346
|
|
|
346
347
|
return {drainTimeoutMs: 0}
|
|
347
348
|
}
|
|
@@ -350,6 +351,7 @@ function normalizeLifecycle(value, key, issues) {
|
|
|
350
351
|
/** @type {LifecycleConfig} */
|
|
351
352
|
const lifecycle = {drainTimeoutMs: nonNegativeOrDefault(drainTimeoutMs, `${key}.drainTimeoutMs`, issues, 0, false)}
|
|
352
353
|
|
|
354
|
+
if (value.activateCommand !== undefined) lifecycle.activateCommand = normalizeString(value.activateCommand, `${key}.activateCommand`, issues)
|
|
353
355
|
if (value.quietCommand !== undefined) lifecycle.quietCommand = normalizeString(value.quietCommand, `${key}.quietCommand`, issues)
|
|
354
356
|
if (value.drainCommand !== undefined) lifecycle.drainCommand = normalizeString(value.drainCommand, `${key}.drainCommand`, issues)
|
|
355
357
|
if (value.stopCommand !== undefined) lifecycle.stopCommand = normalizeString(value.stopCommand, `${key}.stopCommand`, issues)
|
|
@@ -361,6 +363,34 @@ function normalizeLifecycle(value, key, issues) {
|
|
|
361
363
|
return lifecycle
|
|
362
364
|
}
|
|
363
365
|
|
|
366
|
+
/**
|
|
367
|
+
* Validates the opt-in durable generation activation contract.
|
|
368
|
+
* @param {ProcessConfig[]} processes - Normalized process definitions.
|
|
369
|
+
* @param {OwnerRecoveryConfig | undefined} ownerRecovery - Durable owner recovery config.
|
|
370
|
+
* @param {string | undefined} statePath - Durable state transaction anchor.
|
|
371
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
372
|
+
*/
|
|
373
|
+
function validateActivationLifecycle(processes, ownerRecovery, statePath, issues) {
|
|
374
|
+
const activated = processes.filter((processConfig) => processConfig.lifecycle.activateCommand !== undefined)
|
|
375
|
+
|
|
376
|
+
if (activated.length > 1) {
|
|
377
|
+
issues.push({fix: "Configure lifecycle.activateCommand on only one release-generation coordinator.", message: "Config may define at most one lifecycle.activateCommand"})
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
for (const processConfig of activated) {
|
|
381
|
+
if (processConfig.policy !== "service" || processConfig.deployStrategy !== "handoff") {
|
|
382
|
+
issues.push({fix: `Set lifecycle.activateCommand only on a service using deployStrategy: "handoff"; "${processConfig.id}" is ${processConfig.policy}/${processConfig.deployStrategy}.`, message: `Process "${processConfig.id}" can only set lifecycle.activateCommand on a handoff service`})
|
|
383
|
+
}
|
|
384
|
+
if (!processConfig.lifecycle.quietCommand) {
|
|
385
|
+
issues.push({fix: `Add lifecycle.quietCommand to "${processConfig.id}" so every activated generation has a paired retirement command.`, message: `Process "${processConfig.id}" lifecycle.activateCommand requires lifecycle.quietCommand`})
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
if (activated.length > 0 && (!ownerRecovery || !statePath)) {
|
|
390
|
+
issues.push({fix: "Configure statePath and ownerRecovery for durable release-generation transition recovery.", message: "lifecycle.activateCommand requires ownerRecovery and statePath"})
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
364
394
|
/**
|
|
365
395
|
* @param {JsonValue} value - Raw legacy takeover config.
|
|
366
396
|
* @param {ProxyConfig} proxy - Proxy config defaults.
|