rollbridge 0.1.70 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -63
- package/TODO.md +0 -13
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +65 -387
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +22 -212
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -435
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +5 -119
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +21 -125
- package/test/owner-replacement.test.js +14 -175
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +72 -737
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
package/docs/troubleshooting.md
CHANGED
|
@@ -27,33 +27,6 @@ preparation itself fails, check permissions for
|
|
|
27
27
|
`/tmp/rollbridge-<user-id>-<application-hash>-runtime`) before retrying. The directory
|
|
28
28
|
must be private to the invoking user.
|
|
29
29
|
|
|
30
|
-
## Retained guardian on an older rollbridge version
|
|
31
|
-
|
|
32
|
-
**Symptom.** `ensure-daemon` reports `The persistent Rollbridge guardian is version <x> while
|
|
33
|
-
this daemon runs rollbridge <y>` and confirms that the deploy was not sent.
|
|
34
|
-
|
|
35
|
-
**Cause.** The daemon reuses `recovery.guardian` from the state file on every start, so a
|
|
36
|
-
supervisor left running from an earlier release outlives the package it came from. A version
|
|
37
|
-
mismatch is normally reconciled without an operator: `ensure-daemon` starts a coordinator of its
|
|
38
|
-
own version over the retained guardian, the retained guardian keeps supervising the processes it
|
|
39
|
-
already owns, and nothing waits for them to drain. `status.ownerTransition` then reports
|
|
40
|
-
`mode: "guardian-version-handoff"` with `disruptive: true`; the incumbent daemon is retired while
|
|
41
|
-
its supervised application processes keep their exact PIDs.
|
|
42
|
-
|
|
43
|
-
**Fix.** The refusal is the last resort and names both versions plus the exact commands. Run them
|
|
44
|
-
in that order:
|
|
45
|
-
|
|
46
|
-
```sh
|
|
47
|
-
rollbridge shutdown --config /etc/rollbridge/app.cjs
|
|
48
|
-
rollbridge ensure-daemon --config /etc/rollbridge/app.cjs
|
|
49
|
-
```
|
|
50
|
-
|
|
51
|
-
It fires only when no handoff can be authorised: no incumbent daemon to hand guardian ownership
|
|
52
|
-
from, no incumbent PID reported by `ensure-daemon`, a guardian version that cannot be attested, an
|
|
53
|
-
incumbent PID that is not the attested owner, or a config change in the same invocation — the
|
|
54
|
-
bridge keeps the incumbent config identity unchanged, so apply a config or socket change in a
|
|
55
|
-
second invocation once the supervisor is current.
|
|
56
|
-
|
|
57
30
|
If startup instead reports `Rollbridge daemon candidate <pid> exited before
|
|
58
31
|
readiness`, the ensuring CLI observed that exact child exit before it could
|
|
59
32
|
attest status. The diagnostic includes the exit code or signal and spawned
|
|
@@ -81,30 +54,14 @@ Health check failed for http://127.0.0.1:18182/ping: HTTP 503
|
|
|
81
54
|
new release never went live; the previous release stays active.
|
|
82
55
|
|
|
83
56
|
**Diagnose.** The new release's `proxied` process didn't return a healthy
|
|
84
|
-
response in time.
|
|
85
|
-
`
|
|
86
|
-
`
|
|
87
|
-
|
|
88
|
-
`failedProcessId`, `readyProcesses`/`unreadyProcesses`, and the process's
|
|
89
|
-
`lastOutput`. Check that process's output with `rollbridge logs --process <id>`
|
|
90
|
-
and its state/`exitCode` with `rollbridge status`. Common causes: the app doesn't
|
|
91
|
-
listen on the templated `{{port}}`, the `health.path` returns a non-2xx status, or
|
|
92
|
-
the app boots slower than `health.timeoutMs`.
|
|
93
|
-
|
|
94
|
-
If the boot itself is fine but the app waits for a same-release service inside
|
|
95
|
-
its own command, that wait is inside the health deadline. `stage: "dependency"`
|
|
96
|
-
plus a `release process dependency ready` event shows the wait Rollbridge did on
|
|
97
|
-
the dependency's own budget.
|
|
57
|
+
response in time. Check its output with `rollbridge logs --process <id>` and its
|
|
58
|
+
state/`exitCode` with `rollbridge status`. Common causes: the app doesn't listen
|
|
59
|
+
on the templated `{{port}}`, the `health.path` returns a non-2xx status, or the
|
|
60
|
+
app boots slower than `health.timeoutMs`.
|
|
98
61
|
|
|
99
62
|
**Fix.** Make the proxied command bind `{{port}}` and serve `health.path` with a
|
|
100
|
-
2xx status.
|
|
101
|
-
|
|
102
|
-
the app's own boot. For an app that needs longer than the default, raise
|
|
103
|
-
`health.timeoutMs` above the measured boot time (or set `health.startDelayMs` so
|
|
104
|
-
probing begins after the app is up). For a
|
|
105
|
-
same-release service dependency, declare `dependsOn` (see
|
|
106
|
-
[`processes[].dependsOn`](config.md#processesdependson)) so the wait runs on
|
|
107
|
-
`dependencyTimeoutMs` instead of eating the boot budget.
|
|
63
|
+
2xx status. For slow boots, raise `health.timeoutMs` or set `health.startDelayMs`
|
|
64
|
+
so probing begins after the app is up.
|
|
108
65
|
|
|
109
66
|
## Port conflicts / exhausted ranges
|
|
110
67
|
|
|
@@ -191,51 +148,3 @@ separate destructive `--takeover-owner` path. If
|
|
|
191
148
|
`retirementError` is set,
|
|
192
149
|
inspect the quiet-hook events. Rollbridge deliberately leaves that generation
|
|
193
150
|
alive rather than signaling arbitrary PIDs or continuing its stop sequence.
|
|
194
|
-
|
|
195
|
-
**A drain recorded but a process group is still alive.** `release drained`
|
|
196
|
-
appears, the release reports `state: "stopped"`, yet `processes[].pid` or
|
|
197
|
-
`processes[].heldProcessGroup` is still reported and later deploys keep naming
|
|
198
|
-
this generation `wedged`. That is a wedged non-blocking drain: its retirement
|
|
199
|
-
budget expired, the release finished retiring, and the process group it left
|
|
200
|
-
behind still holds the generation.
|
|
201
|
-
|
|
202
|
-
**Diagnose.** Read `drain.reason` for the classification and the evidence behind
|
|
203
|
-
it, and `heldProcessGroup: {liveMemberCount, pgid}` for the group that outlived
|
|
204
|
-
its tracked process. `escalation: refused` in a
|
|
205
|
-
`release wedged generation left alive` line names why the supervisor declined to
|
|
206
|
-
terminate it — usually a runner child that is still live, which is exactly the
|
|
207
|
-
case a `SIGKILL` must never reach.
|
|
208
|
-
|
|
209
|
-
**Fix.** With `releaseRetention.escalateWedgedGenerations: true`, the next
|
|
210
|
-
deploy terminates the group when all four proofs hold (no live connection, every
|
|
211
|
-
retirement budget expired, no runner child remaining, CPU flat across a full
|
|
212
|
-
20 000 ms window) and logs `release wedged generation escalated`. It does nothing
|
|
213
|
-
when any proof is missing, so enabling it does not turn a slow drain into a kill.
|
|
214
|
-
Without it, the log line is the deliverable: it names the pids and process groups
|
|
215
|
-
to signal by hand. A wedge that persists across many deploys with live runner
|
|
216
|
-
children is a drain that cannot finish, not a supervisor fault — see
|
|
217
|
-
[`docs/config.md`](config.md#the-wedged-generation-escalation).
|
|
218
|
-
|
|
219
|
-
## A command is refused with "Another owner mutation is already in progress"
|
|
220
|
-
|
|
221
|
-
**Symptom.** `deploy`, `stop`, `restart`, `rollback` or
|
|
222
|
-
`recover-generation-transition` fails with
|
|
223
|
-
`Another owner mutation is already in progress: owner mutation "<operation>";
|
|
224
|
-
mutation <id>; held by process <pid>; since <ISO-8601>; <n>s elapsed; waiting
|
|
225
|
-
for <blocker>; refused <operation>`.
|
|
226
|
-
|
|
227
|
-
**Cause.** The committed daemon holds the guardian's owner-mutation fence for
|
|
228
|
-
one control mutation at a time, and a second mutation arrived while it was still
|
|
229
|
-
held.
|
|
230
|
-
|
|
231
|
-
**Fix.** Read the named holder before touching anything: the message states
|
|
232
|
-
which mutation holds the fence, its lease id, the PID of the process that
|
|
233
|
-
started it, when it started, how long it has run, and the blocker that holder
|
|
234
|
-
last reported (`waitingFor`, which a running mutation can refine through
|
|
235
|
-
`reportOwnerMutationWait`). That is the command to inspect — not the one that
|
|
236
|
-
was refused. A mutation that outruns its lease (15 minutes by default,
|
|
237
|
-
overridable per command with `mutationTimeoutMs`) is logged as
|
|
238
|
-
`OWNER_MUTATION_LEASE_EXPIRED`, reported to the holder, and released, so the
|
|
239
|
-
next mutation proceeds and the stuck one fails loudly when it next ends its
|
|
240
|
-
lease. A retained jobs generation is not an owner mutation and may still drain
|
|
241
|
-
for hours; only the fence is bounded.
|
package/docs/velocious.md
CHANGED
package/docs/workers.md
CHANGED
|
@@ -26,7 +26,6 @@ fixed port such as `7330`.
|
|
|
26
26
|
{
|
|
27
27
|
id: "background-jobs-worker",
|
|
28
28
|
policy: "companion",
|
|
29
|
-
dependsOn: ["background-jobs-main"],
|
|
30
29
|
env: {VELOCIOUS_BACKGROUND_JOBS_PORT: "{{ports.background-jobs-main}}"},
|
|
31
30
|
command: "npx velocious background-jobs-worker",
|
|
32
31
|
replicas: 4,
|
|
@@ -107,15 +106,5 @@ Returned work may then run in the active generation. Normal retirement does not
|
|
|
107
106
|
simulate a disconnect and does not make a new jobs-main adopt the old worker's
|
|
108
107
|
handoffs.
|
|
109
108
|
|
|
110
|
-
The supervisor's answer to a drain that cannot finish is separate from these
|
|
111
|
-
timeouts. `releaseRetention.escalateWedgedGenerations` (off by default) lets the
|
|
112
|
-
generation supervisor terminate a process group it has proven wedged and holding
|
|
113
|
-
nothing — no live connection, every retirement budget expired, no runner child
|
|
114
|
-
remaining, CPU flat across a full sample window — at the end of a retirement
|
|
115
|
-
transition or at the next deploy. It never waits for a drain and never kills a
|
|
116
|
-
process that still holds a live runner; a wedge it may not terminate is reported
|
|
117
|
-
with its pids instead. See
|
|
118
|
-
[`docs/config.md`](config.md#the-wedged-generation-escalation).
|
|
119
|
-
|
|
120
109
|
See [`docs/velocious.md`](velocious.md) for the complete topology and
|
|
121
110
|
[`docs/config.md`](config.md#processesdeploystrategy) for handoff-service fields.
|
|
@@ -20,7 +20,7 @@ export default {
|
|
|
20
20
|
host: "127.0.0.1",
|
|
21
21
|
port: 4500,
|
|
22
22
|
healthPath: "/ping",
|
|
23
|
-
healthTimeoutMs:
|
|
23
|
+
healthTimeoutMs: 30000,
|
|
24
24
|
drainTimeoutMs: 60000,
|
|
25
25
|
forceStopTimeoutMs: 10000
|
|
26
26
|
},
|
|
@@ -60,7 +60,6 @@ export default {
|
|
|
60
60
|
{
|
|
61
61
|
id: "background-jobs-worker",
|
|
62
62
|
policy: "companion",
|
|
63
|
-
dependsOn: ["background-jobs-main"],
|
|
64
63
|
cwd: "{{releasePath}}/backend",
|
|
65
64
|
env: {
|
|
66
65
|
NODE_ENV: "production",
|
|
@@ -68,7 +67,7 @@ export default {
|
|
|
68
67
|
VELOCIOUS_BEACON_PORT: "{{ports.beacon}}",
|
|
69
68
|
VELOCIOUS_BACKGROUND_JOBS_PORT: "{{ports.background-jobs-main}}"
|
|
70
69
|
},
|
|
71
|
-
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- npx velocious background-jobs-worker",
|
|
70
|
+
command: "wait-for-it 127.0.0.1:{{ports.beacon}} --strict -- wait-for-it 127.0.0.1:{{ports.background-jobs-main}} --strict -- npx velocious background-jobs-worker",
|
|
72
71
|
lifecycle: {
|
|
73
72
|
quietCommand: "appctl jobs-worker-retire --pid $ROLLBRIDGE_PID",
|
|
74
73
|
reactivateCommand: "appctl jobs-worker-reactivate --pid $ROLLBRIDGE_PID"
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -10,7 +10,6 @@ import RollbridgeDaemon, {ownerConfigDigest} from "./daemon.js"
|
|
|
10
10
|
import {loadDaemonRuntimeIdentity, prepareDaemonRuntime} from "./daemon-runtime.js"
|
|
11
11
|
import {loadConfig, parseConfigFile, resolveConfigPath, validateConfig} from "./config.js"
|
|
12
12
|
import {runEnvironmentChecks, runReleaseChecks} from "./doctor.js"
|
|
13
|
-
import {composeUnitStatus, rollComposeUnit, rollbackComposeUnit} from "./compose-unit.js"
|
|
14
13
|
import {predeployCleanup} from "./predeploy-cleanup.js"
|
|
15
14
|
import {recoverOrphans} from "./recover.js"
|
|
16
15
|
import {sendControlCommand} from "./control-client.js"
|
|
@@ -248,49 +247,6 @@ export async function runCli(argv) {
|
|
|
248
247
|
console.log(JSON.stringify(response, null, 2))
|
|
249
248
|
})
|
|
250
249
|
|
|
251
|
-
program
|
|
252
|
-
.command("compose-roll")
|
|
253
|
-
.description("Drain-first roll of one Docker Compose service: quiesce it, wait for in-flight work, recreate it with the target image, gate on activation, and roll back on failure.")
|
|
254
|
-
.option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
|
|
255
|
-
.requiredOption("--unit <id>", "Compose unit id declared in the config")
|
|
256
|
-
.option("--image <reference>", "Target image reference (defaults to the unit's configured image)")
|
|
257
|
-
.action(async (options) => {
|
|
258
|
-
const configPath = await resolveConfigPath(options.config)
|
|
259
|
-
const config = await loadConfig(configPath)
|
|
260
|
-
const result = await rollComposeUnit({config, image: options.image, unit: requireComposeUnit(config, options.unit, configPath)})
|
|
261
|
-
|
|
262
|
-
console.log(JSON.stringify(result, null, 2))
|
|
263
|
-
if (result.status === "failed") process.exitCode = 1
|
|
264
|
-
})
|
|
265
|
-
|
|
266
|
-
program
|
|
267
|
-
.command("compose-rollback")
|
|
268
|
-
.description("Roll one Compose unit back to its retained previous image through the same drain-first path.")
|
|
269
|
-
.option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
|
|
270
|
-
.requiredOption("--unit <id>", "Compose unit id declared in the config")
|
|
271
|
-
.option("--image <reference>", "Earlier image reference (defaults to the retained previous one)")
|
|
272
|
-
.action(async (options) => {
|
|
273
|
-
const configPath = await resolveConfigPath(options.config)
|
|
274
|
-
const config = await loadConfig(configPath)
|
|
275
|
-
const result = await rollbackComposeUnit({config, image: options.image, unit: requireComposeUnit(config, options.unit, configPath)})
|
|
276
|
-
|
|
277
|
-
console.log(JSON.stringify(result, null, 2))
|
|
278
|
-
if (result.status === "failed") process.exitCode = 1
|
|
279
|
-
})
|
|
280
|
-
|
|
281
|
-
program
|
|
282
|
-
.command("compose-status")
|
|
283
|
-
.description("Report a Compose unit's retained image generations and its live container state.")
|
|
284
|
-
.option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
|
|
285
|
-
.requiredOption("--unit <id>", "Compose unit id declared in the config")
|
|
286
|
-
.action(async (options) => {
|
|
287
|
-
const configPath = await resolveConfigPath(options.config)
|
|
288
|
-
const config = await loadConfig(configPath)
|
|
289
|
-
const status = await composeUnitStatus({config, unit: requireComposeUnit(config, options.unit, configPath)})
|
|
290
|
-
|
|
291
|
-
console.log(JSON.stringify(status, null, 2))
|
|
292
|
-
})
|
|
293
|
-
|
|
294
250
|
program
|
|
295
251
|
.command("ensure-daemon")
|
|
296
252
|
.description("Start the daemon if the control socket is not already accepting commands.")
|
|
@@ -411,7 +367,7 @@ export async function runCli(argv) {
|
|
|
411
367
|
const valid = issues.length === 0
|
|
412
368
|
|
|
413
369
|
if (options.json) {
|
|
414
|
-
const summary = valid ? {application: config.application,
|
|
370
|
+
const summary = valid ? {application: config.application, processes: config.processes.length, proxy: {host: config.proxy.host, port: config.proxy.port}} : null
|
|
415
371
|
|
|
416
372
|
console.log(JSON.stringify({config: summary, issues, path: configPath, valid}, null, 2))
|
|
417
373
|
if (!valid) process.exitCode = 1
|
|
@@ -420,10 +376,8 @@ export async function runCli(argv) {
|
|
|
420
376
|
|
|
421
377
|
if (valid) {
|
|
422
378
|
const processCount = config.processes.length
|
|
423
|
-
const composeCount = config.compose?.length ?? 0
|
|
424
|
-
const composeSummary = composeCount > 0 ? `, ${composeCount} Compose unit${composeCount === 1 ? "" : "s"}` : ""
|
|
425
379
|
|
|
426
|
-
console.log(`${configPath} is valid: ${processCount} ${processCount === 1 ? "process" : "processes"}
|
|
380
|
+
console.log(`${configPath} is valid: ${processCount} ${processCount === 1 ? "process" : "processes"}, proxy on ${config.proxy.host}:${config.proxy.port}.`)
|
|
427
381
|
return
|
|
428
382
|
}
|
|
429
383
|
|
|
@@ -605,26 +559,6 @@ export async function runCli(argv) {
|
|
|
605
559
|
await program.parseAsync(argv)
|
|
606
560
|
}
|
|
607
561
|
|
|
608
|
-
/**
|
|
609
|
-
* Resolves a Compose unit by id, naming the configured units when the id is unknown.
|
|
610
|
-
* @param {import("./config.js").RollbridgeConfig} config - Loaded config.
|
|
611
|
-
* @param {string} unitId - Requested Compose unit id.
|
|
612
|
-
* @param {string} configPath - Config path shown in the error.
|
|
613
|
-
* @returns {import("./config.js").ComposeUnitConfig} The configured unit.
|
|
614
|
-
*/
|
|
615
|
-
function requireComposeUnit(config, unitId, configPath) {
|
|
616
|
-
const units = config.compose ?? []
|
|
617
|
-
const unit = units.find((candidate) => candidate.id === unitId)
|
|
618
|
-
|
|
619
|
-
if (unit) return unit
|
|
620
|
-
|
|
621
|
-
if (units.length === 0) {
|
|
622
|
-
throw new Error(`${configPath} declares no compose units. Add a compose: [{id: "${unitId}", ...}] entry to roll it drain-first.`)
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
throw new Error(`No compose unit "${unitId}" in ${configPath}. Configured units: ${units.map((candidate) => candidate.id).join(", ")}.`)
|
|
626
|
-
}
|
|
627
|
-
|
|
628
562
|
/**
|
|
629
563
|
* @param {import("./predeploy-cleanup.js").PredeployCleanupResult} result - Cleanup result.
|
|
630
564
|
* @returns {string} Human-readable summary.
|