rollbridge 0.1.61 → 0.1.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/docs/compose-services.md +8 -6
- package/docs/config.md +1 -0
- package/package.json +1 -1
- package/src/config.js +2 -1
- package/src/managed-process.js +17 -6
- package/src/release-group.js +17 -3
- package/test/managed-process.test.js +36 -0
- package/test/release-group.test.js +28 -0
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
Retired releases now complete their transition after the configurable
|
|
2
|
+
`proxy.nonBlockingDrainTimeoutMs` wait even when a non-blocking process has an
|
|
3
|
+
indefinite graceful stop. Rollbridge records the process id and leaves that
|
|
4
|
+
process alive, preserving in-flight work while allowing the guardian to record
|
|
5
|
+
`release drained` and stop the handoff services.
|
package/docs/compose-services.md
CHANGED
|
@@ -178,7 +178,7 @@ export default {
|
|
|
178
178
|
projectName: "peakflow_builder",
|
|
179
179
|
image: "ghcr.io/kaspernj/peakflow-builder:amd64",
|
|
180
180
|
stopGracePeriodMs: 60000,
|
|
181
|
-
inFlight: {command: "
|
|
181
|
+
inFlight: {command: "node /srv/peakflow-builder/scripts/quiescence-probe.js", intervalMs: 5000, timeoutMs: 1800000},
|
|
182
182
|
activation: {startDelayMs: 5000, timeoutMs: 180000, readyCommand: "docker inspect -f '{{.State.Running}}' peakflow_builder-docker-server-1 | grep -q true"}
|
|
183
183
|
}]
|
|
184
184
|
}
|
|
@@ -210,11 +210,13 @@ roll is strictly better than `up -d --force-recreate` but not yet a true drain.
|
|
|
210
210
|
live afterwards, the previous tag is retained in the ledger, and rolling to an image whose
|
|
211
211
|
healthcheck never passes leaves the previous container serving and answering its readiness
|
|
212
212
|
endpoint. It also proves the proxy-port refusal against a real published port.
|
|
213
|
-
- **Not implemented here**: the
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
`
|
|
217
|
-
ship
|
|
213
|
+
- **Not implemented here**: the roll only runs whatever command a unit names; the hooks themselves
|
|
214
|
+
live in the service images. The Socketduct hub and gateway images now ship `appctl drain` and
|
|
215
|
+
`appctl in-flight` with the exit contract this document gates on (`0` the condition holds,
|
|
216
|
+
`1` it does not hold yet, `2` the probe cannot answer), so the example commands above name
|
|
217
|
+
something those images ship. The builder DinD still has no drain hook — its gate is the
|
|
218
|
+
in-flight probe in `kaspernj/tensorbuzz-builder` (`scripts/quiescence-probe.js`), which reports
|
|
219
|
+
whether a build is mid-flight.
|
|
218
220
|
- **Not changed here**: TensorBuzz's Socketduct provisioning path still issues raw
|
|
219
221
|
`docker compose up -d --force-recreate` when it provisions gateway legs. Switching that backend
|
|
220
222
|
to call `rollbridge compose-roll` is a TensorBuzz change, tracked separately from this one.
|
package/docs/config.md
CHANGED
|
@@ -79,6 +79,7 @@ to let a deploy group talk to the daemon.
|
|
|
79
79
|
| `proxy.healthPath` | string | `"/ping"` | Default health-check path for proxied processes. |
|
|
80
80
|
| `proxy.healthTimeoutMs` | number | `120000` | Default boot budget for proxied processes: how long a release may take to answer its health path for the first time. Measured on TensorBuzz production (server3, 2026-11-06): the web app was still initialising when a `30000` deadline expired, so two deploys were killed mid-boot; `120000` is the budget the same release publishes on today. Waiting for a `dependsOn` service is not charged to it. |
|
|
81
81
|
| `proxy.drainTimeoutMs` | number | `60000` | How long to drain HTTP/WebSocket connections before stopping the retired proxied process. Expiry never stops an independently draining jobs generation. |
|
|
82
|
+
| `proxy.nonBlockingDrainTimeoutMs` | number | `60000` | How long release retirement waits for an independently draining process before recording it as still draining and completing the release. Expiry never sends `SIGKILL`; the process keeps its live work. |
|
|
82
83
|
| `proxy.forceStopTimeoutMs` | number | `10000` | Default per-process graceful-stop timeout (`SIGTERM`, then `SIGKILL`). |
|
|
83
84
|
|
|
84
85
|
## `releaseRetention`
|
package/package.json
CHANGED
package/src/config.js
CHANGED
|
@@ -20,7 +20,7 @@ import {processStartOrder} from "./process-order.js"
|
|
|
20
20
|
* @typedef {"persistent" | "handoff"} ServiceDeployStrategy
|
|
21
21
|
* @typedef {{cwd?: string, dependencyTimeoutMs: number, dependsOn: string[], deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
|
|
22
22
|
* @typedef {{group?: number | string, mode?: number, owner?: number | string, path: string}} ControlConfig
|
|
23
|
-
* @typedef {{drainTimeoutMs: number, forceStopTimeoutMs: number, healthPath: string, healthTimeoutMs: number, host: string, port: number, upstreamHost: string}} ProxyConfig
|
|
23
|
+
* @typedef {{drainTimeoutMs: number, forceStopTimeoutMs: number, healthPath: string, healthTimeoutMs: number, host: string, nonBlockingDrainTimeoutMs: number, port: number, upstreamHost: string}} ProxyConfig
|
|
24
24
|
* @typedef {{includes: string[], name: string}} LegacyTakeoverProcessConfig
|
|
25
25
|
* @typedef {{forceStopTimeoutMs: number, processes: LegacyTakeoverProcessConfig[], screens: string[]}} LegacyTakeoverConfig
|
|
26
26
|
* @typedef {{keep: number, maxAgeMs: number}} ReleaseRetentionConfig
|
|
@@ -226,6 +226,7 @@ function normalizeProxy(source, issues) {
|
|
|
226
226
|
healthPath: normalizeString(source.healthPath, "proxy.healthPath", issues, {default: "/ping"}),
|
|
227
227
|
healthTimeoutMs: normalizeNumber(source.healthTimeoutMs, "proxy.healthTimeoutMs", issues, {default: DEFAULT_HEALTH_TIMEOUT_MS}),
|
|
228
228
|
host,
|
|
229
|
+
nonBlockingDrainTimeoutMs: normalizeNumber(source.nonBlockingDrainTimeoutMs, "proxy.nonBlockingDrainTimeoutMs", issues, {default: 60000}),
|
|
229
230
|
port: normalizeNumber(source.port, "proxy.port", issues, {default: 8182}),
|
|
230
231
|
upstreamHost: normalizeString(source.upstreamHost, "proxy.upstreamHost", issues, {default: defaultUpstreamHost(host)})
|
|
231
232
|
}
|
package/src/managed-process.js
CHANGED
|
@@ -73,6 +73,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
73
73
|
this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
74
74
|
this.quiesceError = /** @type {Error | undefined} */ (undefined)
|
|
75
75
|
this.stopPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
76
|
+
this.retirementTimedOut = false
|
|
76
77
|
this.operationRevision = 0
|
|
77
78
|
this.restartTimer = undefined
|
|
78
79
|
this.child = undefined
|
|
@@ -100,6 +101,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
100
101
|
this.quiescePromise = undefined
|
|
101
102
|
this.quiesceError = undefined
|
|
102
103
|
this.stopPromise = undefined
|
|
104
|
+
this.retirementTimedOut = false
|
|
103
105
|
this.exitCode = undefined
|
|
104
106
|
this.exitSignal = undefined
|
|
105
107
|
this.state = "starting"
|
|
@@ -411,8 +413,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
411
413
|
}
|
|
412
414
|
|
|
413
415
|
/**
|
|
414
|
-
* @param {{timeoutMs?: number}} [options] - Stop options.
|
|
415
|
-
* @returns {Promise<void>} Resolves when stopped.
|
|
416
|
+
* @param {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
|
|
417
|
+
* @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
|
|
416
418
|
*/
|
|
417
419
|
async stop(options = {}) {
|
|
418
420
|
this.operationRevision += 1
|
|
@@ -421,8 +423,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
421
423
|
}
|
|
422
424
|
|
|
423
425
|
/**
|
|
424
|
-
* @param {{timeoutMs?: number}} options - Stop options.
|
|
425
|
-
* @returns {Promise<void>} Resolves when stopped.
|
|
426
|
+
* @param {{retirementTimeoutMs?: number, timeoutMs?: number}} options - Stop options.
|
|
427
|
+
* @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
|
|
426
428
|
*/
|
|
427
429
|
async performStop(options) {
|
|
428
430
|
const pgid = this.child?.pid ?? this.pid
|
|
@@ -448,6 +450,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
448
450
|
// 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
|
|
449
451
|
if (this.processGroupExists(pgid)) {
|
|
450
452
|
const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
|
|
453
|
+
const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
|
|
451
454
|
let gracefulDeadline
|
|
452
455
|
|
|
453
456
|
if (stopCommand) {
|
|
@@ -456,10 +459,18 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
456
459
|
} else {
|
|
457
460
|
this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
|
|
458
461
|
gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
|
|
459
|
-
await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline)
|
|
462
|
+
await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline ?? retirementDeadline)
|
|
460
463
|
}
|
|
461
464
|
|
|
462
|
-
|
|
465
|
+
const exited = await this.waitForProcessGroupExit(pgid, gracefulDeadline === undefined ? retirementDeadline : gracefulDeadline)
|
|
466
|
+
|
|
467
|
+
if (!exited && retirementDeadline !== undefined && gracefulDeadline === undefined) {
|
|
468
|
+
this.retirementTimedOut = true
|
|
469
|
+
this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
|
|
470
|
+
return
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
if (!exited) {
|
|
463
474
|
this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: pgid})
|
|
464
475
|
const killDeadline = Date.now() + 5000
|
|
465
476
|
|
package/src/release-group.js
CHANGED
|
@@ -843,7 +843,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
843
843
|
// their lifecycle drain runs as soon as the release is retired — in parallel with the
|
|
844
844
|
// connection drain, not held until after it. The rest stop once connections have closed.
|
|
845
845
|
const entries = [...this.processes.entries()]
|
|
846
|
-
const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([, processInstance]) =>
|
|
846
|
+
const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
|
|
847
|
+
id,
|
|
848
|
+
promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
|
|
849
|
+
}))
|
|
847
850
|
const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
848
851
|
const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
849
852
|
|
|
@@ -864,12 +867,23 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
864
867
|
}
|
|
865
868
|
|
|
866
869
|
if (this.ownerHandoffPaused) {
|
|
867
|
-
void Promise.allSettled(nonBlockingStops)
|
|
870
|
+
void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
|
|
868
871
|
return
|
|
869
872
|
}
|
|
870
873
|
|
|
871
874
|
await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop()))
|
|
872
|
-
await Promise.allSettled(nonBlockingStops)
|
|
875
|
+
const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
|
|
876
|
+
nonBlockingResults.forEach((result, index) => {
|
|
877
|
+
if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
|
|
878
|
+
const processInstance = this.processes.get(nonBlockingStops[index].id)
|
|
879
|
+
this.logger("release non-blocking process still draining", {
|
|
880
|
+
pid: processInstance?.status().pid ?? null,
|
|
881
|
+
processId: nonBlockingStops[index].id,
|
|
882
|
+
releaseId: this.releaseId,
|
|
883
|
+
timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
|
|
884
|
+
})
|
|
885
|
+
}
|
|
886
|
+
})
|
|
873
887
|
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
|
|
874
888
|
this.state = "stopped"
|
|
875
889
|
this.stoppedAt = new Date().toISOString()
|
|
@@ -308,6 +308,42 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
|
|
|
308
308
|
}
|
|
309
309
|
})
|
|
310
310
|
|
|
311
|
+
test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
|
|
312
|
+
/** @type {string[]} */
|
|
313
|
+
const signals = []
|
|
314
|
+
const managed = new ManagedProcess({
|
|
315
|
+
command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)")}`,
|
|
316
|
+
cwd: undefined,
|
|
317
|
+
env: {},
|
|
318
|
+
id: "worker",
|
|
319
|
+
lifecycle: {drainTimeoutMs: 0},
|
|
320
|
+
logger: () => {},
|
|
321
|
+
outputLines: 50,
|
|
322
|
+
restartDelayMs: 10,
|
|
323
|
+
shouldRestart: () => false,
|
|
324
|
+
stopSignal: "SIGTERM",
|
|
325
|
+
stopTimeoutMs: "indefinite"
|
|
326
|
+
})
|
|
327
|
+
const signalProcessGroup = managed.signalProcessGroup.bind(managed)
|
|
328
|
+
managed.signalProcessGroup = async (signal, pgid, deadline) => {
|
|
329
|
+
signals.push(signal)
|
|
330
|
+
await signalProcessGroup(signal, pgid, deadline)
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
try {
|
|
334
|
+
await managed.start()
|
|
335
|
+
await managed.stop({retirementTimeoutMs: 25})
|
|
336
|
+
|
|
337
|
+
expect(managed.retirementTimedOut).toBe(true)
|
|
338
|
+
expect(signals).toEqual(["SIGTERM"])
|
|
339
|
+
expect(managed.status().pid).toBeDefined()
|
|
340
|
+
} finally {
|
|
341
|
+
const pid = managed.status().pid
|
|
342
|
+
if (pid) process.kill(-pid, "SIGKILL")
|
|
343
|
+
if (managed.exitPromise) await managed.exitPromise
|
|
344
|
+
}
|
|
345
|
+
})
|
|
346
|
+
|
|
311
347
|
test("a configured stopCommand is used instead of the stop signal", async () => {
|
|
312
348
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-hooks-"))
|
|
313
349
|
const logPath = path.join(dir, "stop.log")
|
|
@@ -109,6 +109,34 @@ test("stopping a release abandons its connection count instead of persisting a s
|
|
|
109
109
|
expect(release.localConnections()).toEqual({http: 0, websocket: 0})
|
|
110
110
|
})
|
|
111
111
|
|
|
112
|
+
test("a wedged non-blocking process cannot block release retirement", async () => {
|
|
113
|
+
/** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
|
|
114
|
+
const logs = []
|
|
115
|
+
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
|
|
116
|
+
release.logger = (message, data) => logs.push({message, data})
|
|
117
|
+
release.state = "draining"
|
|
118
|
+
release.nonBlockingDrainIds.add("worker")
|
|
119
|
+
release.handoffServiceIds.add("jobs-main")
|
|
120
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
121
|
+
release.processes.set("worker", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
122
|
+
quiesceStrict: async () => {},
|
|
123
|
+
status: () => ({pid: 4242}),
|
|
124
|
+
retirementTimedOut: true,
|
|
125
|
+
stop: async () => {}
|
|
126
|
+
})))
|
|
127
|
+
let handoffStopped = false
|
|
128
|
+
// eslint-disable-next-line local/no-unknown-jsdoc-type
|
|
129
|
+
release.processes.set("jobs-main", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
|
|
130
|
+
stop: async () => { handoffStopped = true }
|
|
131
|
+
})))
|
|
132
|
+
|
|
133
|
+
await release.drainAndStop(0, {...release.config, proxy: {...release.config.proxy, nonBlockingDrainTimeoutMs: 20}})
|
|
134
|
+
|
|
135
|
+
expect(release.state).toBe("stopped")
|
|
136
|
+
expect(handoffStopped).toBe(true)
|
|
137
|
+
expect(logs).toMatchObject([{message: "release non-blocking process still draining", data: {pid: 4242, processId: "worker"}}])
|
|
138
|
+
})
|
|
139
|
+
|
|
112
140
|
test("restoring a stopped release does not re-adopt a stale connection count", async () => {
|
|
113
141
|
const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
|
|
114
142
|
|