rollbridge 0.1.62 → 0.1.63

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ Retired releases now complete their transition after the configurable
2
+ `proxy.nonBlockingDrainTimeoutMs` wait even when a non-blocking process has an
3
+ indefinite graceful stop. Rollbridge records the process id and leaves that
4
+ process alive, preserving in-flight work while allowing the guardian to record
5
+ `release drained` and stop the handoff services.
@@ -178,7 +178,7 @@ export default {
178
178
  projectName: "peakflow_builder",
179
179
  image: "ghcr.io/kaspernj/peakflow-builder:amd64",
180
180
  stopGracePeriodMs: 60000,
181
- inFlight: {command: "test -z \"$(docker exec peakflow_builder-docker-server-1 sh -c 'ls /var/lib/docker/buildkit 2>/dev/null')\"", intervalMs: 5000, timeoutMs: 1800000},
181
+ inFlight: {command: "node /srv/peakflow-builder/scripts/quiescence-probe.js", intervalMs: 5000, timeoutMs: 1800000},
182
182
  activation: {startDelayMs: 5000, timeoutMs: 180000, readyCommand: "docker inspect -f '{{.State.Running}}' peakflow_builder-docker-server-1 | grep -q true"}
183
183
  }]
184
184
  }
@@ -210,11 +210,13 @@ roll is strictly better than `up -d --force-recreate` but not yet a true drain.
210
210
  live afterwards, the previous tag is retained in the ledger, and rolling to an image whose
211
211
  healthcheck never passes leaves the previous container serving and answering its readiness
212
212
  endpoint. It also proves the proxy-port refusal against a real published port.
213
- - **Not implemented here**: the drain and in-flight hooks only work if the service exposes a
214
- quiescence command and an in-flight probe. The Socketduct hub/gateway images and the builder
215
- DinD need those hooks (or an equivalent probe) before their units can be declared; the example
216
- `appctl drain` / `appctl in-flight` commands above name the contract, not something those images
217
- ship today.
213
+ - **Not implemented here**: the roll only runs whatever command a unit names; the hooks themselves
214
+ live in the service images. The Socketduct hub and gateway images now ship `appctl drain` and
215
+ `appctl in-flight` with the exit contract this document gates on (`0` the condition holds,
216
+ `1` it does not hold yet, `2` the probe cannot answer), so the example commands above name
217
+ something those images ship. The builder DinD still has no drain hook — its gate is the
218
+ in-flight probe in `kaspernj/tensorbuzz-builder` (`scripts/quiescence-probe.js`), which reports
219
+ whether a build is mid-flight.
218
220
  - **Not changed here**: TensorBuzz's Socketduct provisioning path still issues raw
219
221
  `docker compose up -d --force-recreate` when it provisions gateway legs. Switching that backend
220
222
  to call `rollbridge compose-roll` is a TensorBuzz change, tracked separately from this one.
package/docs/config.md CHANGED
@@ -79,6 +79,7 @@ to let a deploy group talk to the daemon.
79
79
  | `proxy.healthPath` | string | `"/ping"` | Default health-check path for proxied processes. |
80
80
  | `proxy.healthTimeoutMs` | number | `120000` | Default boot budget for proxied processes: how long a release may take to answer its health path for the first time. Measured on TensorBuzz production (server3, 2026-11-06): the web app was still initialising when a `30000` deadline expired, so two deploys were killed mid-boot; `120000` is the budget the same release publishes on today. Waiting for a `dependsOn` service is not charged to it. |
81
81
  | `proxy.drainTimeoutMs` | number | `60000` | How long to drain HTTP/WebSocket connections before stopping the retired proxied process. Expiry never stops an independently draining jobs generation. |
82
+ | `proxy.nonBlockingDrainTimeoutMs` | number | `60000` | How long release retirement waits for an independently draining process before recording it as still draining and completing the release. Expiry never sends `SIGKILL`; the process keeps its live work. |
82
83
  | `proxy.forceStopTimeoutMs` | number | `10000` | Default per-process graceful-stop timeout (`SIGTERM`, then `SIGKILL`). |
83
84
 
84
85
  ## `releaseRetention`
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "rollbridge",
3
- "version": "0.1.62",
3
+ "version": "0.1.63",
4
4
  "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
5
5
  "keywords": [
6
6
  "deploy",
package/src/config.js CHANGED
@@ -20,7 +20,7 @@ import {processStartOrder} from "./process-order.js"
20
20
  * @typedef {"persistent" | "handoff"} ServiceDeployStrategy
21
21
  * @typedef {{cwd?: string, dependencyTimeoutMs: number, dependsOn: string[], deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
22
22
  * @typedef {{group?: number | string, mode?: number, owner?: number | string, path: string}} ControlConfig
23
- * @typedef {{drainTimeoutMs: number, forceStopTimeoutMs: number, healthPath: string, healthTimeoutMs: number, host: string, port: number, upstreamHost: string}} ProxyConfig
23
+ * @typedef {{drainTimeoutMs: number, forceStopTimeoutMs: number, healthPath: string, healthTimeoutMs: number, host: string, nonBlockingDrainTimeoutMs: number, port: number, upstreamHost: string}} ProxyConfig
24
24
  * @typedef {{includes: string[], name: string}} LegacyTakeoverProcessConfig
25
25
  * @typedef {{forceStopTimeoutMs: number, processes: LegacyTakeoverProcessConfig[], screens: string[]}} LegacyTakeoverConfig
26
26
  * @typedef {{keep: number, maxAgeMs: number}} ReleaseRetentionConfig
@@ -226,6 +226,7 @@ function normalizeProxy(source, issues) {
226
226
  healthPath: normalizeString(source.healthPath, "proxy.healthPath", issues, {default: "/ping"}),
227
227
  healthTimeoutMs: normalizeNumber(source.healthTimeoutMs, "proxy.healthTimeoutMs", issues, {default: DEFAULT_HEALTH_TIMEOUT_MS}),
228
228
  host,
229
+ nonBlockingDrainTimeoutMs: normalizeNumber(source.nonBlockingDrainTimeoutMs, "proxy.nonBlockingDrainTimeoutMs", issues, {default: 60000}),
229
230
  port: normalizeNumber(source.port, "proxy.port", issues, {default: 8182}),
230
231
  upstreamHost: normalizeString(source.upstreamHost, "proxy.upstreamHost", issues, {default: defaultUpstreamHost(host)})
231
232
  }
@@ -73,6 +73,7 @@ export default class ManagedProcess extends EventEmitter {
73
73
  this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
74
74
  this.quiesceError = /** @type {Error | undefined} */ (undefined)
75
75
  this.stopPromise = /** @type {Promise<void> | undefined} */ (undefined)
76
+ this.retirementTimedOut = false
76
77
  this.operationRevision = 0
77
78
  this.restartTimer = undefined
78
79
  this.child = undefined
@@ -100,6 +101,7 @@ export default class ManagedProcess extends EventEmitter {
100
101
  this.quiescePromise = undefined
101
102
  this.quiesceError = undefined
102
103
  this.stopPromise = undefined
104
+ this.retirementTimedOut = false
103
105
  this.exitCode = undefined
104
106
  this.exitSignal = undefined
105
107
  this.state = "starting"
@@ -411,8 +413,8 @@ export default class ManagedProcess extends EventEmitter {
411
413
  }
412
414
 
413
415
  /**
414
- * @param {{timeoutMs?: number}} [options] - Stop options.
415
- * @returns {Promise<void>} Resolves when stopped.
416
+ * @param {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
417
+ * @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
416
418
  */
417
419
  async stop(options = {}) {
418
420
  this.operationRevision += 1
@@ -421,8 +423,8 @@ export default class ManagedProcess extends EventEmitter {
421
423
  }
422
424
 
423
425
  /**
424
- * @param {{timeoutMs?: number}} options - Stop options.
425
- * @returns {Promise<void>} Resolves when stopped.
426
+ * @param {{retirementTimeoutMs?: number, timeoutMs?: number}} options - Stop options.
427
+ * @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
426
428
  */
427
429
  async performStop(options) {
428
430
  const pgid = this.child?.pid ?? this.pid
@@ -448,6 +450,7 @@ export default class ManagedProcess extends EventEmitter {
448
450
  // 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
449
451
  if (this.processGroupExists(pgid)) {
450
452
  const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
453
+ const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
451
454
  let gracefulDeadline
452
455
 
453
456
  if (stopCommand) {
@@ -456,10 +459,18 @@ export default class ManagedProcess extends EventEmitter {
456
459
  } else {
457
460
  this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
458
461
  gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
459
- await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline)
462
+ await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline ?? retirementDeadline)
460
463
  }
461
464
 
462
- if (!(await this.waitForProcessGroupExit(pgid, gracefulDeadline))) {
465
+ const exited = await this.waitForProcessGroupExit(pgid, gracefulDeadline === undefined ? retirementDeadline : gracefulDeadline)
466
+
467
+ if (!exited && retirementDeadline !== undefined && gracefulDeadline === undefined) {
468
+ this.retirementTimedOut = true
469
+ this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
470
+ return
471
+ }
472
+
473
+ if (!exited) {
463
474
  this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: pgid})
464
475
  const killDeadline = Date.now() + 5000
465
476
 
@@ -843,7 +843,10 @@ export default class ReleaseGroup extends EventEmitter {
843
843
  // their lifecycle drain runs as soon as the release is retired — in parallel with the
844
844
  // connection drain, not held until after it. The rest stop once connections have closed.
845
845
  const entries = [...this.processes.entries()]
846
- const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([, processInstance]) => processInstance.stop())
846
+ const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
847
+ id,
848
+ promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
849
+ }))
847
850
  const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
848
851
  const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
849
852
 
@@ -864,12 +867,23 @@ export default class ReleaseGroup extends EventEmitter {
864
867
  }
865
868
 
866
869
  if (this.ownerHandoffPaused) {
867
- void Promise.allSettled(nonBlockingStops)
870
+ void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
868
871
  return
869
872
  }
870
873
 
871
874
  await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop()))
872
- await Promise.allSettled(nonBlockingStops)
875
+ const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
876
+ nonBlockingResults.forEach((result, index) => {
877
+ if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
878
+ const processInstance = this.processes.get(nonBlockingStops[index].id)
879
+ this.logger("release non-blocking process still draining", {
880
+ pid: processInstance?.status().pid ?? null,
881
+ processId: nonBlockingStops[index].id,
882
+ releaseId: this.releaseId,
883
+ timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
884
+ })
885
+ }
886
+ })
873
887
  await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
874
888
  this.state = "stopped"
875
889
  this.stoppedAt = new Date().toISOString()
@@ -308,6 +308,42 @@ test("runs quiet and drain lifecycle hooks before stopping", async () => {
308
308
  }
309
309
  })
310
310
 
311
+ test("retirement timeout leaves an indefinite worker alive without SIGKILL", async () => {
312
+ /** @type {string[]} */
313
+ const signals = []
314
+ const managed = new ManagedProcess({
315
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)")}`,
316
+ cwd: undefined,
317
+ env: {},
318
+ id: "worker",
319
+ lifecycle: {drainTimeoutMs: 0},
320
+ logger: () => {},
321
+ outputLines: 50,
322
+ restartDelayMs: 10,
323
+ shouldRestart: () => false,
324
+ stopSignal: "SIGTERM",
325
+ stopTimeoutMs: "indefinite"
326
+ })
327
+ const signalProcessGroup = managed.signalProcessGroup.bind(managed)
328
+ managed.signalProcessGroup = async (signal, pgid, deadline) => {
329
+ signals.push(signal)
330
+ await signalProcessGroup(signal, pgid, deadline)
331
+ }
332
+
333
+ try {
334
+ await managed.start()
335
+ await managed.stop({retirementTimeoutMs: 25})
336
+
337
+ expect(managed.retirementTimedOut).toBe(true)
338
+ expect(signals).toEqual(["SIGTERM"])
339
+ expect(managed.status().pid).toBeDefined()
340
+ } finally {
341
+ const pid = managed.status().pid
342
+ if (pid) process.kill(-pid, "SIGKILL")
343
+ if (managed.exitPromise) await managed.exitPromise
344
+ }
345
+ })
346
+
311
347
  test("a configured stopCommand is used instead of the stop signal", async () => {
312
348
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-hooks-"))
313
349
  const logPath = path.join(dir, "stop.log")
@@ -109,6 +109,34 @@ test("stopping a release abandons its connection count instead of persisting a s
109
109
  expect(release.localConnections()).toEqual({http: 0, websocket: 0})
110
110
  })
111
111
 
112
+ test("a wedged non-blocking process cannot block release retirement", async () => {
113
+ /** @type {Array<{message: string, data?: import("../src/json.js").JsonValue}>} */
114
+ const logs = []
115
+ const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
116
+ release.logger = (message, data) => logs.push({message, data})
117
+ release.state = "draining"
118
+ release.nonBlockingDrainIds.add("worker")
119
+ release.handoffServiceIds.add("jobs-main")
120
+ // eslint-disable-next-line local/no-unknown-jsdoc-type
121
+ release.processes.set("worker", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
122
+ quiesceStrict: async () => {},
123
+ status: () => ({pid: 4242}),
124
+ retirementTimedOut: true,
125
+ stop: async () => {}
126
+ })))
127
+ let handoffStopped = false
128
+ // eslint-disable-next-line local/no-unknown-jsdoc-type
129
+ release.processes.set("jobs-main", /** @type {import("../src/managed-process.js").default} */ (/** @type {unknown} */ ({
130
+ stop: async () => { handoffStopped = true }
131
+ })))
132
+
133
+ await release.drainAndStop(0, {...release.config, proxy: {...release.config.proxy, nonBlockingDrainTimeoutMs: 20}})
134
+
135
+ expect(release.state).toBe("stopped")
136
+ expect(handoffStopped).toBe(true)
137
+ expect(logs).toMatchObject([{message: "release non-blocking process still draining", data: {pid: 4242, processId: "worker"}}])
138
+ })
139
+
112
140
  test("restoring a stopped release does not re-adopt a stale connection count", async () => {
113
141
  const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}})
114
142