rollbridge 0.1.62 → 0.1.65

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,10 +2,13 @@
2
2
 
3
3
  import {EventEmitter} from "node:events"
4
4
  import {spawn} from "node:child_process"
5
- import {processGroupHasLiveMembers, processGroupMembers} from "./process-memory.js"
5
+ import {processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "./process-memory.js"
6
6
 
7
7
  const DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS = 30000
8
8
  const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
9
+ // Reading a process group means scanning procfs, and status() is read continuously. A held
10
+ // group is re-verified on this interval instead of on every read.
11
+ const HELD_GROUP_RECHECK_MS = 5000
9
12
 
10
13
  /**
11
14
  * @typedef {import("./json.js").JsonValue} JsonValue
@@ -16,7 +19,8 @@ const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
16
19
  * @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
17
20
  * @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
18
21
  * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
19
- * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
22
+ * @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
23
+ * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
20
24
  */
21
25
 
22
26
  export default class ManagedProcess extends EventEmitter {
@@ -73,6 +77,10 @@ export default class ManagedProcess extends EventEmitter {
73
77
  this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
74
78
  this.quiesceError = /** @type {Error | undefined} */ (undefined)
75
79
  this.stopPromise = /** @type {Promise<void> | undefined} */ (undefined)
80
+ this.retirementTimedOut = false
81
+ this.groupOutlivedStop = /** @type {number | undefined} */ (undefined)
82
+ this.heldProcessGroupCheckedAtMs = /** @type {number | undefined} */ (undefined)
83
+ this.heldProcessGroupCount = /** @type {number | undefined} */ (undefined)
76
84
  this.operationRevision = 0
77
85
  this.restartTimer = undefined
78
86
  this.child = undefined
@@ -100,6 +108,10 @@ export default class ManagedProcess extends EventEmitter {
100
108
  this.quiescePromise = undefined
101
109
  this.quiesceError = undefined
102
110
  this.stopPromise = undefined
111
+ this.retirementTimedOut = false
112
+ this.groupOutlivedStop = undefined
113
+ this.heldProcessGroupCheckedAtMs = undefined
114
+ this.heldProcessGroupCount = undefined
103
115
  this.exitCode = undefined
104
116
  this.exitSignal = undefined
105
117
  this.state = "starting"
@@ -250,6 +262,7 @@ export default class ManagedProcess extends EventEmitter {
250
262
  this.children = []
251
263
  this.clearMemoryMonitor()
252
264
  this.state = wasIntentional ? "stopped" : "failed"
265
+
253
266
  this.logger("process exited", {code, id: this.id, signal})
254
267
  this.emit("exit", {code, signal})
255
268
 
@@ -258,6 +271,69 @@ export default class ManagedProcess extends EventEmitter {
258
271
  }
259
272
  }
260
273
 
274
+ /**
275
+ * Records that the owned process group was still alive when the bounded stop gave up.
276
+ *
277
+ * Every process is spawned `detached: true`, so the tracked pid is the group leader and a
278
+ * member that survives its leader — a `sh` wrapper killed by the stop signal while the app
279
+ * it launched keeps running, or a runner child that outlives its parent — keeps holding the
280
+ * group id while the tracked ChildProcess record is already terminal. Nothing in rollbridge
281
+ * can wait for or signal that group again, yet the group still holds the release's work.
282
+ *
283
+ * Only the stop path can know this, and it already observed the group while waiting for it to
284
+ * leave, so the group id is recorded there instead of rediscovered in `onExit`: that hook
285
+ * fires for every retired process, and a procfs scan costs ~200 ms per exit on a busy host.
286
+ * @param {number} pgid - Process group id the tracked process led.
287
+ * @returns {void}
288
+ */
289
+ markGroupOutlivedStop(pgid) {
290
+ if (this.groupOutlivedStop === pgid) return
291
+
292
+ this.groupOutlivedStop = pgid
293
+ this.heldProcessGroupCheckedAtMs = undefined
294
+ this.heldProcessGroupCount = undefined
295
+ this.logger("process group outlived its tracked process", {id: this.id, pgid})
296
+ }
297
+
298
+ /**
299
+ * Reports the process group left behind by a bounded stop, as observed right now.
300
+ *
301
+ * The record is re-verified rather than trusted forever: once the orphaned members exit, the
302
+ * group is gone and stops holding the key. Re-verification is rate-limited to
303
+ * `HELD_GROUP_RECHECK_MS` because it costs a procfs scan, and when procfs cannot answer at
304
+ * all the last known state is reported instead of guessing that the group is gone.
305
+ * @returns {HeldProcessGroup | undefined} The held group, or undefined when nothing is held.
306
+ */
307
+ currentHeldProcessGroup() {
308
+ const pgid = this.groupOutlivedStop
309
+
310
+ if (pgid === undefined) return undefined
311
+ if (this.state !== "stopped" && this.state !== "failed") return undefined
312
+
313
+ const nowMs = Date.now()
314
+
315
+ if (this.heldProcessGroupCount !== undefined && this.heldProcessGroupCheckedAtMs !== undefined && nowMs - this.heldProcessGroupCheckedAtMs < HELD_GROUP_RECHECK_MS) {
316
+ return {liveMemberCount: this.heldProcessGroupCount, pgid}
317
+ }
318
+
319
+ const sample = sampleProcessGroup(pgid)
320
+
321
+ this.heldProcessGroupCheckedAtMs = nowMs
322
+
323
+ if (sample === undefined) return {liveMemberCount: this.heldProcessGroupCount, pgid}
324
+
325
+ if (sample.memberCount === 0) {
326
+ this.groupOutlivedStop = undefined
327
+ this.heldProcessGroupCount = undefined
328
+
329
+ return undefined
330
+ }
331
+
332
+ this.heldProcessGroupCount = sample.liveMemberCount
333
+
334
+ return {liveMemberCount: sample.liveMemberCount, pgid}
335
+ }
336
+
261
337
  /**
262
338
  * Schedules an automatic restart per the restart policy, or gives up once the policy's limit is reached.
263
339
  * @returns {void}
@@ -411,8 +487,8 @@ export default class ManagedProcess extends EventEmitter {
411
487
  }
412
488
 
413
489
  /**
414
- * @param {{timeoutMs?: number}} [options] - Stop options.
415
- * @returns {Promise<void>} Resolves when stopped.
490
+ * @param {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
491
+ * @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
416
492
  */
417
493
  async stop(options = {}) {
418
494
  this.operationRevision += 1
@@ -421,8 +497,8 @@ export default class ManagedProcess extends EventEmitter {
421
497
  }
422
498
 
423
499
  /**
424
- * @param {{timeoutMs?: number}} options - Stop options.
425
- * @returns {Promise<void>} Resolves when stopped.
500
+ * @param {{retirementTimeoutMs?: number, timeoutMs?: number}} options - Stop options.
501
+ * @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
426
502
  */
427
503
  async performStop(options) {
428
504
  const pgid = this.child?.pid ?? this.pid
@@ -448,6 +524,7 @@ export default class ManagedProcess extends EventEmitter {
448
524
  // 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
449
525
  if (this.processGroupExists(pgid)) {
450
526
  const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
527
+ const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
451
528
  let gracefulDeadline
452
529
 
453
530
  if (stopCommand) {
@@ -456,15 +533,25 @@ export default class ManagedProcess extends EventEmitter {
456
533
  } else {
457
534
  this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
458
535
  gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
459
- await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline)
536
+ await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline ?? retirementDeadline)
537
+ }
538
+
539
+ const exited = await this.waitForProcessGroupExit(pgid, gracefulDeadline === undefined ? retirementDeadline : gracefulDeadline)
540
+
541
+ if (!exited && retirementDeadline !== undefined && gracefulDeadline === undefined) {
542
+ this.retirementTimedOut = true
543
+ this.markGroupOutlivedStop(pgid)
544
+ this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
545
+ return
460
546
  }
461
547
 
462
- if (!(await this.waitForProcessGroupExit(pgid, gracefulDeadline))) {
548
+ if (!exited) {
463
549
  this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: pgid})
464
550
  const killDeadline = Date.now() + 5000
465
551
 
466
552
  await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
467
- await this.waitForProcessGroupExit(pgid, killDeadline)
553
+
554
+ if (!await this.waitForProcessGroupExit(pgid, killDeadline)) this.markGroupOutlivedStop(pgid)
468
555
  }
469
556
  }
470
557
 
@@ -817,6 +904,7 @@ export default class ManagedProcess extends EventEmitter {
817
904
  cwd: this.cwd,
818
905
  exitCode: this.exitCode,
819
906
  exitSignal: this.exitSignal,
907
+ heldProcessGroup: this.currentHeldProcessGroup(),
820
908
  id: this.id,
821
909
  lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
822
910
  lastStartReason: this.lastStartReason,
@@ -825,6 +913,7 @@ export default class ManagedProcess extends EventEmitter {
825
913
  memoryRestarts: this.memoryRestarts,
826
914
  pid: this.pid,
827
915
  restarts: this.restarts,
916
+ retirementTimedOut: this.retirementTimedOut,
828
917
  rssBytes: this.rssBytes,
829
918
  startedAt: this.startedAtMs === undefined ? undefined : new Date(this.startedAtMs).toISOString(),
830
919
  state: this.state,
@@ -832,3 +921,21 @@ export default class ManagedProcess extends EventEmitter {
832
921
  }
833
922
  }
834
923
  }
924
+
925
+ /**
926
+ * The guardian's provenance fence, in one place: a registered child stops fencing a later
927
+ * claim on its key only when its record is terminal *and* it holds no pid, and a process
928
+ * group that outlived the tracked process is not gone either.
929
+ *
930
+ * Both the guardian (re-registering a key under a new provenance) and the daemon (restoring a
931
+ * release that is recorded as drained) must agree on this, or a restore reaches the guardian
932
+ * and fails there with a bare `Guardian provenance mismatch` naming a process the operator
933
+ * cannot see in `status`.
934
+ * @param {Pick<ManagedProcessStatus, "heldProcessGroup" | "pid" | "state">} status - Process status record.
935
+ * @returns {boolean} True when the child is conclusively gone.
936
+ */
937
+ export function childRecordIsGone(status) {
938
+ if (status.heldProcessGroup) return false
939
+
940
+ return (status.state === "stopped" || status.state === "failed") && status.pid === undefined
941
+ }
@@ -8,7 +8,7 @@ import {spawn} from "node:child_process"
8
8
  import {isDeepStrictEqual} from "node:util"
9
9
  import path from "node:path"
10
10
  import GuardianClient from "./guardian-client.js"
11
- import ManagedProcess from "./managed-process.js"
11
+ import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
12
12
  import {writeState} from "./state-store.js"
13
13
 
14
14
  const OWNER_RESTART_RETRY_MS = 1000
@@ -656,10 +656,9 @@ async function execute(request, socket) {
656
656
  // residue of a dead generation, and demanding its provenance wedges every later deploy
657
657
  // against a process that no longer exists (TensorBuzz production, STC-695). The owner
658
658
  // may re-register the key under the new definition; live processes stay fail-closed.
659
- const existingStatus = existing.process.status()
660
- const childIsGone = (existingStatus.state === "stopped" || existingStatus.state === "failed") && existingStatus.pid === undefined
661
-
662
- if (!childIsGone) throw new Error(`Guardian provenance mismatch for ${request.key}`)
659
+ // A group that outlived its tracked process is not gone: it still holds the key's work,
660
+ // so re-registering it would start a second copy of a generation that is still running.
661
+ if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
663
662
  requireOwner(socket, request.command)
664
663
  processes.delete(request.key)
665
664
  }
@@ -4,8 +4,14 @@ import fs from "node:fs"
4
4
 
5
5
  /**
6
6
  * @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
7
+ * @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
7
8
  */
8
9
 
10
+ // Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
11
+ // userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
12
+ // not a rate: two reads of it measure whether a process did any work between them.
13
+ const CLOCK_TICKS_PER_MS = 10
14
+
9
15
  /**
10
16
  * Lists the members of a managed process group with each member's resident memory.
11
17
  * Rollbridge spawns each process detached, so the spawned pid is the process-group
@@ -83,6 +89,51 @@ export function measureProcessGroupRssBytes(pgid) {
83
89
  return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
84
90
  }
85
91
 
92
+ /**
93
+ * Samples one managed process group for drain liveness: how much CPU its members have
94
+ * consumed in total, and how many of them can still run.
95
+ *
96
+ * CPU is the sum of every member's cumulative user+system time, defunct members included:
97
+ * a member that exits takes its consumed time out of the sum, so the difference between two
98
+ * samples is non-zero whenever the group changed at all — new work or a departing member.
99
+ * Only an exactly unchanged difference says the group did nothing between the two reads.
100
+ * @param {number} pgid - Process-group id (the detached spawn's pid).
101
+ * @param {string} [procPath] - Procfs root (overridable for deterministic tests).
102
+ * @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
103
+ * read at all. A group with no members left is a sample of zero, not a missing measurement.
104
+ */
105
+ export function sampleProcessGroup(pgid, procPath = "/proc") {
106
+ /** @type {string[]} */
107
+ let entries
108
+
109
+ try {
110
+ entries = fs.readdirSync(procPath)
111
+ } catch {
112
+ return undefined
113
+ }
114
+
115
+ let cpuMs = 0
116
+ let liveMemberCount = 0
117
+ let memberCount = 0
118
+ let measurable = true
119
+
120
+ for (const entry of entries) {
121
+ if (!/^\d+$/.test(entry)) continue
122
+
123
+ const stat = processStat(entry, procPath)
124
+
125
+ if (stat?.pgrp !== pgid) continue
126
+
127
+ memberCount += 1
128
+ if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
129
+
130
+ if (stat.cpuMs === undefined) measurable = false
131
+ else cpuMs += stat.cpuMs
132
+ }
133
+
134
+ return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
135
+ }
136
+
86
137
  /**
87
138
  * @param {string} pid - Process id.
88
139
  * @returns {string} The process command name (`/proc/<pid>/comm`), or "" when unavailable.
@@ -106,7 +157,7 @@ function processGroupId(pid) {
106
157
  /**
107
158
  * @param {string} pid - Process id.
108
159
  * @param {string} procPath - Procfs root.
109
- * @returns {{pgrp: number, state: string} | undefined} Parsed process state and group.
160
+ * @returns {{cpuMs: number | undefined, pgrp: number, state: string} | undefined} Parsed process state, group, and cumulative CPU.
110
161
  */
111
162
  function processStat(pid, procPath) {
112
163
  let stat
@@ -126,7 +177,16 @@ function processStat(pid, procPath) {
126
177
  const fields = stat.slice(commEnd + 2).split(" ")
127
178
  const pgrp = Number(fields[2])
128
179
 
129
- return Number.isInteger(pgrp) ? {pgrp, state: fields[0]} : undefined
180
+ if (!Number.isInteger(pgrp)) return undefined
181
+
182
+ // utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
183
+ const utimeTicks = Number(fields[11])
184
+ const stimeTicks = Number(fields[12])
185
+ const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
186
+ ? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
187
+ : undefined
188
+
189
+ return {cpuMs, pgrp, state: fields[0]}
130
190
  }
131
191
 
132
192
  /**
@@ -2,19 +2,27 @@
2
2
 
3
3
  import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
+ import {classifyDrainProgress} from "./drain-policy.js"
5
6
  import ManagedProcess from "./managed-process.js"
6
7
  import {findAvailablePort} from "./port-allocator.js"
7
8
  import {processStartOrder} from "./process-order.js"
9
+ import {sampleProcessGroup} from "./process-memory.js"
8
10
  import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
9
11
  import {waitForHealth, waitForListeningPort} from "./health.js"
10
12
 
11
13
  const DEPENDENCY_READY_INTERVAL_MS = 250
14
+ // Shortest gap between two liveness observations that can prove a draining group's CPU is
15
+ // flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
16
+ // with live runner children doing work, so flatness is only evidence below this window.
17
+ const DRAIN_SAMPLE_WINDOW_MS = 20000
12
18
 
13
19
  /**
14
20
  * @typedef {import("./json.js").JsonValue} JsonValue
15
21
  * @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
22
+ * @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
23
+ * @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
16
24
  * @typedef {{http: number, websocket: number}} ReleaseConnections
17
- * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
25
+ * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
18
26
  * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
19
27
  * @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
20
28
  */
@@ -89,9 +97,10 @@ export default class ReleaseGroup extends EventEmitter {
89
97
  * @param {string | undefined} args.revision - Revision.
90
98
  * @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
91
99
  * @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
100
+ * @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
92
101
  * @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
93
102
  */
94
- constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
103
+ constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
95
104
  super()
96
105
 
97
106
  this.config = config
@@ -113,8 +122,11 @@ export default class ReleaseGroup extends EventEmitter {
113
122
  this.servicePorts = servicePorts
114
123
  this.shouldStart = shouldStart
115
124
  this.processFactory = processFactory
125
+ this.procPath = procPath
116
126
  this.portsAllocated = false
117
127
  this.drainStartedAt = /** @type {string | undefined} */ (undefined)
128
+ this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
129
+ this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
118
130
  this.activatedAt = /** @type {string | undefined} */ (undefined)
119
131
  this.stoppedAt = /** @type {string | undefined} */ (undefined)
120
132
  this.retirementError = /** @type {string | undefined} */ (undefined)
@@ -437,6 +449,7 @@ export default class ReleaseGroup extends EventEmitter {
437
449
  activate() {
438
450
  this.state = "active"
439
451
  this.activatedAt = new Date().toISOString()
452
+ this.clearDrainLivenessSamples()
440
453
  }
441
454
 
442
455
  /** Runs the configured release-generation activation acknowledgement, if any. */
@@ -491,6 +504,7 @@ export default class ReleaseGroup extends EventEmitter {
491
504
  this.drainStartedAt = undefined
492
505
  this.retirementError = undefined
493
506
  this.stoppedAt = undefined
507
+ this.clearDrainLivenessSamples()
494
508
  }
495
509
 
496
510
  /**
@@ -843,7 +857,10 @@ export default class ReleaseGroup extends EventEmitter {
843
857
  // their lifecycle drain runs as soon as the release is retired — in parallel with the
844
858
  // connection drain, not held until after it. The rest stop once connections have closed.
845
859
  const entries = [...this.processes.entries()]
846
- const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([, processInstance]) => processInstance.stop())
860
+ const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
861
+ id,
862
+ promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
863
+ }))
847
864
  const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
848
865
  const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
849
866
 
@@ -864,17 +881,29 @@ export default class ReleaseGroup extends EventEmitter {
864
881
  }
865
882
 
866
883
  if (this.ownerHandoffPaused) {
867
- void Promise.allSettled(nonBlockingStops)
884
+ void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
868
885
  return
869
886
  }
870
887
 
871
888
  await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop()))
872
- await Promise.allSettled(nonBlockingStops)
889
+ const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
890
+ nonBlockingResults.forEach((result, index) => {
891
+ if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
892
+ const processInstance = this.processes.get(nonBlockingStops[index].id)
893
+ this.logger("release non-blocking process still draining", {
894
+ pid: processInstance?.status().pid ?? null,
895
+ processId: nonBlockingStops[index].id,
896
+ releaseId: this.releaseId,
897
+ timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
898
+ })
899
+ }
900
+ })
873
901
  await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
874
902
  this.state = "stopped"
875
903
  this.stoppedAt = new Date().toISOString()
876
904
  this.releasePortReservations()
877
905
  this.abandonLocalConnections()
906
+ this.clearDrainLivenessSamples()
878
907
  }
879
908
 
880
909
  /**
@@ -918,6 +947,7 @@ export default class ReleaseGroup extends EventEmitter {
918
947
  this.stoppedAt = new Date().toISOString()
919
948
  this.releasePortReservations()
920
949
  this.abandonLocalConnections()
950
+ this.clearDrainLivenessSamples()
921
951
  }
922
952
 
923
953
  /** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
@@ -925,12 +955,87 @@ export default class ReleaseGroup extends EventEmitter {
925
955
  await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
926
956
  }
927
957
 
958
+ /**
959
+ * Records one liveness observation for a draining generation and classifies its drain.
960
+ *
961
+ * Two observations are needed to prove CPU flatness, and reading a process group means
962
+ * scanning procfs, so sampling is bounded three ways: a generation that still owns live
963
+ * connections is classified without touching procfs at all (connections alone settle it), a
964
+ * new observation is taken only once the newest one is already a full window behind the
965
+ * present read, and every other read reuses the newest observation. A group that is not
966
+ * draining reports no drain.
967
+ * @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
968
+ */
969
+ drainStatus() {
970
+ if (this.state !== "draining") return undefined
971
+
972
+ const nowMs = Date.now()
973
+
974
+ if (this.connectionCount > 0) {
975
+ return classifyDrainProgress({
976
+ connectionCount: this.connectionCount,
977
+ currentSample: this.drainNewestSample ?? {atMs: nowMs, processes: []},
978
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
979
+ previousSample: undefined
980
+ })
981
+ }
982
+
983
+ const newest = this.drainNewestSample
984
+ const currentSample = newest && nowMs - newest.atMs < DRAIN_SAMPLE_WINDOW_MS ? newest : this.takeLivenessSample(nowMs)
985
+ const previousSample = currentSample === newest ? this.drainPriorSample : newest
986
+
987
+ if (currentSample !== newest) {
988
+ this.drainPriorSample = newest
989
+ this.drainNewestSample = currentSample
990
+ }
991
+
992
+ return classifyDrainProgress({
993
+ connectionCount: this.connectionCount,
994
+ currentSample,
995
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
996
+ previousSample
997
+ })
998
+ }
999
+
1000
+ /**
1001
+ * @param {number} nowMs - Observation time.
1002
+ * @returns {DrainLivenessSample} One liveness observation of every process in the generation.
1003
+ */
1004
+ takeLivenessSample(nowMs) {
1005
+ const processes = [...this.processes.entries()].map(([id, processInstance]) => {
1006
+ const processStatus = processInstance.status()
1007
+ const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
1008
+ // A group that outlived its tracked process is still observed by group id: it holds the
1009
+ // generation's work even though no rollbridge record can signal it any more.
1010
+ const pgid = processStatus.pid ?? heldProcessGroupPgid
1011
+ const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
1012
+
1013
+ return {
1014
+ cpuMs: sample?.cpuMs,
1015
+ heldProcessGroupPgid,
1016
+ id,
1017
+ liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
1018
+ pid: processStatus.pid,
1019
+ retirementTimedOut: processInstance.retirementTimedOut === true
1020
+ }
1021
+ })
1022
+
1023
+ return {atMs: nowMs, processes}
1024
+ }
1025
+
1026
+ /** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
1027
+ clearDrainLivenessSamples() {
1028
+ this.drainNewestSample = undefined
1029
+ this.drainPriorSample = undefined
1030
+ }
1031
+
928
1032
  /** @returns {ReleaseStatus} Status payload. */
929
1033
  status() {
930
1034
  return {
931
1035
  activatedAt: this.activatedAt,
932
1036
  connectionCount: this.connectionCount,
933
1037
  connections: {...this.connections},
1038
+ drain: this.drainStatus(),
934
1039
  drainStartedAt: this.drainStartedAt,
935
1040
  ports: {...this.ports},
936
1041
  processes: [...this.processes.values()].map((processInstance) => processInstance.status()),
@@ -705,17 +705,24 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
705
705
  const defaulted = validateRetention(undefined)
706
706
 
707
707
  expect(defaulted.issues).toEqual([])
708
- expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0})
708
+ expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
709
709
 
710
- const custom = validateRetention({keep: 3, maxAgeMs: 60000})
710
+ const custom = validateRetention({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
711
711
 
712
712
  expect(custom.issues).toEqual([])
713
- expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000})
713
+ expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
714
714
 
715
- const invalid = validateRetention({keep: -1, maxAgeMs: -5})
715
+ // 0 is a valid bound: it disables the pre-activation draining-generation check.
716
+ const unbounded = validateRetention({maxConcurrentDrainingReleaseGroups: 0})
717
+
718
+ expect(unbounded.issues).toEqual([])
719
+ expect(unbounded.config.releaseRetention.maxConcurrentDrainingReleaseGroups).toBe(0)
720
+
721
+ const invalid = validateRetention({keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
716
722
 
717
723
  expect(invalid.issues.some((issue) => issue.message === "releaseRetention.keep must be a non-negative integer")).toBeTruthy()
718
724
  expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxAgeMs must be a non-negative number")).toBeTruthy()
725
+ expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxConcurrentDrainingReleaseGroups must be a non-negative integer")).toBeTruthy()
719
726
  })
720
727
 
721
728
  test("validateConfig leaves statePath unset by default, accepts a string, and rejects non-strings", () => {