rollbridge 0.1.63 → 0.1.66

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,10 +2,13 @@
2
2
 
3
3
  import {EventEmitter} from "node:events"
4
4
  import {spawn} from "node:child_process"
5
- import {processGroupHasLiveMembers, processGroupMembers} from "./process-memory.js"
5
+ import {processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "./process-memory.js"
6
6
 
7
7
  const DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS = 30000
8
8
  const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
9
+ // Reading a process group means scanning procfs, and status() is read continuously. A held
10
+ // group is re-verified on this interval instead of on every read.
11
+ const HELD_GROUP_RECHECK_MS = 5000
9
12
 
10
13
  /**
11
14
  * @typedef {import("./json.js").JsonValue} JsonValue
@@ -16,7 +19,8 @@ const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
16
19
  * @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
17
20
  * @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
18
21
  * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
19
- * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
22
+ * @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
23
+ * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
20
24
  */
21
25
 
22
26
  export default class ManagedProcess extends EventEmitter {
@@ -74,6 +78,9 @@ export default class ManagedProcess extends EventEmitter {
74
78
  this.quiesceError = /** @type {Error | undefined} */ (undefined)
75
79
  this.stopPromise = /** @type {Promise<void> | undefined} */ (undefined)
76
80
  this.retirementTimedOut = false
81
+ this.groupOutlivedStop = /** @type {number | undefined} */ (undefined)
82
+ this.heldProcessGroupCheckedAtMs = /** @type {number | undefined} */ (undefined)
83
+ this.heldProcessGroupCount = /** @type {number | undefined} */ (undefined)
77
84
  this.operationRevision = 0
78
85
  this.restartTimer = undefined
79
86
  this.child = undefined
@@ -102,6 +109,9 @@ export default class ManagedProcess extends EventEmitter {
102
109
  this.quiesceError = undefined
103
110
  this.stopPromise = undefined
104
111
  this.retirementTimedOut = false
112
+ this.groupOutlivedStop = undefined
113
+ this.heldProcessGroupCheckedAtMs = undefined
114
+ this.heldProcessGroupCount = undefined
105
115
  this.exitCode = undefined
106
116
  this.exitSignal = undefined
107
117
  this.state = "starting"
@@ -252,6 +262,7 @@ export default class ManagedProcess extends EventEmitter {
252
262
  this.children = []
253
263
  this.clearMemoryMonitor()
254
264
  this.state = wasIntentional ? "stopped" : "failed"
265
+
255
266
  this.logger("process exited", {code, id: this.id, signal})
256
267
  this.emit("exit", {code, signal})
257
268
 
@@ -260,6 +271,69 @@ export default class ManagedProcess extends EventEmitter {
260
271
  }
261
272
  }
262
273
 
274
+ /**
275
+ * Records that the owned process group was still alive when the bounded stop gave up.
276
+ *
277
+ * Every process is spawned `detached: true`, so the tracked pid is the group leader and a
278
+ * member that survives its leader — a `sh` wrapper killed by the stop signal while the app
279
+ * it launched keeps running, or a runner child that outlives its parent — keeps holding the
280
+ * group id while the tracked ChildProcess record is already terminal. Nothing in rollbridge
281
+ * can wait for or signal that group again, yet the group still holds the release's work.
282
+ *
283
+ * Only the stop path can know this, and it already observed the group while waiting for it to
284
+ * leave, so the group id is recorded there instead of rediscovered in `onExit`: that hook
285
+ * fires for every retired process, and a procfs scan costs ~200 ms per exit on a busy host.
286
+ * @param {number} pgid - Process group id the tracked process led.
287
+ * @returns {void}
288
+ */
289
+ markGroupOutlivedStop(pgid) {
290
+ if (this.groupOutlivedStop === pgid) return
291
+
292
+ this.groupOutlivedStop = pgid
293
+ this.heldProcessGroupCheckedAtMs = undefined
294
+ this.heldProcessGroupCount = undefined
295
+ this.logger("process group outlived its tracked process", {id: this.id, pgid})
296
+ }
297
+
298
+ /**
299
+ * Reports the process group left behind by a bounded stop, as observed right now.
300
+ *
301
+ * The record is re-verified rather than trusted forever: once the orphaned members exit, the
302
+ * group is gone and stops holding the key. Re-verification is rate-limited to
303
+ * `HELD_GROUP_RECHECK_MS` because it costs a procfs scan, and when procfs cannot answer at
304
+ * all the last known state is reported instead of guessing that the group is gone.
305
+ * @returns {HeldProcessGroup | undefined} The held group, or undefined when nothing is held.
306
+ */
307
+ currentHeldProcessGroup() {
308
+ const pgid = this.groupOutlivedStop
309
+
310
+ if (pgid === undefined) return undefined
311
+ if (this.state !== "stopped" && this.state !== "failed") return undefined
312
+
313
+ const nowMs = Date.now()
314
+
315
+ if (this.heldProcessGroupCount !== undefined && this.heldProcessGroupCheckedAtMs !== undefined && nowMs - this.heldProcessGroupCheckedAtMs < HELD_GROUP_RECHECK_MS) {
316
+ return {liveMemberCount: this.heldProcessGroupCount, pgid}
317
+ }
318
+
319
+ const sample = sampleProcessGroup(pgid)
320
+
321
+ this.heldProcessGroupCheckedAtMs = nowMs
322
+
323
+ if (sample === undefined) return {liveMemberCount: this.heldProcessGroupCount, pgid}
324
+
325
+ if (sample.memberCount === 0) {
326
+ this.groupOutlivedStop = undefined
327
+ this.heldProcessGroupCount = undefined
328
+
329
+ return undefined
330
+ }
331
+
332
+ this.heldProcessGroupCount = sample.liveMemberCount
333
+
334
+ return {liveMemberCount: sample.liveMemberCount, pgid}
335
+ }
336
+
263
337
  /**
264
338
  * Schedules an automatic restart per the restart policy, or gives up once the policy's limit is reached.
265
339
  * @returns {void}
@@ -466,6 +540,7 @@ export default class ManagedProcess extends EventEmitter {
466
540
 
467
541
  if (!exited && retirementDeadline !== undefined && gracefulDeadline === undefined) {
468
542
  this.retirementTimedOut = true
543
+ this.markGroupOutlivedStop(pgid)
469
544
  this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
470
545
  return
471
546
  }
@@ -475,7 +550,8 @@ export default class ManagedProcess extends EventEmitter {
475
550
  const killDeadline = Date.now() + 5000
476
551
 
477
552
  await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
478
- await this.waitForProcessGroupExit(pgid, killDeadline)
553
+
554
+ if (!await this.waitForProcessGroupExit(pgid, killDeadline)) this.markGroupOutlivedStop(pgid)
479
555
  }
480
556
  }
481
557
 
@@ -828,6 +904,7 @@ export default class ManagedProcess extends EventEmitter {
828
904
  cwd: this.cwd,
829
905
  exitCode: this.exitCode,
830
906
  exitSignal: this.exitSignal,
907
+ heldProcessGroup: this.currentHeldProcessGroup(),
831
908
  id: this.id,
832
909
  lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
833
910
  lastStartReason: this.lastStartReason,
@@ -836,6 +913,7 @@ export default class ManagedProcess extends EventEmitter {
836
913
  memoryRestarts: this.memoryRestarts,
837
914
  pid: this.pid,
838
915
  restarts: this.restarts,
916
+ retirementTimedOut: this.retirementTimedOut,
839
917
  rssBytes: this.rssBytes,
840
918
  startedAt: this.startedAtMs === undefined ? undefined : new Date(this.startedAtMs).toISOString(),
841
919
  state: this.state,
@@ -843,3 +921,21 @@ export default class ManagedProcess extends EventEmitter {
843
921
  }
844
922
  }
845
923
  }
924
+
925
+ /**
926
+ * The guardian's provenance fence, in one place: a registered child stops fencing a later
927
+ * claim on its key only when its record is terminal *and* it holds no pid, and a process
928
+ * group that outlived the tracked process is not gone either.
929
+ *
930
+ * Both the guardian (re-registering a key under a new provenance) and the daemon (restoring a
931
+ * release that is recorded as drained) must agree on this, or a restore reaches the guardian
932
+ * and fails there with a bare `Guardian provenance mismatch` naming a process the operator
933
+ * cannot see in `status`.
934
+ * @param {Pick<ManagedProcessStatus, "heldProcessGroup" | "pid" | "state">} status - Process status record.
935
+ * @returns {boolean} True when the child is conclusively gone.
936
+ */
937
+ export function childRecordIsGone(status) {
938
+ if (status.heldProcessGroup) return false
939
+
940
+ return (status.state === "stopped" || status.state === "failed") && status.pid === undefined
941
+ }
@@ -8,7 +8,7 @@ import {spawn} from "node:child_process"
8
8
  import {isDeepStrictEqual} from "node:util"
9
9
  import path from "node:path"
10
10
  import GuardianClient from "./guardian-client.js"
11
- import ManagedProcess from "./managed-process.js"
11
+ import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
12
12
  import {writeState} from "./state-store.js"
13
13
 
14
14
  const OWNER_RESTART_RETRY_MS = 1000
@@ -656,10 +656,9 @@ async function execute(request, socket) {
656
656
  // residue of a dead generation, and demanding its provenance wedges every later deploy
657
657
  // against a process that no longer exists (TensorBuzz production, STC-695). The owner
658
658
  // may re-register the key under the new definition; live processes stay fail-closed.
659
- const existingStatus = existing.process.status()
660
- const childIsGone = (existingStatus.state === "stopped" || existingStatus.state === "failed") && existingStatus.pid === undefined
661
-
662
- if (!childIsGone) throw new Error(`Guardian provenance mismatch for ${request.key}`)
659
+ // A group that outlived its tracked process is not gone: it still holds the key's work,
660
+ // so re-registering it would start a second copy of a generation that is still running.
661
+ if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
663
662
  requireOwner(socket, request.command)
664
663
  processes.delete(request.key)
665
664
  }
@@ -4,8 +4,14 @@ import fs from "node:fs"
4
4
 
5
5
  /**
6
6
  * @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
7
+ * @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
7
8
  */
8
9
 
10
+ // Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
11
+ // userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
12
+ // not a rate: two reads of it measure whether a process did any work between them.
13
+ const CLOCK_TICKS_PER_MS = 10
14
+
9
15
  /**
10
16
  * Lists the members of a managed process group with each member's resident memory.
11
17
  * Rollbridge spawns each process detached, so the spawned pid is the process-group
@@ -83,6 +89,51 @@ export function measureProcessGroupRssBytes(pgid) {
83
89
  return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
84
90
  }
85
91
 
92
+ /**
93
+ * Samples one managed process group for drain liveness: how much CPU its members have
94
+ * consumed in total, and how many of them can still run.
95
+ *
96
+ * CPU is the sum of every member's cumulative user+system time, defunct members included:
97
+ * a member that exits takes its consumed time out of the sum, so the difference between two
98
+ * samples is non-zero whenever the group changed at all — new work or a departing member.
99
+ * Only an exactly unchanged difference says the group did nothing between the two reads.
100
+ * @param {number} pgid - Process-group id (the detached spawn's pid).
101
+ * @param {string} [procPath] - Procfs root (overridable for deterministic tests).
102
+ * @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
103
+ * read at all. A group with no members left is a sample of zero, not a missing measurement.
104
+ */
105
+ export function sampleProcessGroup(pgid, procPath = "/proc") {
106
+ /** @type {string[]} */
107
+ let entries
108
+
109
+ try {
110
+ entries = fs.readdirSync(procPath)
111
+ } catch {
112
+ return undefined
113
+ }
114
+
115
+ let cpuMs = 0
116
+ let liveMemberCount = 0
117
+ let memberCount = 0
118
+ let measurable = true
119
+
120
+ for (const entry of entries) {
121
+ if (!/^\d+$/.test(entry)) continue
122
+
123
+ const stat = processStat(entry, procPath)
124
+
125
+ if (stat?.pgrp !== pgid) continue
126
+
127
+ memberCount += 1
128
+ if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
129
+
130
+ if (stat.cpuMs === undefined) measurable = false
131
+ else cpuMs += stat.cpuMs
132
+ }
133
+
134
+ return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
135
+ }
136
+
86
137
  /**
87
138
  * @param {string} pid - Process id.
88
139
  * @returns {string} The process command name (`/proc/<pid>/comm`), or "" when unavailable.
@@ -106,7 +157,7 @@ function processGroupId(pid) {
106
157
  /**
107
158
  * @param {string} pid - Process id.
108
159
  * @param {string} procPath - Procfs root.
109
- * @returns {{pgrp: number, state: string} | undefined} Parsed process state and group.
160
+ * @returns {{cpuMs: number | undefined, pgrp: number, state: string} | undefined} Parsed process state, group, and cumulative CPU.
110
161
  */
111
162
  function processStat(pid, procPath) {
112
163
  let stat
@@ -126,7 +177,16 @@ function processStat(pid, procPath) {
126
177
  const fields = stat.slice(commEnd + 2).split(" ")
127
178
  const pgrp = Number(fields[2])
128
179
 
129
- return Number.isInteger(pgrp) ? {pgrp, state: fields[0]} : undefined
180
+ if (!Number.isInteger(pgrp)) return undefined
181
+
182
+ // utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
183
+ const utimeTicks = Number(fields[11])
184
+ const stimeTicks = Number(fields[12])
185
+ const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
186
+ ? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
187
+ : undefined
188
+
189
+ return {cpuMs, pgrp, state: fields[0]}
130
190
  }
131
191
 
132
192
  /**
@@ -2,19 +2,27 @@
2
2
 
3
3
  import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
+ import {classifyDrainProgress} from "./drain-policy.js"
5
6
  import ManagedProcess from "./managed-process.js"
6
7
  import {findAvailablePort} from "./port-allocator.js"
7
8
  import {processStartOrder} from "./process-order.js"
9
+ import {sampleProcessGroup} from "./process-memory.js"
8
10
  import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
9
11
  import {waitForHealth, waitForListeningPort} from "./health.js"
10
12
 
11
13
  const DEPENDENCY_READY_INTERVAL_MS = 250
14
+ // Shortest gap between two liveness observations that can prove a draining group's CPU is
15
+ // flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
16
+ // with live runner children doing work, so flatness is only evidence below this window.
17
+ const DRAIN_SAMPLE_WINDOW_MS = 20000
12
18
 
13
19
  /**
14
20
  * @typedef {import("./json.js").JsonValue} JsonValue
15
21
  * @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
22
+ * @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
23
+ * @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
16
24
  * @typedef {{http: number, websocket: number}} ReleaseConnections
17
- * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
25
+ * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
18
26
  * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
19
27
  * @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
20
28
  */
@@ -89,9 +97,10 @@ export default class ReleaseGroup extends EventEmitter {
89
97
  * @param {string | undefined} args.revision - Revision.
90
98
  * @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
91
99
  * @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
100
+ * @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
92
101
  * @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
93
102
  */
94
- constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
103
+ constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
95
104
  super()
96
105
 
97
106
  this.config = config
@@ -113,8 +122,11 @@ export default class ReleaseGroup extends EventEmitter {
113
122
  this.servicePorts = servicePorts
114
123
  this.shouldStart = shouldStart
115
124
  this.processFactory = processFactory
125
+ this.procPath = procPath
116
126
  this.portsAllocated = false
117
127
  this.drainStartedAt = /** @type {string | undefined} */ (undefined)
128
+ this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
129
+ this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
118
130
  this.activatedAt = /** @type {string | undefined} */ (undefined)
119
131
  this.stoppedAt = /** @type {string | undefined} */ (undefined)
120
132
  this.retirementError = /** @type {string | undefined} */ (undefined)
@@ -437,6 +449,7 @@ export default class ReleaseGroup extends EventEmitter {
437
449
  activate() {
438
450
  this.state = "active"
439
451
  this.activatedAt = new Date().toISOString()
452
+ this.clearDrainLivenessSamples()
440
453
  }
441
454
 
442
455
  /** Runs the configured release-generation activation acknowledgement, if any. */
@@ -491,6 +504,7 @@ export default class ReleaseGroup extends EventEmitter {
491
504
  this.drainStartedAt = undefined
492
505
  this.retirementError = undefined
493
506
  this.stoppedAt = undefined
507
+ this.clearDrainLivenessSamples()
494
508
  }
495
509
 
496
510
  /**
@@ -889,6 +903,7 @@ export default class ReleaseGroup extends EventEmitter {
889
903
  this.stoppedAt = new Date().toISOString()
890
904
  this.releasePortReservations()
891
905
  this.abandonLocalConnections()
906
+ this.clearDrainLivenessSamples()
892
907
  }
893
908
 
894
909
  /**
@@ -932,6 +947,7 @@ export default class ReleaseGroup extends EventEmitter {
932
947
  this.stoppedAt = new Date().toISOString()
933
948
  this.releasePortReservations()
934
949
  this.abandonLocalConnections()
950
+ this.clearDrainLivenessSamples()
935
951
  }
936
952
 
937
953
  /** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
@@ -939,12 +955,87 @@ export default class ReleaseGroup extends EventEmitter {
939
955
  await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
940
956
  }
941
957
 
958
+ /**
959
+ * Records one liveness observation for a draining generation and classifies its drain.
960
+ *
961
+ * Two observations are needed to prove CPU flatness, and reading a process group means
962
+ * scanning procfs, so sampling is bounded three ways: a generation that still owns live
963
+ * connections is classified without touching procfs at all (connections alone settle it), a
964
+ * new observation is taken only once the newest one is already a full window behind the
965
+ * present read, and every other read reuses the newest observation. A group that is not
966
+ * draining reports no drain.
967
+ * @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
968
+ */
969
+ drainStatus() {
970
+ if (this.state !== "draining") return undefined
971
+
972
+ const nowMs = Date.now()
973
+
974
+ if (this.connectionCount > 0) {
975
+ return classifyDrainProgress({
976
+ connectionCount: this.connectionCount,
977
+ currentSample: this.drainNewestSample ?? {atMs: nowMs, processes: []},
978
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
979
+ previousSample: undefined
980
+ })
981
+ }
982
+
983
+ const newest = this.drainNewestSample
984
+ const currentSample = newest && nowMs - newest.atMs < DRAIN_SAMPLE_WINDOW_MS ? newest : this.takeLivenessSample(nowMs)
985
+ const previousSample = currentSample === newest ? this.drainPriorSample : newest
986
+
987
+ if (currentSample !== newest) {
988
+ this.drainPriorSample = newest
989
+ this.drainNewestSample = currentSample
990
+ }
991
+
992
+ return classifyDrainProgress({
993
+ connectionCount: this.connectionCount,
994
+ currentSample,
995
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
996
+ previousSample
997
+ })
998
+ }
999
+
1000
+ /**
1001
+ * @param {number} nowMs - Observation time.
1002
+ * @returns {DrainLivenessSample} One liveness observation of every process in the generation.
1003
+ */
1004
+ takeLivenessSample(nowMs) {
1005
+ const processes = [...this.processes.entries()].map(([id, processInstance]) => {
1006
+ const processStatus = processInstance.status()
1007
+ const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
1008
+ // A group that outlived its tracked process is still observed by group id: it holds the
1009
+ // generation's work even though no rollbridge record can signal it any more.
1010
+ const pgid = processStatus.pid ?? heldProcessGroupPgid
1011
+ const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
1012
+
1013
+ return {
1014
+ cpuMs: sample?.cpuMs,
1015
+ heldProcessGroupPgid,
1016
+ id,
1017
+ liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
1018
+ pid: processStatus.pid,
1019
+ retirementTimedOut: processInstance.retirementTimedOut === true
1020
+ }
1021
+ })
1022
+
1023
+ return {atMs: nowMs, processes}
1024
+ }
1025
+
1026
+ /** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
1027
+ clearDrainLivenessSamples() {
1028
+ this.drainNewestSample = undefined
1029
+ this.drainPriorSample = undefined
1030
+ }
1031
+
942
1032
  /** @returns {ReleaseStatus} Status payload. */
943
1033
  status() {
944
1034
  return {
945
1035
  activatedAt: this.activatedAt,
946
1036
  connectionCount: this.connectionCount,
947
1037
  connections: {...this.connections},
1038
+ drain: this.drainStatus(),
948
1039
  drainStartedAt: this.drainStartedAt,
949
1040
  ports: {...this.ports},
950
1041
  processes: [...this.processes.values()].map((processInstance) => processInstance.status()),
@@ -705,17 +705,24 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
705
705
  const defaulted = validateRetention(undefined)
706
706
 
707
707
  expect(defaulted.issues).toEqual([])
708
- expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0})
708
+ expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
709
709
 
710
- const custom = validateRetention({keep: 3, maxAgeMs: 60000})
710
+ const custom = validateRetention({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
711
711
 
712
712
  expect(custom.issues).toEqual([])
713
- expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000})
713
+ expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
714
714
 
715
- const invalid = validateRetention({keep: -1, maxAgeMs: -5})
715
+ // 0 is a valid bound: it disables the pre-activation draining-generation check.
716
+ const unbounded = validateRetention({maxConcurrentDrainingReleaseGroups: 0})
717
+
718
+ expect(unbounded.issues).toEqual([])
719
+ expect(unbounded.config.releaseRetention.maxConcurrentDrainingReleaseGroups).toBe(0)
720
+
721
+ const invalid = validateRetention({keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
716
722
 
717
723
  expect(invalid.issues.some((issue) => issue.message === "releaseRetention.keep must be a non-negative integer")).toBeTruthy()
718
724
  expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxAgeMs must be a non-negative number")).toBeTruthy()
725
+ expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxConcurrentDrainingReleaseGroups must be a non-negative integer")).toBeTruthy()
719
726
  })
720
727
 
721
728
  test("validateConfig leaves statePath unset by default, accepts a string, and rejects non-strings", () => {