rollbridge 0.1.63 → 0.1.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/docs/cli.md +44 -0
- package/docs/config.md +26 -1
- package/package.json +1 -1
- package/src/config.js +13 -2
- package/src/daemon.js +83 -1
- package/src/drain-policy.js +159 -0
- package/src/managed-process.js +99 -3
- package/src/process-guardian.js +4 -5
- package/src/process-memory.js +62 -2
- package/src/release-group.js +93 -2
- package/test/config-validation.test.js +11 -4
- package/test/drain-policy.test.js +199 -0
- package/test/managed-process.test.js +11 -1
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +127 -2
- package/test/rollbridge.test.js +120 -0
package/src/managed-process.js
CHANGED
|
@@ -2,10 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
import {EventEmitter} from "node:events"
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
|
-
import {processGroupHasLiveMembers, processGroupMembers} from "./process-memory.js"
|
|
5
|
+
import {processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "./process-memory.js"
|
|
6
6
|
|
|
7
7
|
const DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS = 30000
|
|
8
8
|
const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
|
|
9
|
+
// Reading a process group means scanning procfs, and status() is read continuously. A held
|
|
10
|
+
// group is re-verified on this interval instead of on every read.
|
|
11
|
+
const HELD_GROUP_RECHECK_MS = 5000
|
|
9
12
|
|
|
10
13
|
/**
|
|
11
14
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -16,7 +19,8 @@ const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
|
|
|
16
19
|
* @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
|
|
17
20
|
* @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
|
|
18
21
|
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
19
|
-
* @typedef {{
|
|
22
|
+
* @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
|
|
23
|
+
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
20
24
|
*/
|
|
21
25
|
|
|
22
26
|
export default class ManagedProcess extends EventEmitter {
|
|
@@ -74,6 +78,9 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
74
78
|
this.quiesceError = /** @type {Error | undefined} */ (undefined)
|
|
75
79
|
this.stopPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
76
80
|
this.retirementTimedOut = false
|
|
81
|
+
this.groupOutlivedStop = /** @type {number | undefined} */ (undefined)
|
|
82
|
+
this.heldProcessGroupCheckedAtMs = /** @type {number | undefined} */ (undefined)
|
|
83
|
+
this.heldProcessGroupCount = /** @type {number | undefined} */ (undefined)
|
|
77
84
|
this.operationRevision = 0
|
|
78
85
|
this.restartTimer = undefined
|
|
79
86
|
this.child = undefined
|
|
@@ -102,6 +109,9 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
102
109
|
this.quiesceError = undefined
|
|
103
110
|
this.stopPromise = undefined
|
|
104
111
|
this.retirementTimedOut = false
|
|
112
|
+
this.groupOutlivedStop = undefined
|
|
113
|
+
this.heldProcessGroupCheckedAtMs = undefined
|
|
114
|
+
this.heldProcessGroupCount = undefined
|
|
105
115
|
this.exitCode = undefined
|
|
106
116
|
this.exitSignal = undefined
|
|
107
117
|
this.state = "starting"
|
|
@@ -252,6 +262,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
252
262
|
this.children = []
|
|
253
263
|
this.clearMemoryMonitor()
|
|
254
264
|
this.state = wasIntentional ? "stopped" : "failed"
|
|
265
|
+
|
|
255
266
|
this.logger("process exited", {code, id: this.id, signal})
|
|
256
267
|
this.emit("exit", {code, signal})
|
|
257
268
|
|
|
@@ -260,6 +271,69 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
260
271
|
}
|
|
261
272
|
}
|
|
262
273
|
|
|
274
|
+
/**
|
|
275
|
+
* Records that the owned process group was still alive when the bounded stop gave up.
|
|
276
|
+
*
|
|
277
|
+
* Every process is spawned `detached: true`, so the tracked pid is the group leader and a
|
|
278
|
+
* member that survives its leader — a `sh` wrapper killed by the stop signal while the app
|
|
279
|
+
* it launched keeps running, or a runner child that outlives its parent — keeps holding the
|
|
280
|
+
* group id while the tracked ChildProcess record is already terminal. Nothing in rollbridge
|
|
281
|
+
* can wait for or signal that group again, yet the group still holds the release's work.
|
|
282
|
+
*
|
|
283
|
+
* Only the stop path can know this, and it already observed the group while waiting for it to
|
|
284
|
+
* leave, so the group id is recorded there instead of rediscovered in `onExit`: that hook
|
|
285
|
+
* fires for every retired process, and a procfs scan costs ~200 ms per exit on a busy host.
|
|
286
|
+
* @param {number} pgid - Process group id the tracked process led.
|
|
287
|
+
* @returns {void}
|
|
288
|
+
*/
|
|
289
|
+
markGroupOutlivedStop(pgid) {
|
|
290
|
+
if (this.groupOutlivedStop === pgid) return
|
|
291
|
+
|
|
292
|
+
this.groupOutlivedStop = pgid
|
|
293
|
+
this.heldProcessGroupCheckedAtMs = undefined
|
|
294
|
+
this.heldProcessGroupCount = undefined
|
|
295
|
+
this.logger("process group outlived its tracked process", {id: this.id, pgid})
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* Reports the process group left behind by a bounded stop, as observed right now.
|
|
300
|
+
*
|
|
301
|
+
* The record is re-verified rather than trusted forever: once the orphaned members exit, the
|
|
302
|
+
* group is gone and stops holding the key. Re-verification is rate-limited to
|
|
303
|
+
* `HELD_GROUP_RECHECK_MS` because it costs a procfs scan, and when procfs cannot answer at
|
|
304
|
+
* all the last known state is reported instead of guessing that the group is gone.
|
|
305
|
+
* @returns {HeldProcessGroup | undefined} The held group, or undefined when nothing is held.
|
|
306
|
+
*/
|
|
307
|
+
currentHeldProcessGroup() {
|
|
308
|
+
const pgid = this.groupOutlivedStop
|
|
309
|
+
|
|
310
|
+
if (pgid === undefined) return undefined
|
|
311
|
+
if (this.state !== "stopped" && this.state !== "failed") return undefined
|
|
312
|
+
|
|
313
|
+
const nowMs = Date.now()
|
|
314
|
+
|
|
315
|
+
if (this.heldProcessGroupCount !== undefined && this.heldProcessGroupCheckedAtMs !== undefined && nowMs - this.heldProcessGroupCheckedAtMs < HELD_GROUP_RECHECK_MS) {
|
|
316
|
+
return {liveMemberCount: this.heldProcessGroupCount, pgid}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
const sample = sampleProcessGroup(pgid)
|
|
320
|
+
|
|
321
|
+
this.heldProcessGroupCheckedAtMs = nowMs
|
|
322
|
+
|
|
323
|
+
if (sample === undefined) return {liveMemberCount: this.heldProcessGroupCount, pgid}
|
|
324
|
+
|
|
325
|
+
if (sample.memberCount === 0) {
|
|
326
|
+
this.groupOutlivedStop = undefined
|
|
327
|
+
this.heldProcessGroupCount = undefined
|
|
328
|
+
|
|
329
|
+
return undefined
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
this.heldProcessGroupCount = sample.liveMemberCount
|
|
333
|
+
|
|
334
|
+
return {liveMemberCount: sample.liveMemberCount, pgid}
|
|
335
|
+
}
|
|
336
|
+
|
|
263
337
|
/**
|
|
264
338
|
* Schedules an automatic restart per the restart policy, or gives up once the policy's limit is reached.
|
|
265
339
|
* @returns {void}
|
|
@@ -466,6 +540,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
466
540
|
|
|
467
541
|
if (!exited && retirementDeadline !== undefined && gracefulDeadline === undefined) {
|
|
468
542
|
this.retirementTimedOut = true
|
|
543
|
+
this.markGroupOutlivedStop(pgid)
|
|
469
544
|
this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
|
|
470
545
|
return
|
|
471
546
|
}
|
|
@@ -475,7 +550,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
475
550
|
const killDeadline = Date.now() + 5000
|
|
476
551
|
|
|
477
552
|
await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
|
|
478
|
-
|
|
553
|
+
|
|
554
|
+
if (!await this.waitForProcessGroupExit(pgid, killDeadline)) this.markGroupOutlivedStop(pgid)
|
|
479
555
|
}
|
|
480
556
|
}
|
|
481
557
|
|
|
@@ -828,6 +904,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
828
904
|
cwd: this.cwd,
|
|
829
905
|
exitCode: this.exitCode,
|
|
830
906
|
exitSignal: this.exitSignal,
|
|
907
|
+
heldProcessGroup: this.currentHeldProcessGroup(),
|
|
831
908
|
id: this.id,
|
|
832
909
|
lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
|
|
833
910
|
lastStartReason: this.lastStartReason,
|
|
@@ -836,6 +913,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
836
913
|
memoryRestarts: this.memoryRestarts,
|
|
837
914
|
pid: this.pid,
|
|
838
915
|
restarts: this.restarts,
|
|
916
|
+
retirementTimedOut: this.retirementTimedOut,
|
|
839
917
|
rssBytes: this.rssBytes,
|
|
840
918
|
startedAt: this.startedAtMs === undefined ? undefined : new Date(this.startedAtMs).toISOString(),
|
|
841
919
|
state: this.state,
|
|
@@ -843,3 +921,21 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
843
921
|
}
|
|
844
922
|
}
|
|
845
923
|
}
|
|
924
|
+
|
|
925
|
+
/**
|
|
926
|
+
* The guardian's provenance fence, in one place: a registered child stops fencing a later
|
|
927
|
+
* claim on its key only when its record is terminal *and* it holds no pid, and a process
|
|
928
|
+
* group that outlived the tracked process is not gone either.
|
|
929
|
+
*
|
|
930
|
+
* Both the guardian (re-registering a key under a new provenance) and the daemon (restoring a
|
|
931
|
+
* release that is recorded as drained) must agree on this, or a restore reaches the guardian
|
|
932
|
+
* and fails there with a bare `Guardian provenance mismatch` naming a process the operator
|
|
933
|
+
* cannot see in `status`.
|
|
934
|
+
* @param {Pick<ManagedProcessStatus, "heldProcessGroup" | "pid" | "state">} status - Process status record.
|
|
935
|
+
* @returns {boolean} True when the child is conclusively gone.
|
|
936
|
+
*/
|
|
937
|
+
export function childRecordIsGone(status) {
|
|
938
|
+
if (status.heldProcessGroup) return false
|
|
939
|
+
|
|
940
|
+
return (status.state === "stopped" || status.state === "failed") && status.pid === undefined
|
|
941
|
+
}
|
package/src/process-guardian.js
CHANGED
|
@@ -8,7 +8,7 @@ import {spawn} from "node:child_process"
|
|
|
8
8
|
import {isDeepStrictEqual} from "node:util"
|
|
9
9
|
import path from "node:path"
|
|
10
10
|
import GuardianClient from "./guardian-client.js"
|
|
11
|
-
import ManagedProcess from "./managed-process.js"
|
|
11
|
+
import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
|
|
12
12
|
import {writeState} from "./state-store.js"
|
|
13
13
|
|
|
14
14
|
const OWNER_RESTART_RETRY_MS = 1000
|
|
@@ -656,10 +656,9 @@ async function execute(request, socket) {
|
|
|
656
656
|
// residue of a dead generation, and demanding its provenance wedges every later deploy
|
|
657
657
|
// against a process that no longer exists (TensorBuzz production, STC-695). The owner
|
|
658
658
|
// may re-register the key under the new definition; live processes stay fail-closed.
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
if (!childIsGone) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
659
|
+
// A group that outlived its tracked process is not gone: it still holds the key's work,
|
|
660
|
+
// so re-registering it would start a second copy of a generation that is still running.
|
|
661
|
+
if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
663
662
|
requireOwner(socket, request.command)
|
|
664
663
|
processes.delete(request.key)
|
|
665
664
|
}
|
package/src/process-memory.js
CHANGED
|
@@ -4,8 +4,14 @@ import fs from "node:fs"
|
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
|
|
7
|
+
* @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
|
|
7
8
|
*/
|
|
8
9
|
|
|
10
|
+
// Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
|
|
11
|
+
// userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
|
|
12
|
+
// not a rate: two reads of it measure whether a process did any work between them.
|
|
13
|
+
const CLOCK_TICKS_PER_MS = 10
|
|
14
|
+
|
|
9
15
|
/**
|
|
10
16
|
* Lists the members of a managed process group with each member's resident memory.
|
|
11
17
|
* Rollbridge spawns each process detached, so the spawned pid is the process-group
|
|
@@ -83,6 +89,51 @@ export function measureProcessGroupRssBytes(pgid) {
|
|
|
83
89
|
return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
|
|
84
90
|
}
|
|
85
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Samples one managed process group for drain liveness: how much CPU its members have
|
|
94
|
+
* consumed in total, and how many of them can still run.
|
|
95
|
+
*
|
|
96
|
+
* CPU is the sum of every member's cumulative user+system time, defunct members included:
|
|
97
|
+
* a member that exits takes its consumed time out of the sum, so the difference between two
|
|
98
|
+
* samples is non-zero whenever the group changed at all — new work or a departing member.
|
|
99
|
+
* Only an exactly unchanged difference says the group did nothing between the two reads.
|
|
100
|
+
* @param {number} pgid - Process-group id (the detached spawn's pid).
|
|
101
|
+
* @param {string} [procPath] - Procfs root (overridable for deterministic tests).
|
|
102
|
+
* @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
|
|
103
|
+
* read at all. A group with no members left is a sample of zero, not a missing measurement.
|
|
104
|
+
*/
|
|
105
|
+
export function sampleProcessGroup(pgid, procPath = "/proc") {
|
|
106
|
+
/** @type {string[]} */
|
|
107
|
+
let entries
|
|
108
|
+
|
|
109
|
+
try {
|
|
110
|
+
entries = fs.readdirSync(procPath)
|
|
111
|
+
} catch {
|
|
112
|
+
return undefined
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
let cpuMs = 0
|
|
116
|
+
let liveMemberCount = 0
|
|
117
|
+
let memberCount = 0
|
|
118
|
+
let measurable = true
|
|
119
|
+
|
|
120
|
+
for (const entry of entries) {
|
|
121
|
+
if (!/^\d+$/.test(entry)) continue
|
|
122
|
+
|
|
123
|
+
const stat = processStat(entry, procPath)
|
|
124
|
+
|
|
125
|
+
if (stat?.pgrp !== pgid) continue
|
|
126
|
+
|
|
127
|
+
memberCount += 1
|
|
128
|
+
if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
|
|
129
|
+
|
|
130
|
+
if (stat.cpuMs === undefined) measurable = false
|
|
131
|
+
else cpuMs += stat.cpuMs
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
|
|
135
|
+
}
|
|
136
|
+
|
|
86
137
|
/**
|
|
87
138
|
* @param {string} pid - Process id.
|
|
88
139
|
* @returns {string} The process command name (`/proc/<pid>/comm`), or "" when unavailable.
|
|
@@ -106,7 +157,7 @@ function processGroupId(pid) {
|
|
|
106
157
|
/**
|
|
107
158
|
* @param {string} pid - Process id.
|
|
108
159
|
* @param {string} procPath - Procfs root.
|
|
109
|
-
* @returns {{pgrp: number, state: string} | undefined} Parsed process state and
|
|
160
|
+
* @returns {{cpuMs: number | undefined, pgrp: number, state: string} | undefined} Parsed process state, group, and cumulative CPU.
|
|
110
161
|
*/
|
|
111
162
|
function processStat(pid, procPath) {
|
|
112
163
|
let stat
|
|
@@ -126,7 +177,16 @@ function processStat(pid, procPath) {
|
|
|
126
177
|
const fields = stat.slice(commEnd + 2).split(" ")
|
|
127
178
|
const pgrp = Number(fields[2])
|
|
128
179
|
|
|
129
|
-
|
|
180
|
+
if (!Number.isInteger(pgrp)) return undefined
|
|
181
|
+
|
|
182
|
+
// utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
|
|
183
|
+
const utimeTicks = Number(fields[11])
|
|
184
|
+
const stimeTicks = Number(fields[12])
|
|
185
|
+
const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
|
|
186
|
+
? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
|
|
187
|
+
: undefined
|
|
188
|
+
|
|
189
|
+
return {cpuMs, pgrp, state: fields[0]}
|
|
130
190
|
}
|
|
131
191
|
|
|
132
192
|
/**
|
package/src/release-group.js
CHANGED
|
@@ -2,19 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
import {spawn} from "node:child_process"
|
|
4
4
|
import {EventEmitter} from "node:events"
|
|
5
|
+
import {classifyDrainProgress} from "./drain-policy.js"
|
|
5
6
|
import ManagedProcess from "./managed-process.js"
|
|
6
7
|
import {findAvailablePort} from "./port-allocator.js"
|
|
7
8
|
import {processStartOrder} from "./process-order.js"
|
|
9
|
+
import {sampleProcessGroup} from "./process-memory.js"
|
|
8
10
|
import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
|
|
9
11
|
import {waitForHealth, waitForListeningPort} from "./health.js"
|
|
10
12
|
|
|
11
13
|
const DEPENDENCY_READY_INTERVAL_MS = 250
|
|
14
|
+
// Shortest gap between two liveness observations that can prove a draining group's CPU is
|
|
15
|
+
// flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
|
|
16
|
+
// with live runner children doing work, so flatness is only evidence below this window.
|
|
17
|
+
const DRAIN_SAMPLE_WINDOW_MS = 20000
|
|
12
18
|
|
|
13
19
|
/**
|
|
14
20
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
15
21
|
* @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
|
|
22
|
+
* @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
|
|
23
|
+
* @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
|
|
16
24
|
* @typedef {{http: number, websocket: number}} ReleaseConnections
|
|
17
|
-
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
25
|
+
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
18
26
|
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
|
|
19
27
|
* @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
|
|
20
28
|
*/
|
|
@@ -89,9 +97,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
89
97
|
* @param {string | undefined} args.revision - Revision.
|
|
90
98
|
* @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
|
|
91
99
|
* @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
|
|
100
|
+
* @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
|
|
92
101
|
* @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
|
|
93
102
|
*/
|
|
94
|
-
constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
|
|
103
|
+
constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
|
|
95
104
|
super()
|
|
96
105
|
|
|
97
106
|
this.config = config
|
|
@@ -113,8 +122,11 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
113
122
|
this.servicePorts = servicePorts
|
|
114
123
|
this.shouldStart = shouldStart
|
|
115
124
|
this.processFactory = processFactory
|
|
125
|
+
this.procPath = procPath
|
|
116
126
|
this.portsAllocated = false
|
|
117
127
|
this.drainStartedAt = /** @type {string | undefined} */ (undefined)
|
|
128
|
+
this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
|
|
129
|
+
this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
|
|
118
130
|
this.activatedAt = /** @type {string | undefined} */ (undefined)
|
|
119
131
|
this.stoppedAt = /** @type {string | undefined} */ (undefined)
|
|
120
132
|
this.retirementError = /** @type {string | undefined} */ (undefined)
|
|
@@ -437,6 +449,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
437
449
|
activate() {
|
|
438
450
|
this.state = "active"
|
|
439
451
|
this.activatedAt = new Date().toISOString()
|
|
452
|
+
this.clearDrainLivenessSamples()
|
|
440
453
|
}
|
|
441
454
|
|
|
442
455
|
/** Runs the configured release-generation activation acknowledgement, if any. */
|
|
@@ -491,6 +504,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
491
504
|
this.drainStartedAt = undefined
|
|
492
505
|
this.retirementError = undefined
|
|
493
506
|
this.stoppedAt = undefined
|
|
507
|
+
this.clearDrainLivenessSamples()
|
|
494
508
|
}
|
|
495
509
|
|
|
496
510
|
/**
|
|
@@ -889,6 +903,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
889
903
|
this.stoppedAt = new Date().toISOString()
|
|
890
904
|
this.releasePortReservations()
|
|
891
905
|
this.abandonLocalConnections()
|
|
906
|
+
this.clearDrainLivenessSamples()
|
|
892
907
|
}
|
|
893
908
|
|
|
894
909
|
/**
|
|
@@ -932,6 +947,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
932
947
|
this.stoppedAt = new Date().toISOString()
|
|
933
948
|
this.releasePortReservations()
|
|
934
949
|
this.abandonLocalConnections()
|
|
950
|
+
this.clearDrainLivenessSamples()
|
|
935
951
|
}
|
|
936
952
|
|
|
937
953
|
/** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
|
|
@@ -939,12 +955,87 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
939
955
|
await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
|
|
940
956
|
}
|
|
941
957
|
|
|
958
|
+
/**
|
|
959
|
+
* Records one liveness observation for a draining generation and classifies its drain.
|
|
960
|
+
*
|
|
961
|
+
* Two observations are needed to prove CPU flatness, and reading a process group means
|
|
962
|
+
* scanning procfs, so sampling is bounded three ways: a generation that still owns live
|
|
963
|
+
* connections is classified without touching procfs at all (connections alone settle it), a
|
|
964
|
+
* new observation is taken only once the newest one is already a full window behind the
|
|
965
|
+
* present read, and every other read reuses the newest observation. A group that is not
|
|
966
|
+
* draining reports no drain.
|
|
967
|
+
* @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
|
|
968
|
+
*/
|
|
969
|
+
drainStatus() {
|
|
970
|
+
if (this.state !== "draining") return undefined
|
|
971
|
+
|
|
972
|
+
const nowMs = Date.now()
|
|
973
|
+
|
|
974
|
+
if (this.connectionCount > 0) {
|
|
975
|
+
return classifyDrainProgress({
|
|
976
|
+
connectionCount: this.connectionCount,
|
|
977
|
+
currentSample: this.drainNewestSample ?? {atMs: nowMs, processes: []},
|
|
978
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
979
|
+
previousSample: undefined
|
|
980
|
+
})
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
const newest = this.drainNewestSample
|
|
984
|
+
const currentSample = newest && nowMs - newest.atMs < DRAIN_SAMPLE_WINDOW_MS ? newest : this.takeLivenessSample(nowMs)
|
|
985
|
+
const previousSample = currentSample === newest ? this.drainPriorSample : newest
|
|
986
|
+
|
|
987
|
+
if (currentSample !== newest) {
|
|
988
|
+
this.drainPriorSample = newest
|
|
989
|
+
this.drainNewestSample = currentSample
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
return classifyDrainProgress({
|
|
993
|
+
connectionCount: this.connectionCount,
|
|
994
|
+
currentSample,
|
|
995
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
996
|
+
previousSample
|
|
997
|
+
})
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
/**
|
|
1001
|
+
* @param {number} nowMs - Observation time.
|
|
1002
|
+
* @returns {DrainLivenessSample} One liveness observation of every process in the generation.
|
|
1003
|
+
*/
|
|
1004
|
+
takeLivenessSample(nowMs) {
|
|
1005
|
+
const processes = [...this.processes.entries()].map(([id, processInstance]) => {
|
|
1006
|
+
const processStatus = processInstance.status()
|
|
1007
|
+
const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
|
|
1008
|
+
// A group that outlived its tracked process is still observed by group id: it holds the
|
|
1009
|
+
// generation's work even though no rollbridge record can signal it any more.
|
|
1010
|
+
const pgid = processStatus.pid ?? heldProcessGroupPgid
|
|
1011
|
+
const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
|
|
1012
|
+
|
|
1013
|
+
return {
|
|
1014
|
+
cpuMs: sample?.cpuMs,
|
|
1015
|
+
heldProcessGroupPgid,
|
|
1016
|
+
id,
|
|
1017
|
+
liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
|
|
1018
|
+
pid: processStatus.pid,
|
|
1019
|
+
retirementTimedOut: processInstance.retirementTimedOut === true
|
|
1020
|
+
}
|
|
1021
|
+
})
|
|
1022
|
+
|
|
1023
|
+
return {atMs: nowMs, processes}
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
/** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
|
|
1027
|
+
clearDrainLivenessSamples() {
|
|
1028
|
+
this.drainNewestSample = undefined
|
|
1029
|
+
this.drainPriorSample = undefined
|
|
1030
|
+
}
|
|
1031
|
+
|
|
942
1032
|
/** @returns {ReleaseStatus} Status payload. */
|
|
943
1033
|
status() {
|
|
944
1034
|
return {
|
|
945
1035
|
activatedAt: this.activatedAt,
|
|
946
1036
|
connectionCount: this.connectionCount,
|
|
947
1037
|
connections: {...this.connections},
|
|
1038
|
+
drain: this.drainStatus(),
|
|
948
1039
|
drainStartedAt: this.drainStartedAt,
|
|
949
1040
|
ports: {...this.ports},
|
|
950
1041
|
processes: [...this.processes.values()].map((processInstance) => processInstance.status()),
|
|
@@ -705,17 +705,24 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
|
|
|
705
705
|
const defaulted = validateRetention(undefined)
|
|
706
706
|
|
|
707
707
|
expect(defaulted.issues).toEqual([])
|
|
708
|
-
expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0})
|
|
708
|
+
expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
|
|
709
709
|
|
|
710
|
-
const custom = validateRetention({keep: 3, maxAgeMs: 60000})
|
|
710
|
+
const custom = validateRetention({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
711
711
|
|
|
712
712
|
expect(custom.issues).toEqual([])
|
|
713
|
-
expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000})
|
|
713
|
+
expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
714
714
|
|
|
715
|
-
|
|
715
|
+
// 0 is a valid bound: it disables the pre-activation draining-generation check.
|
|
716
|
+
const unbounded = validateRetention({maxConcurrentDrainingReleaseGroups: 0})
|
|
717
|
+
|
|
718
|
+
expect(unbounded.issues).toEqual([])
|
|
719
|
+
expect(unbounded.config.releaseRetention.maxConcurrentDrainingReleaseGroups).toBe(0)
|
|
720
|
+
|
|
721
|
+
const invalid = validateRetention({keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
|
|
716
722
|
|
|
717
723
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.keep must be a non-negative integer")).toBeTruthy()
|
|
718
724
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxAgeMs must be a non-negative number")).toBeTruthy()
|
|
725
|
+
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxConcurrentDrainingReleaseGroups must be a non-negative integer")).toBeTruthy()
|
|
719
726
|
})
|
|
720
727
|
|
|
721
728
|
test("validateConfig leaves statePath unset by default, accepts a string, and rejects non-strings", () => {
|