rollbridge 0.1.62 → 0.1.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/docs/cli.md +44 -0
- package/docs/compose-services.md +8 -6
- package/docs/config.md +27 -1
- package/package.json +1 -1
- package/src/config.js +15 -3
- package/src/daemon.js +83 -1
- package/src/drain-policy.js +159 -0
- package/src/managed-process.js +116 -9
- package/src/process-guardian.js +4 -5
- package/src/process-memory.js +62 -2
- package/src/release-group.js +110 -5
- package/test/config-validation.test.js +11 -4
- package/test/drain-policy.test.js +199 -0
- package/test/managed-process.test.js +47 -1
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +155 -2
- package/test/rollbridge.test.js +120 -0
package/src/managed-process.js
CHANGED
|
@@ -2,10 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
import {EventEmitter} from "node:events"
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
|
-
import {processGroupHasLiveMembers, processGroupMembers} from "./process-memory.js"
|
|
5
|
+
import {processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "./process-memory.js"
|
|
6
6
|
|
|
7
7
|
const DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS = 30000
|
|
8
8
|
const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
|
|
9
|
+
// Reading a process group means scanning procfs, and status() is read continuously. A held
|
|
10
|
+
// group is re-verified on this interval instead of on every read.
|
|
11
|
+
const HELD_GROUP_RECHECK_MS = 5000
|
|
9
12
|
|
|
10
13
|
/**
|
|
11
14
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -16,7 +19,8 @@ const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
|
|
|
16
19
|
* @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
|
|
17
20
|
* @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
|
|
18
21
|
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
19
|
-
* @typedef {{
|
|
22
|
+
* @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
|
|
23
|
+
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
20
24
|
*/
|
|
21
25
|
|
|
22
26
|
export default class ManagedProcess extends EventEmitter {
|
|
@@ -73,6 +77,10 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
73
77
|
this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
74
78
|
this.quiesceError = /** @type {Error | undefined} */ (undefined)
|
|
75
79
|
this.stopPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
80
|
+
this.retirementTimedOut = false
|
|
81
|
+
this.groupOutlivedStop = /** @type {number | undefined} */ (undefined)
|
|
82
|
+
this.heldProcessGroupCheckedAtMs = /** @type {number | undefined} */ (undefined)
|
|
83
|
+
this.heldProcessGroupCount = /** @type {number | undefined} */ (undefined)
|
|
76
84
|
this.operationRevision = 0
|
|
77
85
|
this.restartTimer = undefined
|
|
78
86
|
this.child = undefined
|
|
@@ -100,6 +108,10 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
100
108
|
this.quiescePromise = undefined
|
|
101
109
|
this.quiesceError = undefined
|
|
102
110
|
this.stopPromise = undefined
|
|
111
|
+
this.retirementTimedOut = false
|
|
112
|
+
this.groupOutlivedStop = undefined
|
|
113
|
+
this.heldProcessGroupCheckedAtMs = undefined
|
|
114
|
+
this.heldProcessGroupCount = undefined
|
|
103
115
|
this.exitCode = undefined
|
|
104
116
|
this.exitSignal = undefined
|
|
105
117
|
this.state = "starting"
|
|
@@ -250,6 +262,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
250
262
|
this.children = []
|
|
251
263
|
this.clearMemoryMonitor()
|
|
252
264
|
this.state = wasIntentional ? "stopped" : "failed"
|
|
265
|
+
|
|
253
266
|
this.logger("process exited", {code, id: this.id, signal})
|
|
254
267
|
this.emit("exit", {code, signal})
|
|
255
268
|
|
|
@@ -258,6 +271,69 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
258
271
|
}
|
|
259
272
|
}
|
|
260
273
|
|
|
274
|
+
/**
|
|
275
|
+
* Records that the owned process group was still alive when the bounded stop gave up.
|
|
276
|
+
*
|
|
277
|
+
* Every process is spawned `detached: true`, so the tracked pid is the group leader and a
|
|
278
|
+
* member that survives its leader — a `sh` wrapper killed by the stop signal while the app
|
|
279
|
+
* it launched keeps running, or a runner child that outlives its parent — keeps holding the
|
|
280
|
+
* group id while the tracked ChildProcess record is already terminal. Nothing in rollbridge
|
|
281
|
+
* can wait for or signal that group again, yet the group still holds the release's work.
|
|
282
|
+
*
|
|
283
|
+
* Only the stop path can know this, and it already observed the group while waiting for it to
|
|
284
|
+
* leave, so the group id is recorded there instead of rediscovered in `onExit`: that hook
|
|
285
|
+
* fires for every retired process, and a procfs scan costs ~200 ms per exit on a busy host.
|
|
286
|
+
* @param {number} pgid - Process group id the tracked process led.
|
|
287
|
+
* @returns {void}
|
|
288
|
+
*/
|
|
289
|
+
markGroupOutlivedStop(pgid) {
|
|
290
|
+
if (this.groupOutlivedStop === pgid) return
|
|
291
|
+
|
|
292
|
+
this.groupOutlivedStop = pgid
|
|
293
|
+
this.heldProcessGroupCheckedAtMs = undefined
|
|
294
|
+
this.heldProcessGroupCount = undefined
|
|
295
|
+
this.logger("process group outlived its tracked process", {id: this.id, pgid})
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* Reports the process group left behind by a bounded stop, as observed right now.
|
|
300
|
+
*
|
|
301
|
+
* The record is re-verified rather than trusted forever: once the orphaned members exit, the
|
|
302
|
+
* group is gone and stops holding the key. Re-verification is rate-limited to
|
|
303
|
+
* `HELD_GROUP_RECHECK_MS` because it costs a procfs scan, and when procfs cannot answer at
|
|
304
|
+
* all the last known state is reported instead of guessing that the group is gone.
|
|
305
|
+
* @returns {HeldProcessGroup | undefined} The held group, or undefined when nothing is held.
|
|
306
|
+
*/
|
|
307
|
+
currentHeldProcessGroup() {
|
|
308
|
+
const pgid = this.groupOutlivedStop
|
|
309
|
+
|
|
310
|
+
if (pgid === undefined) return undefined
|
|
311
|
+
if (this.state !== "stopped" && this.state !== "failed") return undefined
|
|
312
|
+
|
|
313
|
+
const nowMs = Date.now()
|
|
314
|
+
|
|
315
|
+
if (this.heldProcessGroupCount !== undefined && this.heldProcessGroupCheckedAtMs !== undefined && nowMs - this.heldProcessGroupCheckedAtMs < HELD_GROUP_RECHECK_MS) {
|
|
316
|
+
return {liveMemberCount: this.heldProcessGroupCount, pgid}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
const sample = sampleProcessGroup(pgid)
|
|
320
|
+
|
|
321
|
+
this.heldProcessGroupCheckedAtMs = nowMs
|
|
322
|
+
|
|
323
|
+
if (sample === undefined) return {liveMemberCount: this.heldProcessGroupCount, pgid}
|
|
324
|
+
|
|
325
|
+
if (sample.memberCount === 0) {
|
|
326
|
+
this.groupOutlivedStop = undefined
|
|
327
|
+
this.heldProcessGroupCount = undefined
|
|
328
|
+
|
|
329
|
+
return undefined
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
this.heldProcessGroupCount = sample.liveMemberCount
|
|
333
|
+
|
|
334
|
+
return {liveMemberCount: sample.liveMemberCount, pgid}
|
|
335
|
+
}
|
|
336
|
+
|
|
261
337
|
/**
|
|
262
338
|
* Schedules an automatic restart per the restart policy, or gives up once the policy's limit is reached.
|
|
263
339
|
* @returns {void}
|
|
@@ -411,8 +487,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
411
487
|
}
|
|
412
488
|
|
|
413
489
|
/**
|
|
414
|
-
* @param {{timeoutMs?: number}} [options] - Stop options.
|
|
415
|
-
* @returns {Promise<void>} Resolves when stopped.
|
|
490
|
+
* @param {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
|
|
491
|
+
* @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
|
|
416
492
|
*/
|
|
417
493
|
async stop(options = {}) {
|
|
418
494
|
this.operationRevision += 1
|
|
@@ -421,8 +497,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
421
497
|
}
|
|
422
498
|
|
|
423
499
|
/**
|
|
424
|
-
* @param {{timeoutMs?: number}} options - Stop options.
|
|
425
|
-
* @returns {Promise<void>} Resolves when stopped.
|
|
500
|
+
* @param {{retirementTimeoutMs?: number, timeoutMs?: number}} options - Stop options.
|
|
501
|
+
* @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
|
|
426
502
|
*/
|
|
427
503
|
async performStop(options) {
|
|
428
504
|
const pgid = this.child?.pid ?? this.pid
|
|
@@ -448,6 +524,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
448
524
|
// 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
|
|
449
525
|
if (this.processGroupExists(pgid)) {
|
|
450
526
|
const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
|
|
527
|
+
const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
|
|
451
528
|
let gracefulDeadline
|
|
452
529
|
|
|
453
530
|
if (stopCommand) {
|
|
@@ -456,15 +533,25 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
456
533
|
} else {
|
|
457
534
|
this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
|
|
458
535
|
gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
|
|
459
|
-
await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline)
|
|
536
|
+
await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline ?? retirementDeadline)
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
const exited = await this.waitForProcessGroupExit(pgid, gracefulDeadline === undefined ? retirementDeadline : gracefulDeadline)
|
|
540
|
+
|
|
541
|
+
if (!exited && retirementDeadline !== undefined && gracefulDeadline === undefined) {
|
|
542
|
+
this.retirementTimedOut = true
|
|
543
|
+
this.markGroupOutlivedStop(pgid)
|
|
544
|
+
this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
|
|
545
|
+
return
|
|
460
546
|
}
|
|
461
547
|
|
|
462
|
-
if (!
|
|
548
|
+
if (!exited) {
|
|
463
549
|
this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: pgid})
|
|
464
550
|
const killDeadline = Date.now() + 5000
|
|
465
551
|
|
|
466
552
|
await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
|
|
467
|
-
|
|
553
|
+
|
|
554
|
+
if (!await this.waitForProcessGroupExit(pgid, killDeadline)) this.markGroupOutlivedStop(pgid)
|
|
468
555
|
}
|
|
469
556
|
}
|
|
470
557
|
|
|
@@ -817,6 +904,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
817
904
|
cwd: this.cwd,
|
|
818
905
|
exitCode: this.exitCode,
|
|
819
906
|
exitSignal: this.exitSignal,
|
|
907
|
+
heldProcessGroup: this.currentHeldProcessGroup(),
|
|
820
908
|
id: this.id,
|
|
821
909
|
lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
|
|
822
910
|
lastStartReason: this.lastStartReason,
|
|
@@ -825,6 +913,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
825
913
|
memoryRestarts: this.memoryRestarts,
|
|
826
914
|
pid: this.pid,
|
|
827
915
|
restarts: this.restarts,
|
|
916
|
+
retirementTimedOut: this.retirementTimedOut,
|
|
828
917
|
rssBytes: this.rssBytes,
|
|
829
918
|
startedAt: this.startedAtMs === undefined ? undefined : new Date(this.startedAtMs).toISOString(),
|
|
830
919
|
state: this.state,
|
|
@@ -832,3 +921,21 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
832
921
|
}
|
|
833
922
|
}
|
|
834
923
|
}
|
|
924
|
+
|
|
925
|
+
/**
|
|
926
|
+
* The guardian's provenance fence, in one place: a registered child stops fencing a later
|
|
927
|
+
* claim on its key only when its record is terminal *and* it holds no pid, and a process
|
|
928
|
+
* group that outlived the tracked process is not gone either.
|
|
929
|
+
*
|
|
930
|
+
* Both the guardian (re-registering a key under a new provenance) and the daemon (restoring a
|
|
931
|
+
* release that is recorded as drained) must agree on this, or a restore reaches the guardian
|
|
932
|
+
* and fails there with a bare `Guardian provenance mismatch` naming a process the operator
|
|
933
|
+
* cannot see in `status`.
|
|
934
|
+
* @param {Pick<ManagedProcessStatus, "heldProcessGroup" | "pid" | "state">} status - Process status record.
|
|
935
|
+
* @returns {boolean} True when the child is conclusively gone.
|
|
936
|
+
*/
|
|
937
|
+
export function childRecordIsGone(status) {
|
|
938
|
+
if (status.heldProcessGroup) return false
|
|
939
|
+
|
|
940
|
+
return (status.state === "stopped" || status.state === "failed") && status.pid === undefined
|
|
941
|
+
}
|
package/src/process-guardian.js
CHANGED
|
@@ -8,7 +8,7 @@ import {spawn} from "node:child_process"
|
|
|
8
8
|
import {isDeepStrictEqual} from "node:util"
|
|
9
9
|
import path from "node:path"
|
|
10
10
|
import GuardianClient from "./guardian-client.js"
|
|
11
|
-
import ManagedProcess from "./managed-process.js"
|
|
11
|
+
import ManagedProcess, {childRecordIsGone} from "./managed-process.js"
|
|
12
12
|
import {writeState} from "./state-store.js"
|
|
13
13
|
|
|
14
14
|
const OWNER_RESTART_RETRY_MS = 1000
|
|
@@ -656,10 +656,9 @@ async function execute(request, socket) {
|
|
|
656
656
|
// residue of a dead generation, and demanding its provenance wedges every later deploy
|
|
657
657
|
// against a process that no longer exists (TensorBuzz production, STC-695). The owner
|
|
658
658
|
// may re-register the key under the new definition; live processes stay fail-closed.
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
if (!childIsGone) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
659
|
+
// A group that outlived its tracked process is not gone: it still holds the key's work,
|
|
660
|
+
// so re-registering it would start a second copy of a generation that is still running.
|
|
661
|
+
if (!childRecordIsGone(existing.process.status())) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
663
662
|
requireOwner(socket, request.command)
|
|
664
663
|
processes.delete(request.key)
|
|
665
664
|
}
|
package/src/process-memory.js
CHANGED
|
@@ -4,8 +4,14 @@ import fs from "node:fs"
|
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
|
|
7
|
+
* @typedef {{cpuMs: number | undefined, liveMemberCount: number, memberCount: number}} ProcessGroupSample
|
|
7
8
|
*/
|
|
8
9
|
|
|
10
|
+
// Linux reports /proc/<pid>/stat CPU time in clock ticks and keeps USER_HZ at 100 for
|
|
11
|
+
// userspace, so ticks * 10 / 1 is milliseconds. This is a monotonic per-process counter,
|
|
12
|
+
// not a rate: two reads of it measure whether a process did any work between them.
|
|
13
|
+
const CLOCK_TICKS_PER_MS = 10
|
|
14
|
+
|
|
9
15
|
/**
|
|
10
16
|
* Lists the members of a managed process group with each member's resident memory.
|
|
11
17
|
* Rollbridge spawns each process detached, so the spawned pid is the process-group
|
|
@@ -83,6 +89,51 @@ export function measureProcessGroupRssBytes(pgid) {
|
|
|
83
89
|
return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
|
|
84
90
|
}
|
|
85
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Samples one managed process group for drain liveness: how much CPU its members have
|
|
94
|
+
* consumed in total, and how many of them can still run.
|
|
95
|
+
*
|
|
96
|
+
* CPU is the sum of every member's cumulative user+system time, defunct members included:
|
|
97
|
+
* a member that exits takes its consumed time out of the sum, so the difference between two
|
|
98
|
+
* samples is non-zero whenever the group changed at all — new work or a departing member.
|
|
99
|
+
* Only an exactly unchanged difference says the group did nothing between the two reads.
|
|
100
|
+
* @param {number} pgid - Process-group id (the detached spawn's pid).
|
|
101
|
+
* @param {string} [procPath] - Procfs root (overridable for deterministic tests).
|
|
102
|
+
* @returns {ProcessGroupSample | undefined} The sample, or undefined only when procfs cannot be
|
|
103
|
+
* read at all. A group with no members left is a sample of zero, not a missing measurement.
|
|
104
|
+
*/
|
|
105
|
+
export function sampleProcessGroup(pgid, procPath = "/proc") {
|
|
106
|
+
/** @type {string[]} */
|
|
107
|
+
let entries
|
|
108
|
+
|
|
109
|
+
try {
|
|
110
|
+
entries = fs.readdirSync(procPath)
|
|
111
|
+
} catch {
|
|
112
|
+
return undefined
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
let cpuMs = 0
|
|
116
|
+
let liveMemberCount = 0
|
|
117
|
+
let memberCount = 0
|
|
118
|
+
let measurable = true
|
|
119
|
+
|
|
120
|
+
for (const entry of entries) {
|
|
121
|
+
if (!/^\d+$/.test(entry)) continue
|
|
122
|
+
|
|
123
|
+
const stat = processStat(entry, procPath)
|
|
124
|
+
|
|
125
|
+
if (stat?.pgrp !== pgid) continue
|
|
126
|
+
|
|
127
|
+
memberCount += 1
|
|
128
|
+
if (stat.state !== "Z" && stat.state !== "X") liveMemberCount += 1
|
|
129
|
+
|
|
130
|
+
if (stat.cpuMs === undefined) measurable = false
|
|
131
|
+
else cpuMs += stat.cpuMs
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return {cpuMs: measurable ? cpuMs : undefined, liveMemberCount, memberCount}
|
|
135
|
+
}
|
|
136
|
+
|
|
86
137
|
/**
|
|
87
138
|
* @param {string} pid - Process id.
|
|
88
139
|
* @returns {string} The process command name (`/proc/<pid>/comm`), or "" when unavailable.
|
|
@@ -106,7 +157,7 @@ function processGroupId(pid) {
|
|
|
106
157
|
/**
|
|
107
158
|
* @param {string} pid - Process id.
|
|
108
159
|
* @param {string} procPath - Procfs root.
|
|
109
|
-
* @returns {{pgrp: number, state: string} | undefined} Parsed process state and
|
|
160
|
+
* @returns {{cpuMs: number | undefined, pgrp: number, state: string} | undefined} Parsed process state, group, and cumulative CPU.
|
|
110
161
|
*/
|
|
111
162
|
function processStat(pid, procPath) {
|
|
112
163
|
let stat
|
|
@@ -126,7 +177,16 @@ function processStat(pid, procPath) {
|
|
|
126
177
|
const fields = stat.slice(commEnd + 2).split(" ")
|
|
127
178
|
const pgrp = Number(fields[2])
|
|
128
179
|
|
|
129
|
-
|
|
180
|
+
if (!Number.isInteger(pgrp)) return undefined
|
|
181
|
+
|
|
182
|
+
// utime and stime are fields 14 and 15 of stat, i.e. index 11 and 12 after the comm field.
|
|
183
|
+
const utimeTicks = Number(fields[11])
|
|
184
|
+
const stimeTicks = Number(fields[12])
|
|
185
|
+
const cpuMs = Number.isFinite(utimeTicks) && Number.isFinite(stimeTicks)
|
|
186
|
+
? (utimeTicks + stimeTicks) * CLOCK_TICKS_PER_MS
|
|
187
|
+
: undefined
|
|
188
|
+
|
|
189
|
+
return {cpuMs, pgrp, state: fields[0]}
|
|
130
190
|
}
|
|
131
191
|
|
|
132
192
|
/**
|
package/src/release-group.js
CHANGED
|
@@ -2,19 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
import {spawn} from "node:child_process"
|
|
4
4
|
import {EventEmitter} from "node:events"
|
|
5
|
+
import {classifyDrainProgress} from "./drain-policy.js"
|
|
5
6
|
import ManagedProcess from "./managed-process.js"
|
|
6
7
|
import {findAvailablePort} from "./port-allocator.js"
|
|
7
8
|
import {processStartOrder} from "./process-order.js"
|
|
9
|
+
import {sampleProcessGroup} from "./process-memory.js"
|
|
8
10
|
import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
|
|
9
11
|
import {waitForHealth, waitForListeningPort} from "./health.js"
|
|
10
12
|
|
|
11
13
|
const DEPENDENCY_READY_INTERVAL_MS = 250
|
|
14
|
+
// Shortest gap between two liveness observations that can prove a draining group's CPU is
|
|
15
|
+
// flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
|
|
16
|
+
// with live runner children doing work, so flatness is only evidence below this window.
|
|
17
|
+
const DRAIN_SAMPLE_WINDOW_MS = 20000
|
|
12
18
|
|
|
13
19
|
/**
|
|
14
20
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
15
21
|
* @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
|
|
22
|
+
* @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
|
|
23
|
+
* @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
|
|
16
24
|
* @typedef {{http: number, websocket: number}} ReleaseConnections
|
|
17
|
-
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
25
|
+
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
18
26
|
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
|
|
19
27
|
* @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
|
|
20
28
|
*/
|
|
@@ -89,9 +97,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
89
97
|
* @param {string | undefined} args.revision - Revision.
|
|
90
98
|
* @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
|
|
91
99
|
* @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
|
|
100
|
+
* @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
|
|
92
101
|
* @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
|
|
93
102
|
*/
|
|
94
|
-
constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
|
|
103
|
+
constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
|
|
95
104
|
super()
|
|
96
105
|
|
|
97
106
|
this.config = config
|
|
@@ -113,8 +122,11 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
113
122
|
this.servicePorts = servicePorts
|
|
114
123
|
this.shouldStart = shouldStart
|
|
115
124
|
this.processFactory = processFactory
|
|
125
|
+
this.procPath = procPath
|
|
116
126
|
this.portsAllocated = false
|
|
117
127
|
this.drainStartedAt = /** @type {string | undefined} */ (undefined)
|
|
128
|
+
this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
|
|
129
|
+
this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
|
|
118
130
|
this.activatedAt = /** @type {string | undefined} */ (undefined)
|
|
119
131
|
this.stoppedAt = /** @type {string | undefined} */ (undefined)
|
|
120
132
|
this.retirementError = /** @type {string | undefined} */ (undefined)
|
|
@@ -437,6 +449,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
437
449
|
activate() {
|
|
438
450
|
this.state = "active"
|
|
439
451
|
this.activatedAt = new Date().toISOString()
|
|
452
|
+
this.clearDrainLivenessSamples()
|
|
440
453
|
}
|
|
441
454
|
|
|
442
455
|
/** Runs the configured release-generation activation acknowledgement, if any. */
|
|
@@ -491,6 +504,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
491
504
|
this.drainStartedAt = undefined
|
|
492
505
|
this.retirementError = undefined
|
|
493
506
|
this.stoppedAt = undefined
|
|
507
|
+
this.clearDrainLivenessSamples()
|
|
494
508
|
}
|
|
495
509
|
|
|
496
510
|
/**
|
|
@@ -843,7 +857,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
843
857
|
// their lifecycle drain runs as soon as the release is retired — in parallel with the
|
|
844
858
|
// connection drain, not held until after it. The rest stop once connections have closed.
|
|
845
859
|
const entries = [...this.processes.entries()]
|
|
846
|
-
const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([, processInstance]) =>
|
|
860
|
+
const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
|
|
861
|
+
id,
|
|
862
|
+
promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
|
|
863
|
+
}))
|
|
847
864
|
const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
848
865
|
const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
849
866
|
|
|
@@ -864,17 +881,29 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
864
881
|
}
|
|
865
882
|
|
|
866
883
|
if (this.ownerHandoffPaused) {
|
|
867
|
-
void Promise.allSettled(nonBlockingStops)
|
|
884
|
+
void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
|
|
868
885
|
return
|
|
869
886
|
}
|
|
870
887
|
|
|
871
888
|
await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop()))
|
|
872
|
-
await Promise.allSettled(nonBlockingStops)
|
|
889
|
+
const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
|
|
890
|
+
nonBlockingResults.forEach((result, index) => {
|
|
891
|
+
if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
|
|
892
|
+
const processInstance = this.processes.get(nonBlockingStops[index].id)
|
|
893
|
+
this.logger("release non-blocking process still draining", {
|
|
894
|
+
pid: processInstance?.status().pid ?? null,
|
|
895
|
+
processId: nonBlockingStops[index].id,
|
|
896
|
+
releaseId: this.releaseId,
|
|
897
|
+
timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
|
|
898
|
+
})
|
|
899
|
+
}
|
|
900
|
+
})
|
|
873
901
|
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
|
|
874
902
|
this.state = "stopped"
|
|
875
903
|
this.stoppedAt = new Date().toISOString()
|
|
876
904
|
this.releasePortReservations()
|
|
877
905
|
this.abandonLocalConnections()
|
|
906
|
+
this.clearDrainLivenessSamples()
|
|
878
907
|
}
|
|
879
908
|
|
|
880
909
|
/**
|
|
@@ -918,6 +947,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
918
947
|
this.stoppedAt = new Date().toISOString()
|
|
919
948
|
this.releasePortReservations()
|
|
920
949
|
this.abandonLocalConnections()
|
|
950
|
+
this.clearDrainLivenessSamples()
|
|
921
951
|
}
|
|
922
952
|
|
|
923
953
|
/** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
|
|
@@ -925,12 +955,87 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
925
955
|
await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
|
|
926
956
|
}
|
|
927
957
|
|
|
958
|
+
/**
|
|
959
|
+
* Records one liveness observation for a draining generation and classifies its drain.
|
|
960
|
+
*
|
|
961
|
+
* Two observations are needed to prove CPU flatness, and reading a process group means
|
|
962
|
+
* scanning procfs, so sampling is bounded three ways: a generation that still owns live
|
|
963
|
+
* connections is classified without touching procfs at all (connections alone settle it), a
|
|
964
|
+
* new observation is taken only once the newest one is already a full window behind the
|
|
965
|
+
* present read, and every other read reuses the newest observation. A group that is not
|
|
966
|
+
* draining reports no drain.
|
|
967
|
+
* @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
|
|
968
|
+
*/
|
|
969
|
+
drainStatus() {
|
|
970
|
+
if (this.state !== "draining") return undefined
|
|
971
|
+
|
|
972
|
+
const nowMs = Date.now()
|
|
973
|
+
|
|
974
|
+
if (this.connectionCount > 0) {
|
|
975
|
+
return classifyDrainProgress({
|
|
976
|
+
connectionCount: this.connectionCount,
|
|
977
|
+
currentSample: this.drainNewestSample ?? {atMs: nowMs, processes: []},
|
|
978
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
979
|
+
previousSample: undefined
|
|
980
|
+
})
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
const newest = this.drainNewestSample
|
|
984
|
+
const currentSample = newest && nowMs - newest.atMs < DRAIN_SAMPLE_WINDOW_MS ? newest : this.takeLivenessSample(nowMs)
|
|
985
|
+
const previousSample = currentSample === newest ? this.drainPriorSample : newest
|
|
986
|
+
|
|
987
|
+
if (currentSample !== newest) {
|
|
988
|
+
this.drainPriorSample = newest
|
|
989
|
+
this.drainNewestSample = currentSample
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
return classifyDrainProgress({
|
|
993
|
+
connectionCount: this.connectionCount,
|
|
994
|
+
currentSample,
|
|
995
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
996
|
+
previousSample
|
|
997
|
+
})
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
/**
|
|
1001
|
+
* @param {number} nowMs - Observation time.
|
|
1002
|
+
* @returns {DrainLivenessSample} One liveness observation of every process in the generation.
|
|
1003
|
+
*/
|
|
1004
|
+
takeLivenessSample(nowMs) {
|
|
1005
|
+
const processes = [...this.processes.entries()].map(([id, processInstance]) => {
|
|
1006
|
+
const processStatus = processInstance.status()
|
|
1007
|
+
const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
|
|
1008
|
+
// A group that outlived its tracked process is still observed by group id: it holds the
|
|
1009
|
+
// generation's work even though no rollbridge record can signal it any more.
|
|
1010
|
+
const pgid = processStatus.pid ?? heldProcessGroupPgid
|
|
1011
|
+
const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
|
|
1012
|
+
|
|
1013
|
+
return {
|
|
1014
|
+
cpuMs: sample?.cpuMs,
|
|
1015
|
+
heldProcessGroupPgid,
|
|
1016
|
+
id,
|
|
1017
|
+
liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
|
|
1018
|
+
pid: processStatus.pid,
|
|
1019
|
+
retirementTimedOut: processInstance.retirementTimedOut === true
|
|
1020
|
+
}
|
|
1021
|
+
})
|
|
1022
|
+
|
|
1023
|
+
return {atMs: nowMs, processes}
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
/** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
|
|
1027
|
+
clearDrainLivenessSamples() {
|
|
1028
|
+
this.drainNewestSample = undefined
|
|
1029
|
+
this.drainPriorSample = undefined
|
|
1030
|
+
}
|
|
1031
|
+
|
|
928
1032
|
/** @returns {ReleaseStatus} Status payload. */
|
|
929
1033
|
status() {
|
|
930
1034
|
return {
|
|
931
1035
|
activatedAt: this.activatedAt,
|
|
932
1036
|
connectionCount: this.connectionCount,
|
|
933
1037
|
connections: {...this.connections},
|
|
1038
|
+
drain: this.drainStatus(),
|
|
934
1039
|
drainStartedAt: this.drainStartedAt,
|
|
935
1040
|
ports: {...this.ports},
|
|
936
1041
|
processes: [...this.processes.values()].map((processInstance) => processInstance.status()),
|
|
@@ -705,17 +705,24 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
|
|
|
705
705
|
const defaulted = validateRetention(undefined)
|
|
706
706
|
|
|
707
707
|
expect(defaulted.issues).toEqual([])
|
|
708
|
-
expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0})
|
|
708
|
+
expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
|
|
709
709
|
|
|
710
|
-
const custom = validateRetention({keep: 3, maxAgeMs: 60000})
|
|
710
|
+
const custom = validateRetention({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
711
711
|
|
|
712
712
|
expect(custom.issues).toEqual([])
|
|
713
|
-
expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000})
|
|
713
|
+
expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
714
714
|
|
|
715
|
-
|
|
715
|
+
// 0 is a valid bound: it disables the pre-activation draining-generation check.
|
|
716
|
+
const unbounded = validateRetention({maxConcurrentDrainingReleaseGroups: 0})
|
|
717
|
+
|
|
718
|
+
expect(unbounded.issues).toEqual([])
|
|
719
|
+
expect(unbounded.config.releaseRetention.maxConcurrentDrainingReleaseGroups).toBe(0)
|
|
720
|
+
|
|
721
|
+
const invalid = validateRetention({keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
|
|
716
722
|
|
|
717
723
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.keep must be a non-negative integer")).toBeTruthy()
|
|
718
724
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxAgeMs must be a non-negative number")).toBeTruthy()
|
|
725
|
+
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxConcurrentDrainingReleaseGroups must be a non-negative integer")).toBeTruthy()
|
|
719
726
|
})
|
|
720
727
|
|
|
721
728
|
test("validateConfig leaves statePath unset by default, accepts a string, and rejects non-strings", () => {
|