rollbridge 0.1.73 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +68 -2
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/daemon.js +411 -78
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +68 -23
- package/src/health.js +99 -8
- package/src/managed-process.js +198 -45
- package/src/process-guardian.js +212 -37
- package/src/process-memory.js +58 -23
- package/src/process-order.js +26 -0
- package/src/release-group.js +438 -45
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +119 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -24
- package/test/owner-recovery.test.js +125 -21
- package/test/owner-replacement.test.js +175 -14
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +408 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/src/managed-process.js
CHANGED
|
@@ -2,10 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
import {EventEmitter} from "node:events"
|
|
4
4
|
import {spawn} from "node:child_process"
|
|
5
|
-
import {processGroupHasLiveMembers, processGroupMembers,
|
|
5
|
+
import {processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "./process-memory.js"
|
|
6
6
|
|
|
7
7
|
const DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS = 30000
|
|
8
8
|
const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
|
|
9
|
+
// Reading a process group means scanning procfs, and status() is read continuously. A held
|
|
10
|
+
// group is re-verified on this interval instead of on every read.
|
|
11
|
+
const HELD_GROUP_RECHECK_MS = 5000
|
|
12
|
+
// How long an escalated SIGKILL is given to clear the group before the group is recorded as
|
|
13
|
+
// outliving it again. Bounded so the escalation itself can never become the unbounded wait it
|
|
14
|
+
// replaced.
|
|
15
|
+
const ESCALATION_KILL_CONFIRM_MS = 5000
|
|
9
16
|
|
|
10
17
|
/**
|
|
11
18
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -15,8 +22,9 @@ const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
|
|
|
15
22
|
* @typedef {import("node:child_process").ChildProcess["signalCode"]} ProcessExitSignal
|
|
16
23
|
* @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
|
|
17
24
|
* @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
|
|
18
|
-
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
19
|
-
* @typedef {{
|
|
25
|
+
* @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, shouldStop?: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
|
|
26
|
+
* @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
|
|
27
|
+
* @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
|
|
20
28
|
*/
|
|
21
29
|
|
|
22
30
|
export default class ManagedProcess extends EventEmitter {
|
|
@@ -33,10 +41,11 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
33
41
|
* @param {import("./config.js").RestartConfig} [args.restart] - Restart policy (defaults to unlimited restarts with a constant delay).
|
|
34
42
|
* @param {number} args.restartDelayMs - Restart delay.
|
|
35
43
|
* @param {() => boolean} args.shouldRestart - Restart policy callback.
|
|
44
|
+
* @param {() => boolean} [args.shouldStop] - Whether an unexpected exit is terminal because its dependency retired.
|
|
36
45
|
* @param {string} [args.stopSignal] - Signal sent to gracefully stop the process (default "SIGTERM").
|
|
37
46
|
* @param {StopTimeoutMs} args.stopTimeoutMs - Stop timeout.
|
|
38
47
|
*/
|
|
39
|
-
constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, stopSignal = "SIGTERM", stopTimeoutMs}) {
|
|
48
|
+
constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, shouldStop = () => false, stopSignal = "SIGTERM", stopTimeoutMs}) {
|
|
40
49
|
super()
|
|
41
50
|
|
|
42
51
|
this.command = command
|
|
@@ -50,6 +59,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
50
59
|
this.restart = restart
|
|
51
60
|
this.restartDelayMs = restartDelayMs
|
|
52
61
|
this.shouldRestart = shouldRestart
|
|
62
|
+
this.shouldStop = shouldStop
|
|
53
63
|
this.stopSignal = stopSignal
|
|
54
64
|
this.stopTimeoutMs = stopTimeoutMs
|
|
55
65
|
this.state = /** @type {ManagedProcessState} */ ("stopped")
|
|
@@ -73,12 +83,15 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
73
83
|
this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
74
84
|
this.quiesceError = /** @type {Error | undefined} */ (undefined)
|
|
75
85
|
this.stopPromise = /** @type {Promise<void> | undefined} */ (undefined)
|
|
86
|
+
this.retirementTimedOut = false
|
|
87
|
+
this.groupOutlivedStop = /** @type {number | undefined} */ (undefined)
|
|
88
|
+
this.heldProcessGroupCheckedAtMs = /** @type {number | undefined} */ (undefined)
|
|
89
|
+
this.heldProcessGroupCount = /** @type {number | undefined} */ (undefined)
|
|
76
90
|
this.operationRevision = 0
|
|
77
91
|
this.restartTimer = undefined
|
|
78
92
|
this.child = undefined
|
|
79
93
|
this.exitPromise = undefined
|
|
80
94
|
this.pid = undefined
|
|
81
|
-
this.processStartTime = undefined
|
|
82
95
|
this.exitCode = undefined
|
|
83
96
|
this.exitSignal = undefined
|
|
84
97
|
}
|
|
@@ -101,6 +114,10 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
101
114
|
this.quiescePromise = undefined
|
|
102
115
|
this.quiesceError = undefined
|
|
103
116
|
this.stopPromise = undefined
|
|
117
|
+
this.retirementTimedOut = false
|
|
118
|
+
this.groupOutlivedStop = undefined
|
|
119
|
+
this.heldProcessGroupCheckedAtMs = undefined
|
|
120
|
+
this.heldProcessGroupCount = undefined
|
|
104
121
|
this.exitCode = undefined
|
|
105
122
|
this.exitSignal = undefined
|
|
106
123
|
this.state = "starting"
|
|
@@ -127,7 +144,6 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
127
144
|
child.once("spawn", () => {
|
|
128
145
|
void (async () => {
|
|
129
146
|
this.startedAtMs = Date.now()
|
|
130
|
-
this.processStartTime = child.pid ? processStartTime(child.pid) : undefined
|
|
131
147
|
this.lastStartReason = reason
|
|
132
148
|
const lifecycleRestore = this.restoreLifecycleRole()
|
|
133
149
|
const lifecycleRestoreBarrier = lifecycleRestore.then(() => undefined, () => undefined)
|
|
@@ -163,7 +179,6 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
163
179
|
if (this.child === child) {
|
|
164
180
|
this.child = undefined
|
|
165
181
|
this.pid = undefined
|
|
166
|
-
this.processStartTime = undefined
|
|
167
182
|
this.exitPromise = undefined
|
|
168
183
|
}
|
|
169
184
|
reject(error)
|
|
@@ -177,29 +192,6 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
177
192
|
})
|
|
178
193
|
}
|
|
179
194
|
|
|
180
|
-
/**
|
|
181
|
-
* Re-adopts an already-live process after a guardian/version handoff.
|
|
182
|
-
* @param {{pid: number, processStartTime: string}} identity - Exact process identity.
|
|
183
|
-
* @param {string} [_configDigest] - Owner digest accepted by guardian-backed implementations.
|
|
184
|
-
* @returns {Promise<void>}
|
|
185
|
-
*/
|
|
186
|
-
async adopt({pid, processStartTime: expectedStartTime}, _configDigest) {
|
|
187
|
-
if (!Number.isInteger(pid) || pid <= 0 || !/^\d+$/.test(expectedStartTime)) throw new Error(`Process ${this.id} adoption identity is invalid`)
|
|
188
|
-
const actualStartTime = processStartTime(pid)
|
|
189
|
-
|
|
190
|
-
if (actualStartTime !== expectedStartTime) throw new Error(`Process ${this.id} adoption rejected: PID ${pid} start time does not match the committed identity`)
|
|
191
|
-
if (this.child) throw new Error(`Process ${this.id} adoption rejected while the process is already supervised`)
|
|
192
|
-
|
|
193
|
-
this.pid = pid
|
|
194
|
-
this.processStartTime = actualStartTime
|
|
195
|
-
this.state = "running"
|
|
196
|
-
this.intentionalStop = false
|
|
197
|
-
this.intentionalStopSignal = undefined
|
|
198
|
-
this.exitCode = undefined
|
|
199
|
-
this.exitSignal = undefined
|
|
200
|
-
this.lastStartReason = "deploy"
|
|
201
|
-
}
|
|
202
|
-
|
|
203
195
|
/**
|
|
204
196
|
* Updates the command template used for future restarts without touching the currently running child.
|
|
205
197
|
* @param {ManagedProcessDefinition} definition - Replacement process definition.
|
|
@@ -217,6 +209,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
217
209
|
this.restart = definition.restart
|
|
218
210
|
this.restartDelayMs = definition.restartDelayMs
|
|
219
211
|
this.shouldRestart = definition.shouldRestart
|
|
212
|
+
this.shouldStop = definition.shouldStop ?? (() => false)
|
|
220
213
|
this.stopSignal = definition.stopSignal
|
|
221
214
|
this.stopTimeoutMs = definition.stopTimeoutMs
|
|
222
215
|
}
|
|
@@ -271,20 +264,84 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
271
264
|
this.exitSignal = signal ?? this.intentionalStopSignal
|
|
272
265
|
this.child = undefined
|
|
273
266
|
this.pid = undefined
|
|
274
|
-
this.processStartTime = undefined
|
|
275
267
|
this.exitPromise = undefined
|
|
276
268
|
this.rssBytes = undefined
|
|
277
269
|
this.children = []
|
|
278
270
|
this.clearMemoryMonitor()
|
|
279
|
-
|
|
271
|
+
const terminalExit = !wasIntentional && this.shouldStop()
|
|
272
|
+
this.state = wasIntentional || terminalExit ? "stopped" : "failed"
|
|
273
|
+
|
|
280
274
|
this.logger("process exited", {code, id: this.id, signal})
|
|
281
275
|
this.emit("exit", {code, signal})
|
|
282
276
|
|
|
283
|
-
if (!wasIntentional && this.shouldRestart()) {
|
|
277
|
+
if (!wasIntentional && !terminalExit && this.shouldRestart()) {
|
|
284
278
|
this.scheduleRestart()
|
|
285
279
|
}
|
|
286
280
|
}
|
|
287
281
|
|
|
282
|
+
/**
|
|
283
|
+
* Records that the owned process group was still alive when the bounded stop gave up.
|
|
284
|
+
*
|
|
285
|
+
* Every process is spawned `detached: true`, so the tracked pid is the group leader and a
|
|
286
|
+
* member that survives its leader — a `sh` wrapper killed by the stop signal while the app
|
|
287
|
+
* it launched keeps running, or a runner child that outlives its parent — keeps holding the
|
|
288
|
+
* group id while the tracked ChildProcess record is already terminal. Nothing in rollbridge
|
|
289
|
+
* can wait for or signal that group again, yet the group still holds the release's work.
|
|
290
|
+
*
|
|
291
|
+
* Only the stop path can know this, and it already observed the group while waiting for it to
|
|
292
|
+
* leave, so the group id is recorded there instead of rediscovered in `onExit`: that hook
|
|
293
|
+
* fires for every retired process, and a procfs scan costs ~200 ms per exit on a busy host.
|
|
294
|
+
* @param {number} pgid - Process group id the tracked process led.
|
|
295
|
+
* @returns {void}
|
|
296
|
+
*/
|
|
297
|
+
markGroupOutlivedStop(pgid) {
|
|
298
|
+
if (this.groupOutlivedStop === pgid) return
|
|
299
|
+
|
|
300
|
+
this.groupOutlivedStop = pgid
|
|
301
|
+
this.heldProcessGroupCheckedAtMs = undefined
|
|
302
|
+
this.heldProcessGroupCount = undefined
|
|
303
|
+
this.logger("process group outlived its tracked process", {id: this.id, pgid})
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Reports the process group left behind by a bounded stop, as observed right now.
|
|
308
|
+
*
|
|
309
|
+
* The record is re-verified rather than trusted forever: once the orphaned members exit, the
|
|
310
|
+
* group is gone and stops holding the key. Re-verification is rate-limited to
|
|
311
|
+
* `HELD_GROUP_RECHECK_MS` because it costs a procfs scan, and when procfs cannot answer at
|
|
312
|
+
* all the last known state is reported instead of guessing that the group is gone.
|
|
313
|
+
* @returns {HeldProcessGroup | undefined} The held group, or undefined when nothing is held.
|
|
314
|
+
*/
|
|
315
|
+
currentHeldProcessGroup() {
|
|
316
|
+
const pgid = this.groupOutlivedStop
|
|
317
|
+
|
|
318
|
+
if (pgid === undefined) return undefined
|
|
319
|
+
if (this.state !== "stopped" && this.state !== "failed") return undefined
|
|
320
|
+
|
|
321
|
+
const nowMs = Date.now()
|
|
322
|
+
|
|
323
|
+
if (this.heldProcessGroupCount !== undefined && this.heldProcessGroupCheckedAtMs !== undefined && nowMs - this.heldProcessGroupCheckedAtMs < HELD_GROUP_RECHECK_MS) {
|
|
324
|
+
return {liveMemberCount: this.heldProcessGroupCount, pgid}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const sample = sampleProcessGroup(pgid)
|
|
328
|
+
|
|
329
|
+
this.heldProcessGroupCheckedAtMs = nowMs
|
|
330
|
+
|
|
331
|
+
if (sample === undefined) return {liveMemberCount: this.heldProcessGroupCount, pgid}
|
|
332
|
+
|
|
333
|
+
if (sample.memberCount === 0) {
|
|
334
|
+
this.groupOutlivedStop = undefined
|
|
335
|
+
this.heldProcessGroupCount = undefined
|
|
336
|
+
|
|
337
|
+
return undefined
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
this.heldProcessGroupCount = sample.liveMemberCount
|
|
341
|
+
|
|
342
|
+
return {liveMemberCount: sample.liveMemberCount, pgid}
|
|
343
|
+
}
|
|
344
|
+
|
|
288
345
|
/**
|
|
289
346
|
* Schedules an automatic restart per the restart policy, or gives up once the policy's limit is reached.
|
|
290
347
|
* @returns {void}
|
|
@@ -438,8 +495,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
438
495
|
}
|
|
439
496
|
|
|
440
497
|
/**
|
|
441
|
-
* @param {{timeoutMs?: number}} [options] - Stop options.
|
|
442
|
-
* @returns {Promise<void>} Resolves when stopped.
|
|
498
|
+
* @param {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
|
|
499
|
+
* @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
|
|
443
500
|
*/
|
|
444
501
|
async stop(options = {}) {
|
|
445
502
|
this.operationRevision += 1
|
|
@@ -448,12 +505,13 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
448
505
|
}
|
|
449
506
|
|
|
450
507
|
/**
|
|
451
|
-
* @param {{timeoutMs?: number}} options - Stop options.
|
|
452
|
-
* @returns {Promise<void>} Resolves when stopped.
|
|
508
|
+
* @param {{retirementTimeoutMs?: number, timeoutMs?: number}} options - Stop options.
|
|
509
|
+
* @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
|
|
453
510
|
*/
|
|
454
511
|
async performStop(options) {
|
|
455
512
|
const pgid = this.child?.pid ?? this.pid
|
|
456
513
|
const exitPromise = this.exitPromise
|
|
514
|
+
const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
|
|
457
515
|
await this.quiesce()
|
|
458
516
|
|
|
459
517
|
if (!pgid) {
|
|
@@ -475,23 +533,43 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
475
533
|
// 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
|
|
476
534
|
if (this.processGroupExists(pgid)) {
|
|
477
535
|
const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
|
|
478
|
-
|
|
536
|
+
const gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
|
|
537
|
+
// A retirement budget is the caller's decision that this process must not be waited for and
|
|
538
|
+
// must not be killed, so it caps the graceful window rather than only standing in for a
|
|
539
|
+
// missing one. Read the other way round, a finite `gracefulStopMs` outvotes the retirement
|
|
540
|
+
// bound and the wait runs for the whole window: TensorBuzz production moved its worker from
|
|
541
|
+
// `"indefinite"` to 4,5 h and the stop would still have held the control plane for 4,5 h.
|
|
542
|
+
const retirementBound = retirementDeadline !== undefined && (gracefulDeadline === undefined || retirementDeadline < gracefulDeadline)
|
|
543
|
+
const waitDeadline = retirementBound ? retirementDeadline : gracefulDeadline
|
|
479
544
|
|
|
480
545
|
if (stopCommand) {
|
|
481
546
|
await this.runHook(stopCommand, hookTimeoutMs, "stop command", pgid)
|
|
482
|
-
gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
|
|
483
547
|
} else {
|
|
484
548
|
this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
|
|
485
|
-
|
|
486
|
-
await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline)
|
|
549
|
+
await this.signalProcessGroup(this.stopSignal, pgid, waitDeadline)
|
|
487
550
|
}
|
|
488
551
|
|
|
489
|
-
|
|
552
|
+
const exited = await this.waitForProcessGroupExit(pgid, waitDeadline)
|
|
553
|
+
|
|
554
|
+
if (!exited && retirementBound) {
|
|
555
|
+
this.retirementTimedOut = true
|
|
556
|
+
this.markGroupOutlivedStop(pgid)
|
|
557
|
+
// The tracked process has completed its supervised stop even though its process group is
|
|
558
|
+
// still draining. Keep the residue in heldProcessGroup rather than leaving the process
|
|
559
|
+
// permanently quiesced, which would make a drained generation look active to restore
|
|
560
|
+
// admission and keep the owner state from describing the actual stopped generation.
|
|
561
|
+
this.state = "stopped"
|
|
562
|
+
this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
|
|
563
|
+
return
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
if (!exited) {
|
|
490
567
|
this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: pgid})
|
|
491
568
|
const killDeadline = Date.now() + 5000
|
|
492
569
|
|
|
493
570
|
await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
|
|
494
|
-
|
|
571
|
+
|
|
572
|
+
if (!await this.waitForProcessGroupExit(pgid, killDeadline)) this.markGroupOutlivedStop(pgid)
|
|
495
573
|
}
|
|
496
574
|
}
|
|
497
575
|
|
|
@@ -500,6 +578,49 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
500
578
|
this.state = "stopped"
|
|
501
579
|
}
|
|
502
580
|
|
|
581
|
+
/**
|
|
582
|
+
* Delivers the SIGKILL that retirement proved it must not wait for.
|
|
583
|
+
*
|
|
584
|
+
* Retirement deliberately leaves a non-blocking process alive: STOE-1077 measured what a
|
|
585
|
+
* stop-timeout kill costs when the process still holds live lanes (67 builds, 27.3
|
|
586
|
+
* build-hours). That is right while the process can still drain and wrong once it cannot — a
|
|
587
|
+
* wedged generation keeps its release open, keeps its port reservation, and makes every
|
|
588
|
+
* later deploy wait on it until an operator signals the exact PIDs by hand. This is the
|
|
589
|
+
* acting half of that decision, and it re-checks the two invariants this class owns before
|
|
590
|
+
* sending anything: the process must already be recorded as past its retirement budget, and
|
|
591
|
+
* its group must still have a member that can be signalled.
|
|
592
|
+
* @param {{reason: string}} options - Named evidence behind the escalation.
|
|
593
|
+
* @returns {Promise<boolean>} True when a live process group was signalled.
|
|
594
|
+
*/
|
|
595
|
+
async escalateRetirement({reason}) {
|
|
596
|
+
const pgid = this.pid ?? this.groupOutlivedStop
|
|
597
|
+
|
|
598
|
+
if (pgid === undefined) return false
|
|
599
|
+
|
|
600
|
+
if (!this.retirementTimedOut) {
|
|
601
|
+
this.logger("retirement escalation refused; process is not past its retirement budget", {id: this.id, pid: pgid})
|
|
602
|
+
|
|
603
|
+
return false
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
if (!this.processGroupExists(pgid)) {
|
|
607
|
+
this.groupOutlivedStop = undefined
|
|
608
|
+
this.heldProcessGroupCount = undefined
|
|
609
|
+
|
|
610
|
+
return false
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
this.logger("process retirement escalated to SIGKILL", {id: this.id, pid: pgid, reason})
|
|
614
|
+
|
|
615
|
+
const killDeadline = Date.now() + ESCALATION_KILL_CONFIRM_MS
|
|
616
|
+
|
|
617
|
+
await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
|
|
618
|
+
|
|
619
|
+
if (!await this.waitForProcessGroupExit(pgid, killDeadline)) this.markGroupOutlivedStop(pgid)
|
|
620
|
+
|
|
621
|
+
return true
|
|
622
|
+
}
|
|
623
|
+
|
|
503
624
|
/** @returns {Promise<void>} Stops restarts and waits until the process has stopped accepting new work. */
|
|
504
625
|
async quiesce() {
|
|
505
626
|
if (this.quiescePromise) return await this.quiescePromise
|
|
@@ -511,7 +632,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
511
632
|
this.restartTimer = undefined
|
|
512
633
|
}
|
|
513
634
|
if (this.lifecycleRestoreBarrier) await this.lifecycleRestoreBarrier
|
|
514
|
-
if (!this.child?.pid
|
|
635
|
+
if (!this.child?.pid) {
|
|
515
636
|
this.state = "stopped"
|
|
516
637
|
if (this.hasRestorableLifecycle()) this.lifecycleRole = "retired"
|
|
517
638
|
return
|
|
@@ -642,6 +763,19 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
642
763
|
return this.stopTimeoutMs
|
|
643
764
|
}
|
|
644
765
|
|
|
766
|
+
/**
|
|
767
|
+
* Reports whether this process's graceful window has no upper bound.
|
|
768
|
+
*
|
|
769
|
+
* `"indefinite"` is the consumer's statement that the supervisor must never kill this process;
|
|
770
|
+
* it is not a statement that a caller may wait for it forever. A stop that has no retirement
|
|
771
|
+
* budget for such a process can only hang, and when that stop runs inside an owner mutation it
|
|
772
|
+
* locks out every later mutation (STOE-2015).
|
|
773
|
+
* @returns {boolean} True when no deadline can ever end the wait.
|
|
774
|
+
*/
|
|
775
|
+
hasUnboundedGracefulWindow() {
|
|
776
|
+
return this.stopTimeoutMs === "indefinite"
|
|
777
|
+
}
|
|
778
|
+
|
|
645
779
|
/**
|
|
646
780
|
* Runs a lifecycle hook command, bounded by a timeout so a hung hook can never block stop().
|
|
647
781
|
* Failures are logged and returned so generation retirement can surface a failed quiet hook;
|
|
@@ -844,6 +978,7 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
844
978
|
cwd: this.cwd,
|
|
845
979
|
exitCode: this.exitCode,
|
|
846
980
|
exitSignal: this.exitSignal,
|
|
981
|
+
heldProcessGroup: this.currentHeldProcessGroup(),
|
|
847
982
|
id: this.id,
|
|
848
983
|
lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
|
|
849
984
|
lastStartReason: this.lastStartReason,
|
|
@@ -851,8 +986,8 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
851
986
|
logs: this.logs.slice(-this.outputLines),
|
|
852
987
|
memoryRestarts: this.memoryRestarts,
|
|
853
988
|
pid: this.pid,
|
|
854
|
-
processStartTime: this.processStartTime,
|
|
855
989
|
restarts: this.restarts,
|
|
990
|
+
retirementTimedOut: this.retirementTimedOut,
|
|
856
991
|
rssBytes: this.rssBytes,
|
|
857
992
|
startedAt: this.startedAtMs === undefined ? undefined : new Date(this.startedAtMs).toISOString(),
|
|
858
993
|
state: this.state,
|
|
@@ -860,3 +995,21 @@ export default class ManagedProcess extends EventEmitter {
|
|
|
860
995
|
}
|
|
861
996
|
}
|
|
862
997
|
}
|
|
998
|
+
|
|
999
|
+
/**
|
|
1000
|
+
* The guardian's provenance fence, in one place: a registered child stops fencing a later
|
|
1001
|
+
* claim on its key only when its record is terminal *and* it holds no pid, and a process
|
|
1002
|
+
* group that outlived the tracked process is not gone either.
|
|
1003
|
+
*
|
|
1004
|
+
* Both the guardian (re-registering a key under a new provenance) and the daemon (restoring a
|
|
1005
|
+
* release that is recorded as drained) must agree on this, or a restore reaches the guardian
|
|
1006
|
+
* and fails there with a bare `Guardian provenance mismatch` naming a process the operator
|
|
1007
|
+
* cannot see in `status`.
|
|
1008
|
+
* @param {Pick<ManagedProcessStatus, "heldProcessGroup" | "pid" | "state">} status - Process status record.
|
|
1009
|
+
* @returns {boolean} True when the child is conclusively gone.
|
|
1010
|
+
*/
|
|
1011
|
+
export function childRecordIsGone(status) {
|
|
1012
|
+
if (status.heldProcessGroup) return false
|
|
1013
|
+
|
|
1014
|
+
return (status.state === "stopped" || status.state === "failed") && status.pid === undefined
|
|
1015
|
+
}
|