rollbridge 0.1.73 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +68 -2
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/daemon.js +411 -78
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +68 -23
- package/src/health.js +99 -8
- package/src/managed-process.js +198 -45
- package/src/process-guardian.js +212 -37
- package/src/process-memory.js +58 -23
- package/src/process-order.js +26 -0
- package/src/release-group.js +438 -45
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +119 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -24
- package/test/owner-recovery.test.js +125 -21
- package/test/owner-replacement.test.js +175 -14
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +408 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/src/release-group.js
CHANGED
|
@@ -2,17 +2,29 @@
|
|
|
2
2
|
|
|
3
3
|
import {spawn} from "node:child_process"
|
|
4
4
|
import {EventEmitter} from "node:events"
|
|
5
|
+
import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
|
|
5
6
|
import ManagedProcess from "./managed-process.js"
|
|
6
7
|
import {findAvailablePort} from "./port-allocator.js"
|
|
8
|
+
import {processStartOrder} from "./process-order.js"
|
|
9
|
+
import {sampleProcessGroup} from "./process-memory.js"
|
|
7
10
|
import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
|
|
8
|
-
import {waitForHealth} from "./health.js"
|
|
11
|
+
import {waitForHealth, waitForListeningPort} from "./health.js"
|
|
12
|
+
|
|
13
|
+
const DEPENDENCY_READY_INTERVAL_MS = 250
|
|
14
|
+
// Shortest gap between two liveness observations that can prove a draining group's CPU is
|
|
15
|
+
// flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
|
|
16
|
+
// with live runner children doing work, so flatness is only evidence below this window.
|
|
17
|
+
const DRAIN_SAMPLE_WINDOW_MS = 20000
|
|
9
18
|
|
|
10
19
|
/**
|
|
11
20
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
12
21
|
* @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
|
|
22
|
+
* @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
|
|
23
|
+
* @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
|
|
13
24
|
* @typedef {{http: number, websocket: number}} ReleaseConnections
|
|
14
|
-
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
15
|
-
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
|
|
25
|
+
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
26
|
+
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean, shouldStop?: () => boolean}} BuildProcessOptions
|
|
27
|
+
* @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
|
|
16
28
|
*/
|
|
17
29
|
|
|
18
30
|
|
|
@@ -57,6 +69,14 @@ function envId(id) {
|
|
|
57
69
|
return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
|
|
58
70
|
}
|
|
59
71
|
|
|
72
|
+
/**
|
|
73
|
+
* @param {number | undefined} startedAtMs - When the measured phase began.
|
|
74
|
+
* @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
|
|
75
|
+
*/
|
|
76
|
+
function elapsedSince(startedAtMs) {
|
|
77
|
+
return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
|
|
78
|
+
}
|
|
79
|
+
|
|
60
80
|
/**
|
|
61
81
|
* @param {import("./config.js").ProcessConfig} processConfig - Process config.
|
|
62
82
|
* @param {number} index - Zero-based replica index.
|
|
@@ -77,9 +97,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
77
97
|
* @param {string | undefined} args.revision - Revision.
|
|
78
98
|
* @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
|
|
79
99
|
* @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
|
|
100
|
+
* @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
|
|
80
101
|
* @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
|
|
81
102
|
*/
|
|
82
|
-
constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
|
|
103
|
+
constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
|
|
83
104
|
super()
|
|
84
105
|
|
|
85
106
|
this.config = config
|
|
@@ -101,22 +122,36 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
101
122
|
this.servicePorts = servicePorts
|
|
102
123
|
this.shouldStart = shouldStart
|
|
103
124
|
this.processFactory = processFactory
|
|
125
|
+
this.procPath = procPath
|
|
104
126
|
this.portsAllocated = false
|
|
105
127
|
this.drainStartedAt = /** @type {string | undefined} */ (undefined)
|
|
128
|
+
this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
|
|
129
|
+
this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
|
|
106
130
|
this.activatedAt = /** @type {string | undefined} */ (undefined)
|
|
107
131
|
this.stoppedAt = /** @type {string | undefined} */ (undefined)
|
|
108
132
|
this.retirementError = /** @type {string | undefined} */ (undefined)
|
|
109
133
|
this.preserveConfigOnRetirement = false
|
|
134
|
+
this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
|
|
135
|
+
this.readyProcessIds = /** @type {Set<string>} */ (new Set())
|
|
136
|
+
this.startedProcessIds = /** @type {Set<string>} */ (new Set())
|
|
137
|
+
this.startupDependencyWaitMs = 0
|
|
138
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
|
|
139
|
+
this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
|
|
110
140
|
}
|
|
111
141
|
|
|
112
142
|
/** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
|
|
113
143
|
async start() {
|
|
114
144
|
this.state = "starting"
|
|
145
|
+
this.startupStartedAtMs = Date.now()
|
|
115
146
|
|
|
116
147
|
try {
|
|
117
148
|
await this.allocatePorts()
|
|
118
149
|
|
|
119
150
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
151
|
+
await this.waitForConfiguredDependencies(processConfig)
|
|
152
|
+
/** @type {string[]} */
|
|
153
|
+
const instanceIds = []
|
|
154
|
+
|
|
120
155
|
for (let index = 0; index < processConfig.replicas; index += 1) {
|
|
121
156
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
122
157
|
|
|
@@ -127,17 +162,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
127
162
|
if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
|
|
128
163
|
if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
|
|
129
164
|
await processInstance.start("deploy")
|
|
165
|
+
this.processLaunchAtMs.set(instanceId, Date.now())
|
|
166
|
+
this.startedProcessIds.add(instanceId)
|
|
167
|
+
instanceIds.push(instanceId)
|
|
130
168
|
}
|
|
131
169
|
|
|
132
170
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
133
|
-
await
|
|
134
|
-
health: processConfig.health,
|
|
135
|
-
host: this.config.proxy.upstreamHost,
|
|
136
|
-
port: this.ports[processConfig.id]
|
|
137
|
-
})
|
|
138
|
-
|
|
139
|
-
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
171
|
+
await this.waitForProxiedHealth(processConfig)
|
|
140
172
|
}
|
|
173
|
+
|
|
174
|
+
for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
|
|
175
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
141
176
|
}
|
|
142
177
|
} catch (error) {
|
|
143
178
|
this.state = "failed"
|
|
@@ -151,9 +186,9 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
151
186
|
/**
|
|
152
187
|
* Reconstructs this release around processes still owned by the durable guardian.
|
|
153
188
|
* @param {ReleaseStatus} snapshot - Persisted release snapshot.
|
|
154
|
-
* @param {{
|
|
189
|
+
* @param {{synchronizeLifecycleRole?: boolean}} [options] - Guardian role synchronization options.
|
|
155
190
|
*/
|
|
156
|
-
async restore(snapshot, {
|
|
191
|
+
async restore(snapshot, {synchronizeLifecycleRole = true} = {}) {
|
|
157
192
|
if (!snapshot || snapshot.releaseId !== this.releaseId || snapshot.releasePath !== this.releasePath || snapshot.revision !== this.revision) {
|
|
158
193
|
throw new Error(`Persisted release identity mismatch for ${this.releaseId}`)
|
|
159
194
|
}
|
|
@@ -214,12 +249,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
214
249
|
this.processes.set(processStatus.id, processInstance)
|
|
215
250
|
if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(processStatus.id)
|
|
216
251
|
if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(processStatus.id)
|
|
217
|
-
if ("recover" in processInstance && typeof processInstance.recover === "function")
|
|
218
|
-
const adopt = adoptCommittedGeneration && processStatus.state === "stopped" && processStatus.pid && processStatus.processStartTime && configDigest
|
|
219
|
-
? {pid: processStatus.pid, processStartTime: processStatus.processStartTime, configDigest}
|
|
220
|
-
: undefined
|
|
221
|
-
await processInstance.recover(adopt ? {adopt} : undefined)
|
|
222
|
-
}
|
|
252
|
+
if ("recover" in processInstance && typeof processInstance.recover === "function") await processInstance.recover()
|
|
223
253
|
}
|
|
224
254
|
if (synchronizeLifecycleRole) await this.synchronizeLifecycleRoles()
|
|
225
255
|
}
|
|
@@ -262,16 +292,132 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
262
292
|
}
|
|
263
293
|
|
|
264
294
|
/**
|
|
265
|
-
*
|
|
295
|
+
* Waits for the ports a process declares it depends on, on a budget that is not the
|
|
296
|
+
* health budget. The dependency leg of a start is not app boot, and charging it to the
|
|
297
|
+
* health deadline measures the dependency instead of the app.
|
|
298
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
|
|
299
|
+
* @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
|
|
300
|
+
*/
|
|
301
|
+
async waitForConfiguredDependencies(processConfig) {
|
|
302
|
+
if (processConfig.dependsOn.length === 0) return
|
|
303
|
+
|
|
304
|
+
for (const dependencyId of processConfig.dependsOn) {
|
|
305
|
+
const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
|
|
306
|
+
|
|
307
|
+
if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
|
|
308
|
+
|
|
309
|
+
this.logger("release process dependency wait", {
|
|
310
|
+
dependencyPort: port,
|
|
311
|
+
dependencyProcessId: dependencyId,
|
|
312
|
+
processId: processConfig.id,
|
|
313
|
+
releaseId: this.releaseId,
|
|
314
|
+
timeoutMs: processConfig.dependencyTimeoutMs
|
|
315
|
+
})
|
|
316
|
+
|
|
317
|
+
const waitStartedAtMs = Date.now()
|
|
318
|
+
|
|
319
|
+
try {
|
|
320
|
+
const {attempts, elapsedMs} = await waitForListeningPort({
|
|
321
|
+
host: this.config.proxy.upstreamHost,
|
|
322
|
+
intervalMs: DEPENDENCY_READY_INTERVAL_MS,
|
|
323
|
+
port,
|
|
324
|
+
timeoutMs: processConfig.dependencyTimeoutMs
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
this.startupDependencyWaitMs += elapsedMs
|
|
328
|
+
this.logger("release process dependency ready", {
|
|
329
|
+
attempts,
|
|
330
|
+
dependencyPort: port,
|
|
331
|
+
dependencyProcessId: dependencyId,
|
|
332
|
+
elapsedMs,
|
|
333
|
+
processId: processConfig.id,
|
|
334
|
+
releaseId: this.releaseId
|
|
335
|
+
})
|
|
336
|
+
} catch (error) {
|
|
337
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
338
|
+
deadlineMs: processConfig.dependencyTimeoutMs,
|
|
339
|
+
dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
|
|
340
|
+
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
341
|
+
failedProcessId: processConfig.id,
|
|
342
|
+
probes: undefined,
|
|
343
|
+
stage: "dependency"
|
|
344
|
+
})
|
|
345
|
+
throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Health checks the proxied process on its own boot budget and records what the boot cost.
|
|
354
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
|
|
355
|
+
* @returns {Promise<void>} Resolves once the process answers its health path.
|
|
356
|
+
*/
|
|
357
|
+
async waitForProxiedHealth(processConfig) {
|
|
358
|
+
const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
|
|
359
|
+
const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
|
|
360
|
+
|
|
361
|
+
try {
|
|
362
|
+
const {elapsedMs} = await waitForHealth({
|
|
363
|
+
health,
|
|
364
|
+
host: this.config.proxy.upstreamHost,
|
|
365
|
+
port: /** @type {number} */ (this.ports[processConfig.id]),
|
|
366
|
+
processId: processConfig.id
|
|
367
|
+
})
|
|
368
|
+
|
|
369
|
+
this.logger("release candidate health ready", {
|
|
370
|
+
bootMs: Date.now() - launchedAtMs,
|
|
371
|
+
deadlineMs: health.timeoutMs,
|
|
372
|
+
healthGateMs: elapsedMs,
|
|
373
|
+
path: health.path,
|
|
374
|
+
port: this.ports[processConfig.id],
|
|
375
|
+
processId: processConfig.id,
|
|
376
|
+
releaseId: this.releaseId,
|
|
377
|
+
waitedForDependenciesMs: this.startupDependencyWaitMs
|
|
378
|
+
})
|
|
379
|
+
} catch (error) {
|
|
380
|
+
const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
|
|
381
|
+
|
|
382
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
383
|
+
deadlineMs: health.timeoutMs,
|
|
384
|
+
dependencyWaitMs: this.startupDependencyWaitMs,
|
|
385
|
+
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
386
|
+
failedProcessId: processConfig.id,
|
|
387
|
+
probes: healthCheck?.attempts,
|
|
388
|
+
stage: "health"
|
|
389
|
+
})
|
|
390
|
+
throw error
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
|
|
266
396
|
* @param {Error | string} error - Startup failure.
|
|
267
397
|
* @param {{phase: string}} options - Diagnostic phase.
|
|
268
398
|
* @returns {void}
|
|
269
399
|
*/
|
|
270
400
|
logStartupFailure(error, {phase}) {
|
|
401
|
+
const diagnostics = this.startupFailureDiagnostics
|
|
402
|
+
const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
|
|
403
|
+
const lastLog = failedLogs?.at(-1)
|
|
404
|
+
const readyProcesses = [...this.readyProcessIds]
|
|
405
|
+
|
|
271
406
|
this.logger("release startup failed", {
|
|
407
|
+
deadlineMs: diagnostics?.deadlineMs ?? null,
|
|
408
|
+
dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
|
|
409
|
+
elapsedMs: diagnostics?.elapsedMs ?? null,
|
|
272
410
|
error: error instanceof Error ? error.message : error,
|
|
411
|
+
failedProcessId: diagnostics?.failedProcessId ?? null,
|
|
412
|
+
lastOutput: lastLog?.line ?? null,
|
|
413
|
+
lastOutputAt: lastLog?.at ?? null,
|
|
273
414
|
phase,
|
|
274
|
-
|
|
415
|
+
probes: diagnostics?.probes ?? null,
|
|
416
|
+
readyProcesses,
|
|
417
|
+
releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
418
|
+
releaseId: this.releaseId,
|
|
419
|
+
stage: diagnostics?.stage ?? null,
|
|
420
|
+
unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
|
|
275
421
|
})
|
|
276
422
|
|
|
277
423
|
for (const processInstance of this.processes.values()) {
|
|
@@ -296,18 +442,14 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
296
442
|
* @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
|
|
297
443
|
*/
|
|
298
444
|
releaseProcessStartOrder() {
|
|
299
|
-
|
|
300
|
-
const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
|
|
301
|
-
const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
|
|
302
|
-
const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
|
|
303
|
-
|
|
304
|
-
return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
|
|
445
|
+
return processStartOrder(this.config.processes)
|
|
305
446
|
}
|
|
306
447
|
|
|
307
448
|
/** @returns {void} Marks this release active. */
|
|
308
449
|
activate() {
|
|
309
450
|
this.state = "active"
|
|
310
451
|
this.activatedAt = new Date().toISOString()
|
|
452
|
+
this.clearDrainLivenessSamples()
|
|
311
453
|
}
|
|
312
454
|
|
|
313
455
|
/** Runs the configured release-generation activation acknowledgement, if any. */
|
|
@@ -362,6 +504,16 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
362
504
|
this.drainStartedAt = undefined
|
|
363
505
|
this.retirementError = undefined
|
|
364
506
|
this.stoppedAt = undefined
|
|
507
|
+
this.clearDrainLivenessSamples()
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/** Stops only the proxied processes when a failed compensation cannot restore jobs. */
|
|
511
|
+
async stopProxyTraffic() {
|
|
512
|
+
const proxiedIds = new Set(this.config.processes.filter((processConfig) => processConfig.policy === "proxied").map((processConfig) => processConfig.id))
|
|
513
|
+
|
|
514
|
+
await Promise.allSettled([...this.processes.entries()]
|
|
515
|
+
.filter(([id]) => proxiedIds.has(id.replace(/#\d+$/, "")))
|
|
516
|
+
.map(([, processInstance]) => processInstance.stop()))
|
|
365
517
|
}
|
|
366
518
|
|
|
367
519
|
/**
|
|
@@ -372,24 +524,25 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
372
524
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
373
525
|
this.assertCommittedGenerationRecoverable()
|
|
374
526
|
this.state = "starting"
|
|
527
|
+
this.startupStartedAtMs = Date.now()
|
|
375
528
|
try {
|
|
376
529
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
530
|
+
await this.waitForConfiguredDependencies(processConfig)
|
|
377
531
|
const instances = this.getProcesses(processConfig.id)
|
|
378
532
|
|
|
379
533
|
if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
|
|
380
|
-
for (const {process} of instances) {
|
|
534
|
+
for (const {id, process} of instances) {
|
|
381
535
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
382
536
|
await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
|
|
537
|
+
this.processLaunchAtMs.set(id, Date.now())
|
|
538
|
+
this.startedProcessIds.add(id)
|
|
383
539
|
}
|
|
384
540
|
|
|
385
541
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
386
|
-
await
|
|
387
|
-
health: processConfig.health,
|
|
388
|
-
host: this.config.proxy.upstreamHost,
|
|
389
|
-
port: this.ports[processConfig.id]
|
|
390
|
-
})
|
|
542
|
+
await this.waitForProxiedHealth(processConfig)
|
|
391
543
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
392
544
|
}
|
|
545
|
+
for (const {id} of instances) this.readyProcessIds.add(id)
|
|
393
546
|
}
|
|
394
547
|
} catch (error) {
|
|
395
548
|
await this.abortCommittedGenerationRestart()
|
|
@@ -481,6 +634,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
481
634
|
...this.baseEnvironment(processConfig, {count, index}),
|
|
482
635
|
...renderedEnv
|
|
483
636
|
}
|
|
637
|
+
const dependencyRetired = () => processConfig.dependsOn.some((dependencyId) => {
|
|
638
|
+
const dependency = this.processes.get(dependencyId)
|
|
639
|
+
return dependency?.lifecycleRole === "retired" || dependency?.status().lifecycleRole === "retired"
|
|
640
|
+
})
|
|
484
641
|
|
|
485
642
|
return {
|
|
486
643
|
command: renderTemplate(processConfig.command, context),
|
|
@@ -493,7 +650,15 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
493
650
|
outputLines: processConfig.outputLines,
|
|
494
651
|
restart: processConfig.restart,
|
|
495
652
|
restartDelayMs: processConfig.restartDelayMs,
|
|
496
|
-
shouldRestart: options.shouldRestart || (() =>
|
|
653
|
+
shouldRestart: options.shouldRestart || (() => {
|
|
654
|
+
if (this.state !== "active" && this.state !== "starting") return false
|
|
655
|
+
|
|
656
|
+
// A companion whose same-generation service dependency has retired can never
|
|
657
|
+
// satisfy its startup gate again. Treat that dependency lifecycle as terminal
|
|
658
|
+
// for this process so an exit does not become an endless supervisor restart loop.
|
|
659
|
+
return !dependencyRetired()
|
|
660
|
+
}),
|
|
661
|
+
shouldStop: options.shouldStop || dependencyRetired,
|
|
497
662
|
stopSignal: processConfig.stopSignal,
|
|
498
663
|
stopTimeoutMs: processConfig.gracefulStopMs
|
|
499
664
|
}
|
|
@@ -713,10 +878,21 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
713
878
|
// their lifecycle drain runs as soon as the release is retired — in parallel with the
|
|
714
879
|
// connection drain, not held until after it. The rest stop once connections have closed.
|
|
715
880
|
const entries = [...this.processes.entries()]
|
|
716
|
-
const
|
|
717
|
-
|
|
881
|
+
const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
|
|
882
|
+
id,
|
|
883
|
+
promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
|
|
884
|
+
}))
|
|
718
885
|
const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
719
886
|
const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
887
|
+
// Bracket the retirement wait with a pair of observations: this one, and the one taken when
|
|
888
|
+
// every stop has returned. The gap between them is the wait the transition already performs
|
|
889
|
+
// (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
|
|
890
|
+
// itself rather than from a sampler running alongside it.
|
|
891
|
+
const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
|
|
892
|
+
? this.takeLivenessSample(Date.now())
|
|
893
|
+
: undefined
|
|
894
|
+
|
|
895
|
+
if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
|
|
720
896
|
|
|
721
897
|
if (this.connectionCount > 0) {
|
|
722
898
|
await new Promise((resolve) => {
|
|
@@ -735,17 +911,40 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
735
911
|
}
|
|
736
912
|
|
|
737
913
|
if (this.ownerHandoffPaused) {
|
|
738
|
-
void Promise.allSettled(nonBlockingStops)
|
|
914
|
+
void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
|
|
739
915
|
return
|
|
740
916
|
}
|
|
741
917
|
|
|
742
|
-
|
|
743
|
-
await Promise.allSettled(
|
|
744
|
-
await Promise.allSettled(
|
|
918
|
+
const retirementTimeoutMs = config.proxy.nonBlockingDrainTimeoutMs
|
|
919
|
+
await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
|
|
920
|
+
const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
|
|
921
|
+
nonBlockingResults.forEach((result, index) => {
|
|
922
|
+
if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
|
|
923
|
+
const processInstance = this.processes.get(nonBlockingStops[index].id)
|
|
924
|
+
this.logger("release non-blocking process still draining", {
|
|
925
|
+
pid: processInstance?.status().pid ?? null,
|
|
926
|
+
processId: nonBlockingStops[index].id,
|
|
927
|
+
releaseId: this.releaseId,
|
|
928
|
+
timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
|
|
929
|
+
})
|
|
930
|
+
}
|
|
931
|
+
})
|
|
932
|
+
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
|
|
933
|
+
|
|
934
|
+
// Last act of the retirement transition, before the release is recorded stopped: whatever
|
|
935
|
+
// is still alive here is the residue, and the bracket pair above can now say whether it is
|
|
936
|
+
// wedged. A generation that stops with a live process group is the state that made
|
|
937
|
+
// operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
|
|
938
|
+
// it does not end silently with the process still holding the release.
|
|
939
|
+
if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
|
|
940
|
+
|
|
745
941
|
this.state = "stopped"
|
|
746
942
|
this.stoppedAt = new Date().toISOString()
|
|
747
943
|
this.releasePortReservations()
|
|
748
944
|
this.abandonLocalConnections()
|
|
945
|
+
// The bracket pair is deliberately kept: a generation that stopped with a live process
|
|
946
|
+
// group stays observable, and the deploy path needs the older observation to prove the
|
|
947
|
+
// residue is still flat rather than starting a fresh window from zero.
|
|
749
948
|
}
|
|
750
949
|
|
|
751
950
|
/**
|
|
@@ -765,12 +964,18 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
765
964
|
this.drainStartedAt = new Date().toISOString()
|
|
766
965
|
}
|
|
767
966
|
|
|
967
|
+
// Refresh the live process definitions with the current retirement policy,
|
|
968
|
+
// but retain this generation's exact config identity for owner recovery.
|
|
969
|
+
// Replacing the stored generation config here would make a still-running
|
|
970
|
+
// singleton from the previous release fail the guardian provenance fence.
|
|
971
|
+
const generationConfig = this.config
|
|
972
|
+
|
|
768
973
|
this.refreshProcessDefinitions(config)
|
|
974
|
+
this.config = generationConfig
|
|
769
975
|
const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
|
|
770
976
|
const results = await Promise.allSettled([...this.processes.entries()]
|
|
771
977
|
.filter(([id]) => generationIds.has(id))
|
|
772
978
|
.map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
|
|
773
|
-
|
|
774
979
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
775
980
|
|
|
776
981
|
if (errors.length > 0) {
|
|
@@ -781,15 +986,44 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
781
986
|
this.retirementError = undefined
|
|
782
987
|
}
|
|
783
988
|
|
|
784
|
-
/**
|
|
785
|
-
|
|
786
|
-
|
|
989
|
+
/**
|
|
990
|
+
* Stops every process owned by the generation.
|
|
991
|
+
* @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
|
|
992
|
+
* supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
|
|
993
|
+
* is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
|
|
994
|
+
* `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
|
|
995
|
+
* the wait holds the mutation fence and every later deploy is refused (STOE-2015).
|
|
996
|
+
* @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
|
|
997
|
+
*/
|
|
998
|
+
async stop(options = {}) {
|
|
999
|
+
const {retirementTimeoutMs} = options
|
|
1000
|
+
/** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
|
|
1001
|
+
const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
|
|
1002
|
+
const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
|
|
1003
|
+
|
|
1004
|
+
return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
|
|
1005
|
+
})
|
|
1006
|
+
|
|
1007
|
+
await Promise.allSettled(stopTasks.map(({promise}) => promise))
|
|
1008
|
+
|
|
1009
|
+
// A stop that stopped waiting leaves the process running. Name it, so the operator sees the
|
|
1010
|
+
// residue instead of a command that produced nothing.
|
|
1011
|
+
for (const {bounded, id, instance} of stopTasks) {
|
|
1012
|
+
if (!bounded || !instance.retirementTimedOut) continue
|
|
1013
|
+
|
|
1014
|
+
this.logger("release stop gave up waiting for a retiring process", {
|
|
1015
|
+
pid: instance.status().pid ?? null,
|
|
1016
|
+
processId: id,
|
|
1017
|
+
releaseId: this.releaseId,
|
|
1018
|
+
timeoutMs: retirementTimeoutMs
|
|
1019
|
+
})
|
|
1020
|
+
}
|
|
787
1021
|
|
|
788
|
-
await Promise.allSettled(stopTasks)
|
|
789
1022
|
this.state = "stopped"
|
|
790
1023
|
this.stoppedAt = new Date().toISOString()
|
|
791
1024
|
this.releasePortReservations()
|
|
792
1025
|
this.abandonLocalConnections()
|
|
1026
|
+
this.clearDrainLivenessSamples()
|
|
793
1027
|
}
|
|
794
1028
|
|
|
795
1029
|
/** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
|
|
@@ -797,12 +1031,171 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
797
1031
|
await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
|
|
798
1032
|
}
|
|
799
1033
|
|
|
1034
|
+
/**
|
|
1035
|
+
* Records one liveness observation for a draining generation and classifies its drain.
|
|
1036
|
+
*
|
|
1037
|
+
* Two observations are needed to prove CPU flatness, and reading a process group means
|
|
1038
|
+
* scanning procfs, so sampling is bounded three ways: a generation that still owns live
|
|
1039
|
+
* connections is classified without touching procfs at all (connections alone settle it), a
|
|
1040
|
+
* new observation is taken only once the newest one is already a full window behind the
|
|
1041
|
+
* present read, and every other read reuses the newest observation. A group that is not
|
|
1042
|
+
* draining reports no drain.
|
|
1043
|
+
* @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
|
|
1044
|
+
*/
|
|
1045
|
+
drainStatus() {
|
|
1046
|
+
if (this.state !== "draining") return undefined
|
|
1047
|
+
|
|
1048
|
+
return this.observeRetirement({nowMs: Date.now()}).classification
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
/**
|
|
1052
|
+
* Records the newest liveness observation of this generation and classifies its retirement.
|
|
1053
|
+
* @param {object} options - Observation options.
|
|
1054
|
+
* @param {number} options.nowMs - Read time.
|
|
1055
|
+
* @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
|
|
1056
|
+
* @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
|
|
1057
|
+
*/
|
|
1058
|
+
observeRetirement({nowMs, previousSample}) {
|
|
1059
|
+
const drainBudgetMs = this.config.proxy.drainTimeoutMs
|
|
1060
|
+
const drainElapsedMs = this.drainStartedAt === undefined ? 0 : nowMs - Date.parse(this.drainStartedAt)
|
|
1061
|
+
|
|
1062
|
+
if (previousSample === undefined && this.connectionCount > 0) {
|
|
1063
|
+
const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
|
|
1064
|
+
|
|
1065
|
+
return {
|
|
1066
|
+
classification: classifyDrainProgress({
|
|
1067
|
+
connectionCount: this.connectionCount,
|
|
1068
|
+
currentSample,
|
|
1069
|
+
drainBudgetMs,
|
|
1070
|
+
drainElapsedMs,
|
|
1071
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1072
|
+
previousSample: undefined
|
|
1073
|
+
}),
|
|
1074
|
+
currentSample
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
const newest = this.drainNewestSample
|
|
1079
|
+
const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
|
|
1080
|
+
? this.takeLivenessSample(nowMs)
|
|
1081
|
+
: newest
|
|
1082
|
+
const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
|
|
1083
|
+
|
|
1084
|
+
if (currentSample !== newest) {
|
|
1085
|
+
this.drainPriorSample = newest
|
|
1086
|
+
this.drainNewestSample = currentSample
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
return {
|
|
1090
|
+
classification: classifyDrainProgress({
|
|
1091
|
+
connectionCount: this.connectionCount,
|
|
1092
|
+
currentSample,
|
|
1093
|
+
drainBudgetMs,
|
|
1094
|
+
drainElapsedMs,
|
|
1095
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1096
|
+
previousSample: priorSample
|
|
1097
|
+
}),
|
|
1098
|
+
currentSample
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
/**
|
|
1103
|
+
* Applies the wedge escalation to this generation: terminates the processes a proven wedge
|
|
1104
|
+
* proves are holding nothing, and names the ones it refused to terminate.
|
|
1105
|
+
*
|
|
1106
|
+
* Both halves are required. The card this path was written for (STOE-1687) is about a
|
|
1107
|
+
* wedged worker holding a release open for four hours while deploys stack behind it; the
|
|
1108
|
+
* boundary on the same card (STOE-1077) is that no process still holding live work may ever
|
|
1109
|
+
* be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
|
|
1110
|
+
* with the pids and process groups it is holding, which is the information an operator needs
|
|
1111
|
+
* in place of guessing.
|
|
1112
|
+
* @param {object} options - Escalation context.
|
|
1113
|
+
* @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
|
|
1114
|
+
* @param {string} options.reason - Named context recorded with the decision.
|
|
1115
|
+
* @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
|
|
1116
|
+
*/
|
|
1117
|
+
async escalateWedgedGeneration({previousSample, reason}) {
|
|
1118
|
+
const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
|
|
1119
|
+
const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
|
|
1120
|
+
const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
|
|
1121
|
+
|
|
1122
|
+
if (decision.decision === "report") {
|
|
1123
|
+
if (classification.status === "wedged") {
|
|
1124
|
+
this.logger("release wedged generation left alive", {
|
|
1125
|
+
escalation: "refused",
|
|
1126
|
+
heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
|
|
1127
|
+
pids: liveObservations.map((process) => process.pid ?? null),
|
|
1128
|
+
processIds: liveObservations.map((process) => process.id),
|
|
1129
|
+
reason: decision.reason,
|
|
1130
|
+
releaseId: this.releaseId,
|
|
1131
|
+
trigger: reason
|
|
1132
|
+
})
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
return []
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
const escalated = []
|
|
1139
|
+
|
|
1140
|
+
for (const target of decision.targets) {
|
|
1141
|
+
const processInstance = this.processes.get(target.processId)
|
|
1142
|
+
|
|
1143
|
+
if (!processInstance) continue
|
|
1144
|
+
if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
if (escalated.length > 0) {
|
|
1148
|
+
this.logger("release wedged generation escalated", {
|
|
1149
|
+
escalation: "SIGKILL",
|
|
1150
|
+
processIds: escalated,
|
|
1151
|
+
reason: decision.reason,
|
|
1152
|
+
releaseId: this.releaseId,
|
|
1153
|
+
trigger: reason
|
|
1154
|
+
})
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
return escalated
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
/**
|
|
1161
|
+
* @param {number} nowMs - Observation time.
|
|
1162
|
+
* @returns {DrainLivenessSample} One liveness observation of every process in the generation.
|
|
1163
|
+
*/
|
|
1164
|
+
takeLivenessSample(nowMs) {
|
|
1165
|
+
const processes = [...this.processes.entries()].map(([id, processInstance]) => {
|
|
1166
|
+
const processStatus = processInstance.status()
|
|
1167
|
+
const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
|
|
1168
|
+
// A group that outlived its tracked process is still observed by group id: it holds the
|
|
1169
|
+
// generation's work even though no rollbridge record can signal it any more.
|
|
1170
|
+
const pgid = processStatus.pid ?? heldProcessGroupPgid
|
|
1171
|
+
const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
|
|
1172
|
+
|
|
1173
|
+
return {
|
|
1174
|
+
cpuMs: sample?.cpuMs,
|
|
1175
|
+
heldProcessGroupPgid,
|
|
1176
|
+
id,
|
|
1177
|
+
liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
|
|
1178
|
+
pid: processStatus.pid,
|
|
1179
|
+
retirementTimedOut: processInstance.retirementTimedOut === true
|
|
1180
|
+
}
|
|
1181
|
+
})
|
|
1182
|
+
|
|
1183
|
+
return {atMs: nowMs, processes}
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
/** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
|
|
1187
|
+
clearDrainLivenessSamples() {
|
|
1188
|
+
this.drainNewestSample = undefined
|
|
1189
|
+
this.drainPriorSample = undefined
|
|
1190
|
+
}
|
|
1191
|
+
|
|
800
1192
|
/** @returns {ReleaseStatus} Status payload. */
|
|
801
1193
|
status() {
|
|
802
1194
|
return {
|
|
803
1195
|
activatedAt: this.activatedAt,
|
|
804
1196
|
connectionCount: this.connectionCount,
|
|
805
1197
|
connections: {...this.connections},
|
|
1198
|
+
drain: this.drainStatus(),
|
|
806
1199
|
drainStartedAt: this.drainStartedAt,
|
|
807
1200
|
ports: {...this.ports},
|
|
808
1201
|
processes: [...this.processes.values()].map((processInstance) => processInstance.status()),
|