rollbridge 0.1.72 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +68 -2
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/daemon.js +410 -73
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +67 -8
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +435 -37
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +119 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +125 -21
- package/test/owner-replacement.test.js +175 -14
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +408 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/src/release-group.js
CHANGED
|
@@ -2,17 +2,29 @@
|
|
|
2
2
|
|
|
3
3
|
import {spawn} from "node:child_process"
|
|
4
4
|
import {EventEmitter} from "node:events"
|
|
5
|
+
import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
|
|
5
6
|
import ManagedProcess from "./managed-process.js"
|
|
6
7
|
import {findAvailablePort} from "./port-allocator.js"
|
|
8
|
+
import {processStartOrder} from "./process-order.js"
|
|
9
|
+
import {sampleProcessGroup} from "./process-memory.js"
|
|
7
10
|
import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
|
|
8
|
-
import {waitForHealth} from "./health.js"
|
|
11
|
+
import {waitForHealth, waitForListeningPort} from "./health.js"
|
|
12
|
+
|
|
13
|
+
const DEPENDENCY_READY_INTERVAL_MS = 250
|
|
14
|
+
// Shortest gap between two liveness observations that can prove a draining group's CPU is
|
|
15
|
+
// flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
|
|
16
|
+
// with live runner children doing work, so flatness is only evidence below this window.
|
|
17
|
+
const DRAIN_SAMPLE_WINDOW_MS = 20000
|
|
9
18
|
|
|
10
19
|
/**
|
|
11
20
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
12
21
|
* @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
|
|
22
|
+
* @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
|
|
23
|
+
* @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
|
|
13
24
|
* @typedef {{http: number, websocket: number}} ReleaseConnections
|
|
14
|
-
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
15
|
-
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
|
|
25
|
+
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
26
|
+
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean, shouldStop?: () => boolean}} BuildProcessOptions
|
|
27
|
+
* @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
|
|
16
28
|
*/
|
|
17
29
|
|
|
18
30
|
|
|
@@ -57,6 +69,14 @@ function envId(id) {
|
|
|
57
69
|
return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
|
|
58
70
|
}
|
|
59
71
|
|
|
72
|
+
/**
|
|
73
|
+
* @param {number | undefined} startedAtMs - When the measured phase began.
|
|
74
|
+
* @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
|
|
75
|
+
*/
|
|
76
|
+
function elapsedSince(startedAtMs) {
|
|
77
|
+
return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
|
|
78
|
+
}
|
|
79
|
+
|
|
60
80
|
/**
|
|
61
81
|
* @param {import("./config.js").ProcessConfig} processConfig - Process config.
|
|
62
82
|
* @param {number} index - Zero-based replica index.
|
|
@@ -77,9 +97,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
77
97
|
* @param {string | undefined} args.revision - Revision.
|
|
78
98
|
* @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
|
|
79
99
|
* @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
|
|
100
|
+
* @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
|
|
80
101
|
* @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
|
|
81
102
|
*/
|
|
82
|
-
constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
|
|
103
|
+
constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
|
|
83
104
|
super()
|
|
84
105
|
|
|
85
106
|
this.config = config
|
|
@@ -101,22 +122,36 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
101
122
|
this.servicePorts = servicePorts
|
|
102
123
|
this.shouldStart = shouldStart
|
|
103
124
|
this.processFactory = processFactory
|
|
125
|
+
this.procPath = procPath
|
|
104
126
|
this.portsAllocated = false
|
|
105
127
|
this.drainStartedAt = /** @type {string | undefined} */ (undefined)
|
|
128
|
+
this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
|
|
129
|
+
this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
|
|
106
130
|
this.activatedAt = /** @type {string | undefined} */ (undefined)
|
|
107
131
|
this.stoppedAt = /** @type {string | undefined} */ (undefined)
|
|
108
132
|
this.retirementError = /** @type {string | undefined} */ (undefined)
|
|
109
133
|
this.preserveConfigOnRetirement = false
|
|
134
|
+
this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
|
|
135
|
+
this.readyProcessIds = /** @type {Set<string>} */ (new Set())
|
|
136
|
+
this.startedProcessIds = /** @type {Set<string>} */ (new Set())
|
|
137
|
+
this.startupDependencyWaitMs = 0
|
|
138
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
|
|
139
|
+
this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
|
|
110
140
|
}
|
|
111
141
|
|
|
112
142
|
/** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
|
|
113
143
|
async start() {
|
|
114
144
|
this.state = "starting"
|
|
145
|
+
this.startupStartedAtMs = Date.now()
|
|
115
146
|
|
|
116
147
|
try {
|
|
117
148
|
await this.allocatePorts()
|
|
118
149
|
|
|
119
150
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
151
|
+
await this.waitForConfiguredDependencies(processConfig)
|
|
152
|
+
/** @type {string[]} */
|
|
153
|
+
const instanceIds = []
|
|
154
|
+
|
|
120
155
|
for (let index = 0; index < processConfig.replicas; index += 1) {
|
|
121
156
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
122
157
|
|
|
@@ -127,17 +162,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
127
162
|
if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
|
|
128
163
|
if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
|
|
129
164
|
await processInstance.start("deploy")
|
|
165
|
+
this.processLaunchAtMs.set(instanceId, Date.now())
|
|
166
|
+
this.startedProcessIds.add(instanceId)
|
|
167
|
+
instanceIds.push(instanceId)
|
|
130
168
|
}
|
|
131
169
|
|
|
132
170
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
133
|
-
await
|
|
134
|
-
health: processConfig.health,
|
|
135
|
-
host: this.config.proxy.upstreamHost,
|
|
136
|
-
port: this.ports[processConfig.id]
|
|
137
|
-
})
|
|
138
|
-
|
|
139
|
-
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
171
|
+
await this.waitForProxiedHealth(processConfig)
|
|
140
172
|
}
|
|
173
|
+
|
|
174
|
+
for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
|
|
175
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
141
176
|
}
|
|
142
177
|
} catch (error) {
|
|
143
178
|
this.state = "failed"
|
|
@@ -257,16 +292,132 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
257
292
|
}
|
|
258
293
|
|
|
259
294
|
/**
|
|
260
|
-
*
|
|
295
|
+
* Waits for the ports a process declares it depends on, on a budget that is not the
|
|
296
|
+
* health budget. The dependency leg of a start is not app boot, and charging it to the
|
|
297
|
+
* health deadline measures the dependency instead of the app.
|
|
298
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
|
|
299
|
+
* @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
|
|
300
|
+
*/
|
|
301
|
+
async waitForConfiguredDependencies(processConfig) {
|
|
302
|
+
if (processConfig.dependsOn.length === 0) return
|
|
303
|
+
|
|
304
|
+
for (const dependencyId of processConfig.dependsOn) {
|
|
305
|
+
const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
|
|
306
|
+
|
|
307
|
+
if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
|
|
308
|
+
|
|
309
|
+
this.logger("release process dependency wait", {
|
|
310
|
+
dependencyPort: port,
|
|
311
|
+
dependencyProcessId: dependencyId,
|
|
312
|
+
processId: processConfig.id,
|
|
313
|
+
releaseId: this.releaseId,
|
|
314
|
+
timeoutMs: processConfig.dependencyTimeoutMs
|
|
315
|
+
})
|
|
316
|
+
|
|
317
|
+
const waitStartedAtMs = Date.now()
|
|
318
|
+
|
|
319
|
+
try {
|
|
320
|
+
const {attempts, elapsedMs} = await waitForListeningPort({
|
|
321
|
+
host: this.config.proxy.upstreamHost,
|
|
322
|
+
intervalMs: DEPENDENCY_READY_INTERVAL_MS,
|
|
323
|
+
port,
|
|
324
|
+
timeoutMs: processConfig.dependencyTimeoutMs
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
this.startupDependencyWaitMs += elapsedMs
|
|
328
|
+
this.logger("release process dependency ready", {
|
|
329
|
+
attempts,
|
|
330
|
+
dependencyPort: port,
|
|
331
|
+
dependencyProcessId: dependencyId,
|
|
332
|
+
elapsedMs,
|
|
333
|
+
processId: processConfig.id,
|
|
334
|
+
releaseId: this.releaseId
|
|
335
|
+
})
|
|
336
|
+
} catch (error) {
|
|
337
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
338
|
+
deadlineMs: processConfig.dependencyTimeoutMs,
|
|
339
|
+
dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
|
|
340
|
+
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
341
|
+
failedProcessId: processConfig.id,
|
|
342
|
+
probes: undefined,
|
|
343
|
+
stage: "dependency"
|
|
344
|
+
})
|
|
345
|
+
throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Health checks the proxied process on its own boot budget and records what the boot cost.
|
|
354
|
+
* @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
|
|
355
|
+
* @returns {Promise<void>} Resolves once the process answers its health path.
|
|
356
|
+
*/
|
|
357
|
+
async waitForProxiedHealth(processConfig) {
|
|
358
|
+
const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
|
|
359
|
+
const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
|
|
360
|
+
|
|
361
|
+
try {
|
|
362
|
+
const {elapsedMs} = await waitForHealth({
|
|
363
|
+
health,
|
|
364
|
+
host: this.config.proxy.upstreamHost,
|
|
365
|
+
port: /** @type {number} */ (this.ports[processConfig.id]),
|
|
366
|
+
processId: processConfig.id
|
|
367
|
+
})
|
|
368
|
+
|
|
369
|
+
this.logger("release candidate health ready", {
|
|
370
|
+
bootMs: Date.now() - launchedAtMs,
|
|
371
|
+
deadlineMs: health.timeoutMs,
|
|
372
|
+
healthGateMs: elapsedMs,
|
|
373
|
+
path: health.path,
|
|
374
|
+
port: this.ports[processConfig.id],
|
|
375
|
+
processId: processConfig.id,
|
|
376
|
+
releaseId: this.releaseId,
|
|
377
|
+
waitedForDependenciesMs: this.startupDependencyWaitMs
|
|
378
|
+
})
|
|
379
|
+
} catch (error) {
|
|
380
|
+
const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
|
|
381
|
+
|
|
382
|
+
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
383
|
+
deadlineMs: health.timeoutMs,
|
|
384
|
+
dependencyWaitMs: this.startupDependencyWaitMs,
|
|
385
|
+
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
386
|
+
failedProcessId: processConfig.id,
|
|
387
|
+
probes: healthCheck?.attempts,
|
|
388
|
+
stage: "health"
|
|
389
|
+
})
|
|
390
|
+
throw error
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
|
|
261
396
|
* @param {Error | string} error - Startup failure.
|
|
262
397
|
* @param {{phase: string}} options - Diagnostic phase.
|
|
263
398
|
* @returns {void}
|
|
264
399
|
*/
|
|
265
400
|
logStartupFailure(error, {phase}) {
|
|
401
|
+
const diagnostics = this.startupFailureDiagnostics
|
|
402
|
+
const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
|
|
403
|
+
const lastLog = failedLogs?.at(-1)
|
|
404
|
+
const readyProcesses = [...this.readyProcessIds]
|
|
405
|
+
|
|
266
406
|
this.logger("release startup failed", {
|
|
407
|
+
deadlineMs: diagnostics?.deadlineMs ?? null,
|
|
408
|
+
dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
|
|
409
|
+
elapsedMs: diagnostics?.elapsedMs ?? null,
|
|
267
410
|
error: error instanceof Error ? error.message : error,
|
|
411
|
+
failedProcessId: diagnostics?.failedProcessId ?? null,
|
|
412
|
+
lastOutput: lastLog?.line ?? null,
|
|
413
|
+
lastOutputAt: lastLog?.at ?? null,
|
|
268
414
|
phase,
|
|
269
|
-
|
|
415
|
+
probes: diagnostics?.probes ?? null,
|
|
416
|
+
readyProcesses,
|
|
417
|
+
releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
418
|
+
releaseId: this.releaseId,
|
|
419
|
+
stage: diagnostics?.stage ?? null,
|
|
420
|
+
unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
|
|
270
421
|
})
|
|
271
422
|
|
|
272
423
|
for (const processInstance of this.processes.values()) {
|
|
@@ -291,18 +442,14 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
291
442
|
* @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
|
|
292
443
|
*/
|
|
293
444
|
releaseProcessStartOrder() {
|
|
294
|
-
|
|
295
|
-
const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
|
|
296
|
-
const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
|
|
297
|
-
const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
|
|
298
|
-
|
|
299
|
-
return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
|
|
445
|
+
return processStartOrder(this.config.processes)
|
|
300
446
|
}
|
|
301
447
|
|
|
302
448
|
/** @returns {void} Marks this release active. */
|
|
303
449
|
activate() {
|
|
304
450
|
this.state = "active"
|
|
305
451
|
this.activatedAt = new Date().toISOString()
|
|
452
|
+
this.clearDrainLivenessSamples()
|
|
306
453
|
}
|
|
307
454
|
|
|
308
455
|
/** Runs the configured release-generation activation acknowledgement, if any. */
|
|
@@ -357,6 +504,16 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
357
504
|
this.drainStartedAt = undefined
|
|
358
505
|
this.retirementError = undefined
|
|
359
506
|
this.stoppedAt = undefined
|
|
507
|
+
this.clearDrainLivenessSamples()
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/** Stops only the proxied processes when a failed compensation cannot restore jobs. */
|
|
511
|
+
async stopProxyTraffic() {
|
|
512
|
+
const proxiedIds = new Set(this.config.processes.filter((processConfig) => processConfig.policy === "proxied").map((processConfig) => processConfig.id))
|
|
513
|
+
|
|
514
|
+
await Promise.allSettled([...this.processes.entries()]
|
|
515
|
+
.filter(([id]) => proxiedIds.has(id.replace(/#\d+$/, "")))
|
|
516
|
+
.map(([, processInstance]) => processInstance.stop()))
|
|
360
517
|
}
|
|
361
518
|
|
|
362
519
|
/**
|
|
@@ -367,24 +524,25 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
367
524
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
368
525
|
this.assertCommittedGenerationRecoverable()
|
|
369
526
|
this.state = "starting"
|
|
527
|
+
this.startupStartedAtMs = Date.now()
|
|
370
528
|
try {
|
|
371
529
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
530
|
+
await this.waitForConfiguredDependencies(processConfig)
|
|
372
531
|
const instances = this.getProcesses(processConfig.id)
|
|
373
532
|
|
|
374
533
|
if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
|
|
375
|
-
for (const {process} of instances) {
|
|
534
|
+
for (const {id, process} of instances) {
|
|
376
535
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
377
536
|
await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
|
|
537
|
+
this.processLaunchAtMs.set(id, Date.now())
|
|
538
|
+
this.startedProcessIds.add(id)
|
|
378
539
|
}
|
|
379
540
|
|
|
380
541
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
381
|
-
await
|
|
382
|
-
health: processConfig.health,
|
|
383
|
-
host: this.config.proxy.upstreamHost,
|
|
384
|
-
port: this.ports[processConfig.id]
|
|
385
|
-
})
|
|
542
|
+
await this.waitForProxiedHealth(processConfig)
|
|
386
543
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
387
544
|
}
|
|
545
|
+
for (const {id} of instances) this.readyProcessIds.add(id)
|
|
388
546
|
}
|
|
389
547
|
} catch (error) {
|
|
390
548
|
await this.abortCommittedGenerationRestart()
|
|
@@ -476,6 +634,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
476
634
|
...this.baseEnvironment(processConfig, {count, index}),
|
|
477
635
|
...renderedEnv
|
|
478
636
|
}
|
|
637
|
+
const dependencyRetired = () => processConfig.dependsOn.some((dependencyId) => {
|
|
638
|
+
const dependency = this.processes.get(dependencyId)
|
|
639
|
+
return dependency?.lifecycleRole === "retired" || dependency?.status().lifecycleRole === "retired"
|
|
640
|
+
})
|
|
479
641
|
|
|
480
642
|
return {
|
|
481
643
|
command: renderTemplate(processConfig.command, context),
|
|
@@ -488,7 +650,15 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
488
650
|
outputLines: processConfig.outputLines,
|
|
489
651
|
restart: processConfig.restart,
|
|
490
652
|
restartDelayMs: processConfig.restartDelayMs,
|
|
491
|
-
shouldRestart: options.shouldRestart || (() =>
|
|
653
|
+
shouldRestart: options.shouldRestart || (() => {
|
|
654
|
+
if (this.state !== "active" && this.state !== "starting") return false
|
|
655
|
+
|
|
656
|
+
// A companion whose same-generation service dependency has retired can never
|
|
657
|
+
// satisfy its startup gate again. Treat that dependency lifecycle as terminal
|
|
658
|
+
// for this process so an exit does not become an endless supervisor restart loop.
|
|
659
|
+
return !dependencyRetired()
|
|
660
|
+
}),
|
|
661
|
+
shouldStop: options.shouldStop || dependencyRetired,
|
|
492
662
|
stopSignal: processConfig.stopSignal,
|
|
493
663
|
stopTimeoutMs: processConfig.gracefulStopMs
|
|
494
664
|
}
|
|
@@ -708,10 +878,21 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
708
878
|
// their lifecycle drain runs as soon as the release is retired — in parallel with the
|
|
709
879
|
// connection drain, not held until after it. The rest stop once connections have closed.
|
|
710
880
|
const entries = [...this.processes.entries()]
|
|
711
|
-
const
|
|
712
|
-
|
|
881
|
+
const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
|
|
882
|
+
id,
|
|
883
|
+
promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
|
|
884
|
+
}))
|
|
713
885
|
const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
714
886
|
const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
887
|
+
// Bracket the retirement wait with a pair of observations: this one, and the one taken when
|
|
888
|
+
// every stop has returned. The gap between them is the wait the transition already performs
|
|
889
|
+
// (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
|
|
890
|
+
// itself rather than from a sampler running alongside it.
|
|
891
|
+
const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
|
|
892
|
+
? this.takeLivenessSample(Date.now())
|
|
893
|
+
: undefined
|
|
894
|
+
|
|
895
|
+
if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
|
|
715
896
|
|
|
716
897
|
if (this.connectionCount > 0) {
|
|
717
898
|
await new Promise((resolve) => {
|
|
@@ -730,17 +911,40 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
730
911
|
}
|
|
731
912
|
|
|
732
913
|
if (this.ownerHandoffPaused) {
|
|
733
|
-
void Promise.allSettled(nonBlockingStops)
|
|
914
|
+
void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
|
|
734
915
|
return
|
|
735
916
|
}
|
|
736
917
|
|
|
737
|
-
|
|
738
|
-
await Promise.allSettled(
|
|
739
|
-
await Promise.allSettled(
|
|
918
|
+
const retirementTimeoutMs = config.proxy.nonBlockingDrainTimeoutMs
|
|
919
|
+
await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
|
|
920
|
+
const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
|
|
921
|
+
nonBlockingResults.forEach((result, index) => {
|
|
922
|
+
if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
|
|
923
|
+
const processInstance = this.processes.get(nonBlockingStops[index].id)
|
|
924
|
+
this.logger("release non-blocking process still draining", {
|
|
925
|
+
pid: processInstance?.status().pid ?? null,
|
|
926
|
+
processId: nonBlockingStops[index].id,
|
|
927
|
+
releaseId: this.releaseId,
|
|
928
|
+
timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
|
|
929
|
+
})
|
|
930
|
+
}
|
|
931
|
+
})
|
|
932
|
+
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
|
|
933
|
+
|
|
934
|
+
// Last act of the retirement transition, before the release is recorded stopped: whatever
|
|
935
|
+
// is still alive here is the residue, and the bracket pair above can now say whether it is
|
|
936
|
+
// wedged. A generation that stops with a live process group is the state that made
|
|
937
|
+
// operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
|
|
938
|
+
// it does not end silently with the process still holding the release.
|
|
939
|
+
if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
|
|
940
|
+
|
|
740
941
|
this.state = "stopped"
|
|
741
942
|
this.stoppedAt = new Date().toISOString()
|
|
742
943
|
this.releasePortReservations()
|
|
743
944
|
this.abandonLocalConnections()
|
|
945
|
+
// The bracket pair is deliberately kept: a generation that stopped with a live process
|
|
946
|
+
// group stays observable, and the deploy path needs the older observation to prove the
|
|
947
|
+
// residue is still flat rather than starting a fresh window from zero.
|
|
744
948
|
}
|
|
745
949
|
|
|
746
950
|
/**
|
|
@@ -760,12 +964,18 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
760
964
|
this.drainStartedAt = new Date().toISOString()
|
|
761
965
|
}
|
|
762
966
|
|
|
967
|
+
// Refresh the live process definitions with the current retirement policy,
|
|
968
|
+
// but retain this generation's exact config identity for owner recovery.
|
|
969
|
+
// Replacing the stored generation config here would make a still-running
|
|
970
|
+
// singleton from the previous release fail the guardian provenance fence.
|
|
971
|
+
const generationConfig = this.config
|
|
972
|
+
|
|
763
973
|
this.refreshProcessDefinitions(config)
|
|
974
|
+
this.config = generationConfig
|
|
764
975
|
const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
|
|
765
976
|
const results = await Promise.allSettled([...this.processes.entries()]
|
|
766
977
|
.filter(([id]) => generationIds.has(id))
|
|
767
978
|
.map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
|
|
768
|
-
|
|
769
979
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
770
980
|
|
|
771
981
|
if (errors.length > 0) {
|
|
@@ -776,15 +986,44 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
776
986
|
this.retirementError = undefined
|
|
777
987
|
}
|
|
778
988
|
|
|
779
|
-
/**
|
|
780
|
-
|
|
781
|
-
|
|
989
|
+
/**
|
|
990
|
+
* Stops every process owned by the generation.
|
|
991
|
+
* @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
|
|
992
|
+
* supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
|
|
993
|
+
* is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
|
|
994
|
+
* `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
|
|
995
|
+
* the wait holds the mutation fence and every later deploy is refused (STOE-2015).
|
|
996
|
+
* @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
|
|
997
|
+
*/
|
|
998
|
+
async stop(options = {}) {
|
|
999
|
+
const {retirementTimeoutMs} = options
|
|
1000
|
+
/** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
|
|
1001
|
+
const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
|
|
1002
|
+
const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
|
|
1003
|
+
|
|
1004
|
+
return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
|
|
1005
|
+
})
|
|
1006
|
+
|
|
1007
|
+
await Promise.allSettled(stopTasks.map(({promise}) => promise))
|
|
1008
|
+
|
|
1009
|
+
// A stop that stopped waiting leaves the process running. Name it, so the operator sees the
|
|
1010
|
+
// residue instead of a command that produced nothing.
|
|
1011
|
+
for (const {bounded, id, instance} of stopTasks) {
|
|
1012
|
+
if (!bounded || !instance.retirementTimedOut) continue
|
|
1013
|
+
|
|
1014
|
+
this.logger("release stop gave up waiting for a retiring process", {
|
|
1015
|
+
pid: instance.status().pid ?? null,
|
|
1016
|
+
processId: id,
|
|
1017
|
+
releaseId: this.releaseId,
|
|
1018
|
+
timeoutMs: retirementTimeoutMs
|
|
1019
|
+
})
|
|
1020
|
+
}
|
|
782
1021
|
|
|
783
|
-
await Promise.allSettled(stopTasks)
|
|
784
1022
|
this.state = "stopped"
|
|
785
1023
|
this.stoppedAt = new Date().toISOString()
|
|
786
1024
|
this.releasePortReservations()
|
|
787
1025
|
this.abandonLocalConnections()
|
|
1026
|
+
this.clearDrainLivenessSamples()
|
|
788
1027
|
}
|
|
789
1028
|
|
|
790
1029
|
/** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
|
|
@@ -792,12 +1031,171 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
792
1031
|
await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
|
|
793
1032
|
}
|
|
794
1033
|
|
|
1034
|
+
/**
|
|
1035
|
+
* Records one liveness observation for a draining generation and classifies its drain.
|
|
1036
|
+
*
|
|
1037
|
+
* Two observations are needed to prove CPU flatness, and reading a process group means
|
|
1038
|
+
* scanning procfs, so sampling is bounded three ways: a generation that still owns live
|
|
1039
|
+
* connections is classified without touching procfs at all (connections alone settle it), a
|
|
1040
|
+
* new observation is taken only once the newest one is already a full window behind the
|
|
1041
|
+
* present read, and every other read reuses the newest observation. A group that is not
|
|
1042
|
+
* draining reports no drain.
|
|
1043
|
+
* @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
|
|
1044
|
+
*/
|
|
1045
|
+
drainStatus() {
|
|
1046
|
+
if (this.state !== "draining") return undefined
|
|
1047
|
+
|
|
1048
|
+
return this.observeRetirement({nowMs: Date.now()}).classification
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
/**
|
|
1052
|
+
* Records the newest liveness observation of this generation and classifies its retirement.
|
|
1053
|
+
* @param {object} options - Observation options.
|
|
1054
|
+
* @param {number} options.nowMs - Read time.
|
|
1055
|
+
* @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
|
|
1056
|
+
* @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
|
|
1057
|
+
*/
|
|
1058
|
+
observeRetirement({nowMs, previousSample}) {
|
|
1059
|
+
const drainBudgetMs = this.config.proxy.drainTimeoutMs
|
|
1060
|
+
const drainElapsedMs = this.drainStartedAt === undefined ? 0 : nowMs - Date.parse(this.drainStartedAt)
|
|
1061
|
+
|
|
1062
|
+
if (previousSample === undefined && this.connectionCount > 0) {
|
|
1063
|
+
const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
|
|
1064
|
+
|
|
1065
|
+
return {
|
|
1066
|
+
classification: classifyDrainProgress({
|
|
1067
|
+
connectionCount: this.connectionCount,
|
|
1068
|
+
currentSample,
|
|
1069
|
+
drainBudgetMs,
|
|
1070
|
+
drainElapsedMs,
|
|
1071
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1072
|
+
previousSample: undefined
|
|
1073
|
+
}),
|
|
1074
|
+
currentSample
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
const newest = this.drainNewestSample
|
|
1079
|
+
const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
|
|
1080
|
+
? this.takeLivenessSample(nowMs)
|
|
1081
|
+
: newest
|
|
1082
|
+
const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
|
|
1083
|
+
|
|
1084
|
+
if (currentSample !== newest) {
|
|
1085
|
+
this.drainPriorSample = newest
|
|
1086
|
+
this.drainNewestSample = currentSample
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
return {
|
|
1090
|
+
classification: classifyDrainProgress({
|
|
1091
|
+
connectionCount: this.connectionCount,
|
|
1092
|
+
currentSample,
|
|
1093
|
+
drainBudgetMs,
|
|
1094
|
+
drainElapsedMs,
|
|
1095
|
+
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1096
|
+
previousSample: priorSample
|
|
1097
|
+
}),
|
|
1098
|
+
currentSample
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
/**
|
|
1103
|
+
* Applies the wedge escalation to this generation: terminates the processes a proven wedge
|
|
1104
|
+
* proves are holding nothing, and names the ones it refused to terminate.
|
|
1105
|
+
*
|
|
1106
|
+
* Both halves are required. The card this path was written for (STOE-1687) is about a
|
|
1107
|
+
* wedged worker holding a release open for four hours while deploys stack behind it; the
|
|
1108
|
+
* boundary on the same card (STOE-1077) is that no process still holding live work may ever
|
|
1109
|
+
* be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
|
|
1110
|
+
* with the pids and process groups it is holding, which is the information an operator needs
|
|
1111
|
+
* in place of guessing.
|
|
1112
|
+
* @param {object} options - Escalation context.
|
|
1113
|
+
* @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
|
|
1114
|
+
* @param {string} options.reason - Named context recorded with the decision.
|
|
1115
|
+
* @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
|
|
1116
|
+
*/
|
|
1117
|
+
async escalateWedgedGeneration({previousSample, reason}) {
|
|
1118
|
+
const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
|
|
1119
|
+
const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
|
|
1120
|
+
const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
|
|
1121
|
+
|
|
1122
|
+
if (decision.decision === "report") {
|
|
1123
|
+
if (classification.status === "wedged") {
|
|
1124
|
+
this.logger("release wedged generation left alive", {
|
|
1125
|
+
escalation: "refused",
|
|
1126
|
+
heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
|
|
1127
|
+
pids: liveObservations.map((process) => process.pid ?? null),
|
|
1128
|
+
processIds: liveObservations.map((process) => process.id),
|
|
1129
|
+
reason: decision.reason,
|
|
1130
|
+
releaseId: this.releaseId,
|
|
1131
|
+
trigger: reason
|
|
1132
|
+
})
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
return []
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
const escalated = []
|
|
1139
|
+
|
|
1140
|
+
for (const target of decision.targets) {
|
|
1141
|
+
const processInstance = this.processes.get(target.processId)
|
|
1142
|
+
|
|
1143
|
+
if (!processInstance) continue
|
|
1144
|
+
if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
if (escalated.length > 0) {
|
|
1148
|
+
this.logger("release wedged generation escalated", {
|
|
1149
|
+
escalation: "SIGKILL",
|
|
1150
|
+
processIds: escalated,
|
|
1151
|
+
reason: decision.reason,
|
|
1152
|
+
releaseId: this.releaseId,
|
|
1153
|
+
trigger: reason
|
|
1154
|
+
})
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
return escalated
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
/**
|
|
1161
|
+
* @param {number} nowMs - Observation time.
|
|
1162
|
+
* @returns {DrainLivenessSample} One liveness observation of every process in the generation.
|
|
1163
|
+
*/
|
|
1164
|
+
takeLivenessSample(nowMs) {
|
|
1165
|
+
const processes = [...this.processes.entries()].map(([id, processInstance]) => {
|
|
1166
|
+
const processStatus = processInstance.status()
|
|
1167
|
+
const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
|
|
1168
|
+
// A group that outlived its tracked process is still observed by group id: it holds the
|
|
1169
|
+
// generation's work even though no rollbridge record can signal it any more.
|
|
1170
|
+
const pgid = processStatus.pid ?? heldProcessGroupPgid
|
|
1171
|
+
const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
|
|
1172
|
+
|
|
1173
|
+
return {
|
|
1174
|
+
cpuMs: sample?.cpuMs,
|
|
1175
|
+
heldProcessGroupPgid,
|
|
1176
|
+
id,
|
|
1177
|
+
liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
|
|
1178
|
+
pid: processStatus.pid,
|
|
1179
|
+
retirementTimedOut: processInstance.retirementTimedOut === true
|
|
1180
|
+
}
|
|
1181
|
+
})
|
|
1182
|
+
|
|
1183
|
+
return {atMs: nowMs, processes}
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
/** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
|
|
1187
|
+
clearDrainLivenessSamples() {
|
|
1188
|
+
this.drainNewestSample = undefined
|
|
1189
|
+
this.drainPriorSample = undefined
|
|
1190
|
+
}
|
|
1191
|
+
|
|
795
1192
|
/** @returns {ReleaseStatus} Status payload. */
|
|
796
1193
|
status() {
|
|
797
1194
|
return {
|
|
798
1195
|
activatedAt: this.activatedAt,
|
|
799
1196
|
connectionCount: this.connectionCount,
|
|
800
1197
|
connections: {...this.connections},
|
|
1198
|
+
drain: this.drainStatus(),
|
|
801
1199
|
drainStartedAt: this.drainStartedAt,
|
|
802
1200
|
ports: {...this.ports},
|
|
803
1201
|
processes: [...this.processes.values()].map((processInstance) => processInstance.status()),
|