rollbridge 0.1.69 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -53
- package/TODO.md +0 -12
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +69 -370
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +21 -211
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -427
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +4 -118
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +28 -132
- package/test/owner-replacement.test.js +11 -172
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +82 -766
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
package/src/release-group.js
CHANGED
|
@@ -2,29 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
import {spawn} from "node:child_process"
|
|
4
4
|
import {EventEmitter} from "node:events"
|
|
5
|
-
import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
|
|
6
5
|
import ManagedProcess from "./managed-process.js"
|
|
7
6
|
import {findAvailablePort} from "./port-allocator.js"
|
|
8
|
-
import {processStartOrder} from "./process-order.js"
|
|
9
|
-
import {sampleProcessGroup} from "./process-memory.js"
|
|
10
7
|
import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
|
|
11
|
-
import {waitForHealth
|
|
12
|
-
|
|
13
|
-
const DEPENDENCY_READY_INTERVAL_MS = 250
|
|
14
|
-
// Shortest gap between two liveness observations that can prove a draining group's CPU is
|
|
15
|
-
// flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
|
|
16
|
-
// with live runner children doing work, so flatness is only evidence below this window.
|
|
17
|
-
const DRAIN_SAMPLE_WINDOW_MS = 20000
|
|
8
|
+
import {waitForHealth} from "./health.js"
|
|
18
9
|
|
|
19
10
|
/**
|
|
20
11
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
21
12
|
* @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
|
|
22
|
-
* @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
|
|
23
|
-
* @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
|
|
24
13
|
* @typedef {{http: number, websocket: number}} ReleaseConnections
|
|
25
|
-
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections,
|
|
26
|
-
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean
|
|
27
|
-
* @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
|
|
14
|
+
* @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
|
|
15
|
+
* @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
|
|
28
16
|
*/
|
|
29
17
|
|
|
30
18
|
|
|
@@ -69,14 +57,6 @@ function envId(id) {
|
|
|
69
57
|
return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
|
|
70
58
|
}
|
|
71
59
|
|
|
72
|
-
/**
|
|
73
|
-
* @param {number | undefined} startedAtMs - When the measured phase began.
|
|
74
|
-
* @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
|
|
75
|
-
*/
|
|
76
|
-
function elapsedSince(startedAtMs) {
|
|
77
|
-
return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
|
|
78
|
-
}
|
|
79
|
-
|
|
80
60
|
/**
|
|
81
61
|
* @param {import("./config.js").ProcessConfig} processConfig - Process config.
|
|
82
62
|
* @param {number} index - Zero-based replica index.
|
|
@@ -97,10 +77,9 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
97
77
|
* @param {string | undefined} args.revision - Revision.
|
|
98
78
|
* @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
|
|
99
79
|
* @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
|
|
100
|
-
* @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
|
|
101
80
|
* @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
|
|
102
81
|
*/
|
|
103
|
-
constructor({config, logger, portReservations = new Set(), processFactory,
|
|
82
|
+
constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
|
|
104
83
|
super()
|
|
105
84
|
|
|
106
85
|
this.config = config
|
|
@@ -122,36 +101,22 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
122
101
|
this.servicePorts = servicePorts
|
|
123
102
|
this.shouldStart = shouldStart
|
|
124
103
|
this.processFactory = processFactory
|
|
125
|
-
this.procPath = procPath
|
|
126
104
|
this.portsAllocated = false
|
|
127
105
|
this.drainStartedAt = /** @type {string | undefined} */ (undefined)
|
|
128
|
-
this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
|
|
129
|
-
this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
|
|
130
106
|
this.activatedAt = /** @type {string | undefined} */ (undefined)
|
|
131
107
|
this.stoppedAt = /** @type {string | undefined} */ (undefined)
|
|
132
108
|
this.retirementError = /** @type {string | undefined} */ (undefined)
|
|
133
109
|
this.preserveConfigOnRetirement = false
|
|
134
|
-
this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
|
|
135
|
-
this.readyProcessIds = /** @type {Set<string>} */ (new Set())
|
|
136
|
-
this.startedProcessIds = /** @type {Set<string>} */ (new Set())
|
|
137
|
-
this.startupDependencyWaitMs = 0
|
|
138
|
-
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
|
|
139
|
-
this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
|
|
140
110
|
}
|
|
141
111
|
|
|
142
112
|
/** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
|
|
143
113
|
async start() {
|
|
144
114
|
this.state = "starting"
|
|
145
|
-
this.startupStartedAtMs = Date.now()
|
|
146
115
|
|
|
147
116
|
try {
|
|
148
117
|
await this.allocatePorts()
|
|
149
118
|
|
|
150
119
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
151
|
-
await this.waitForConfiguredDependencies(processConfig)
|
|
152
|
-
/** @type {string[]} */
|
|
153
|
-
const instanceIds = []
|
|
154
|
-
|
|
155
120
|
for (let index = 0; index < processConfig.replicas; index += 1) {
|
|
156
121
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
157
122
|
|
|
@@ -162,17 +127,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
162
127
|
if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
|
|
163
128
|
if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
|
|
164
129
|
await processInstance.start("deploy")
|
|
165
|
-
this.processLaunchAtMs.set(instanceId, Date.now())
|
|
166
|
-
this.startedProcessIds.add(instanceId)
|
|
167
|
-
instanceIds.push(instanceId)
|
|
168
130
|
}
|
|
169
131
|
|
|
170
132
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
171
|
-
await
|
|
172
|
-
|
|
133
|
+
await waitForHealth({
|
|
134
|
+
health: processConfig.health,
|
|
135
|
+
host: this.config.proxy.upstreamHost,
|
|
136
|
+
port: this.ports[processConfig.id]
|
|
137
|
+
})
|
|
173
138
|
|
|
174
|
-
|
|
175
|
-
|
|
139
|
+
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
140
|
+
}
|
|
176
141
|
}
|
|
177
142
|
} catch (error) {
|
|
178
143
|
this.state = "failed"
|
|
@@ -292,132 +257,16 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
292
257
|
}
|
|
293
258
|
|
|
294
259
|
/**
|
|
295
|
-
*
|
|
296
|
-
* health budget. The dependency leg of a start is not app boot, and charging it to the
|
|
297
|
-
* health deadline measures the dependency instead of the app.
|
|
298
|
-
* @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
|
|
299
|
-
* @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
|
|
300
|
-
*/
|
|
301
|
-
async waitForConfiguredDependencies(processConfig) {
|
|
302
|
-
if (processConfig.dependsOn.length === 0) return
|
|
303
|
-
|
|
304
|
-
for (const dependencyId of processConfig.dependsOn) {
|
|
305
|
-
const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
|
|
306
|
-
|
|
307
|
-
if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
|
|
308
|
-
|
|
309
|
-
this.logger("release process dependency wait", {
|
|
310
|
-
dependencyPort: port,
|
|
311
|
-
dependencyProcessId: dependencyId,
|
|
312
|
-
processId: processConfig.id,
|
|
313
|
-
releaseId: this.releaseId,
|
|
314
|
-
timeoutMs: processConfig.dependencyTimeoutMs
|
|
315
|
-
})
|
|
316
|
-
|
|
317
|
-
const waitStartedAtMs = Date.now()
|
|
318
|
-
|
|
319
|
-
try {
|
|
320
|
-
const {attempts, elapsedMs} = await waitForListeningPort({
|
|
321
|
-
host: this.config.proxy.upstreamHost,
|
|
322
|
-
intervalMs: DEPENDENCY_READY_INTERVAL_MS,
|
|
323
|
-
port,
|
|
324
|
-
timeoutMs: processConfig.dependencyTimeoutMs
|
|
325
|
-
})
|
|
326
|
-
|
|
327
|
-
this.startupDependencyWaitMs += elapsedMs
|
|
328
|
-
this.logger("release process dependency ready", {
|
|
329
|
-
attempts,
|
|
330
|
-
dependencyPort: port,
|
|
331
|
-
dependencyProcessId: dependencyId,
|
|
332
|
-
elapsedMs,
|
|
333
|
-
processId: processConfig.id,
|
|
334
|
-
releaseId: this.releaseId
|
|
335
|
-
})
|
|
336
|
-
} catch (error) {
|
|
337
|
-
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
338
|
-
deadlineMs: processConfig.dependencyTimeoutMs,
|
|
339
|
-
dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
|
|
340
|
-
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
341
|
-
failedProcessId: processConfig.id,
|
|
342
|
-
probes: undefined,
|
|
343
|
-
stage: "dependency"
|
|
344
|
-
})
|
|
345
|
-
throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
/**
|
|
353
|
-
* Health checks the proxied process on its own boot budget and records what the boot cost.
|
|
354
|
-
* @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
|
|
355
|
-
* @returns {Promise<void>} Resolves once the process answers its health path.
|
|
356
|
-
*/
|
|
357
|
-
async waitForProxiedHealth(processConfig) {
|
|
358
|
-
const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
|
|
359
|
-
const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
|
|
360
|
-
|
|
361
|
-
try {
|
|
362
|
-
const {elapsedMs} = await waitForHealth({
|
|
363
|
-
health,
|
|
364
|
-
host: this.config.proxy.upstreamHost,
|
|
365
|
-
port: /** @type {number} */ (this.ports[processConfig.id]),
|
|
366
|
-
processId: processConfig.id
|
|
367
|
-
})
|
|
368
|
-
|
|
369
|
-
this.logger("release candidate health ready", {
|
|
370
|
-
bootMs: Date.now() - launchedAtMs,
|
|
371
|
-
deadlineMs: health.timeoutMs,
|
|
372
|
-
healthGateMs: elapsedMs,
|
|
373
|
-
path: health.path,
|
|
374
|
-
port: this.ports[processConfig.id],
|
|
375
|
-
processId: processConfig.id,
|
|
376
|
-
releaseId: this.releaseId,
|
|
377
|
-
waitedForDependenciesMs: this.startupDependencyWaitMs
|
|
378
|
-
})
|
|
379
|
-
} catch (error) {
|
|
380
|
-
const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
|
|
381
|
-
|
|
382
|
-
this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
|
|
383
|
-
deadlineMs: health.timeoutMs,
|
|
384
|
-
dependencyWaitMs: this.startupDependencyWaitMs,
|
|
385
|
-
elapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
386
|
-
failedProcessId: processConfig.id,
|
|
387
|
-
probes: healthCheck?.attempts,
|
|
388
|
-
stage: "health"
|
|
389
|
-
})
|
|
390
|
-
throw error
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
/**
|
|
395
|
-
* Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
|
|
260
|
+
* Logs process diagnostics around failed startup cleanup.
|
|
396
261
|
* @param {Error | string} error - Startup failure.
|
|
397
262
|
* @param {{phase: string}} options - Diagnostic phase.
|
|
398
263
|
* @returns {void}
|
|
399
264
|
*/
|
|
400
265
|
logStartupFailure(error, {phase}) {
|
|
401
|
-
const diagnostics = this.startupFailureDiagnostics
|
|
402
|
-
const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
|
|
403
|
-
const lastLog = failedLogs?.at(-1)
|
|
404
|
-
const readyProcesses = [...this.readyProcessIds]
|
|
405
|
-
|
|
406
266
|
this.logger("release startup failed", {
|
|
407
|
-
deadlineMs: diagnostics?.deadlineMs ?? null,
|
|
408
|
-
dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
|
|
409
|
-
elapsedMs: diagnostics?.elapsedMs ?? null,
|
|
410
267
|
error: error instanceof Error ? error.message : error,
|
|
411
|
-
failedProcessId: diagnostics?.failedProcessId ?? null,
|
|
412
|
-
lastOutput: lastLog?.line ?? null,
|
|
413
|
-
lastOutputAt: lastLog?.at ?? null,
|
|
414
268
|
phase,
|
|
415
|
-
|
|
416
|
-
readyProcesses,
|
|
417
|
-
releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
|
|
418
|
-
releaseId: this.releaseId,
|
|
419
|
-
stage: diagnostics?.stage ?? null,
|
|
420
|
-
unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
|
|
269
|
+
releaseId: this.releaseId
|
|
421
270
|
})
|
|
422
271
|
|
|
423
272
|
for (const processInstance of this.processes.values()) {
|
|
@@ -442,14 +291,18 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
442
291
|
* @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
|
|
443
292
|
*/
|
|
444
293
|
releaseProcessStartOrder() {
|
|
445
|
-
|
|
294
|
+
const releaseProcesses = this.config.processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
|
|
295
|
+
const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
|
|
296
|
+
const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
|
|
297
|
+
const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
|
|
298
|
+
|
|
299
|
+
return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
|
|
446
300
|
}
|
|
447
301
|
|
|
448
302
|
/** @returns {void} Marks this release active. */
|
|
449
303
|
activate() {
|
|
450
304
|
this.state = "active"
|
|
451
305
|
this.activatedAt = new Date().toISOString()
|
|
452
|
-
this.clearDrainLivenessSamples()
|
|
453
306
|
}
|
|
454
307
|
|
|
455
308
|
/** Runs the configured release-generation activation acknowledgement, if any. */
|
|
@@ -504,16 +357,6 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
504
357
|
this.drainStartedAt = undefined
|
|
505
358
|
this.retirementError = undefined
|
|
506
359
|
this.stoppedAt = undefined
|
|
507
|
-
this.clearDrainLivenessSamples()
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
/** Stops only the proxied processes when a failed compensation cannot restore jobs. */
|
|
511
|
-
async stopProxyTraffic() {
|
|
512
|
-
const proxiedIds = new Set(this.config.processes.filter((processConfig) => processConfig.policy === "proxied").map((processConfig) => processConfig.id))
|
|
513
|
-
|
|
514
|
-
await Promise.allSettled([...this.processes.entries()]
|
|
515
|
-
.filter(([id]) => proxiedIds.has(id.replace(/#\d+$/, "")))
|
|
516
|
-
.map(([, processInstance]) => processInstance.stop()))
|
|
517
360
|
}
|
|
518
361
|
|
|
519
362
|
/**
|
|
@@ -524,25 +367,24 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
524
367
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
525
368
|
this.assertCommittedGenerationRecoverable()
|
|
526
369
|
this.state = "starting"
|
|
527
|
-
this.startupStartedAtMs = Date.now()
|
|
528
370
|
try {
|
|
529
371
|
for (const processConfig of this.releaseProcessStartOrder()) {
|
|
530
|
-
await this.waitForConfiguredDependencies(processConfig)
|
|
531
372
|
const instances = this.getProcesses(processConfig.id)
|
|
532
373
|
|
|
533
374
|
if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
|
|
534
|
-
for (const {
|
|
375
|
+
for (const {process} of instances) {
|
|
535
376
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
536
377
|
await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
|
|
537
|
-
this.processLaunchAtMs.set(id, Date.now())
|
|
538
|
-
this.startedProcessIds.add(id)
|
|
539
378
|
}
|
|
540
379
|
|
|
541
380
|
if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
|
|
542
|
-
await
|
|
381
|
+
await waitForHealth({
|
|
382
|
+
health: processConfig.health,
|
|
383
|
+
host: this.config.proxy.upstreamHost,
|
|
384
|
+
port: this.ports[processConfig.id]
|
|
385
|
+
})
|
|
543
386
|
if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
|
|
544
387
|
}
|
|
545
|
-
for (const {id} of instances) this.readyProcessIds.add(id)
|
|
546
388
|
}
|
|
547
389
|
} catch (error) {
|
|
548
390
|
await this.abortCommittedGenerationRestart()
|
|
@@ -634,10 +476,6 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
634
476
|
...this.baseEnvironment(processConfig, {count, index}),
|
|
635
477
|
...renderedEnv
|
|
636
478
|
}
|
|
637
|
-
const dependencyRetired = () => processConfig.dependsOn.some((dependencyId) => {
|
|
638
|
-
const dependency = this.processes.get(dependencyId)
|
|
639
|
-
return dependency?.lifecycleRole === "retired" || dependency?.status().lifecycleRole === "retired"
|
|
640
|
-
})
|
|
641
479
|
|
|
642
480
|
return {
|
|
643
481
|
command: renderTemplate(processConfig.command, context),
|
|
@@ -650,15 +488,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
650
488
|
outputLines: processConfig.outputLines,
|
|
651
489
|
restart: processConfig.restart,
|
|
652
490
|
restartDelayMs: processConfig.restartDelayMs,
|
|
653
|
-
shouldRestart: options.shouldRestart || (() =>
|
|
654
|
-
if (this.state !== "active" && this.state !== "starting") return false
|
|
655
|
-
|
|
656
|
-
// A companion whose same-generation service dependency has retired can never
|
|
657
|
-
// satisfy its startup gate again. Treat that dependency lifecycle as terminal
|
|
658
|
-
// for this process so an exit does not become an endless supervisor restart loop.
|
|
659
|
-
return !dependencyRetired()
|
|
660
|
-
}),
|
|
661
|
-
shouldStop: options.shouldStop || dependencyRetired,
|
|
491
|
+
shouldRestart: options.shouldRestart || (() => this.state === "active" || this.state === "starting"),
|
|
662
492
|
stopSignal: processConfig.stopSignal,
|
|
663
493
|
stopTimeoutMs: processConfig.gracefulStopMs
|
|
664
494
|
}
|
|
@@ -878,21 +708,10 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
878
708
|
// their lifecycle drain runs as soon as the release is retired — in parallel with the
|
|
879
709
|
// connection drain, not held until after it. The rest stop once connections have closed.
|
|
880
710
|
const entries = [...this.processes.entries()]
|
|
881
|
-
const
|
|
882
|
-
|
|
883
|
-
promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
|
|
884
|
-
}))
|
|
711
|
+
const stopOptions = {timeoutMs: config.proxy.forceStopTimeoutMs}
|
|
712
|
+
const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([, processInstance]) => processInstance.stop(stopOptions))
|
|
885
713
|
const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
886
714
|
const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
|
|
887
|
-
// Bracket the retirement wait with a pair of observations: this one, and the one taken when
|
|
888
|
-
// every stop has returned. The gap between them is the wait the transition already performs
|
|
889
|
-
// (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
|
|
890
|
-
// itself rather than from a sampler running alongside it.
|
|
891
|
-
const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
|
|
892
|
-
? this.takeLivenessSample(Date.now())
|
|
893
|
-
: undefined
|
|
894
|
-
|
|
895
|
-
if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
|
|
896
715
|
|
|
897
716
|
if (this.connectionCount > 0) {
|
|
898
717
|
await new Promise((resolve) => {
|
|
@@ -911,39 +730,17 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
911
730
|
}
|
|
912
731
|
|
|
913
732
|
if (this.ownerHandoffPaused) {
|
|
914
|
-
void Promise.allSettled(nonBlockingStops
|
|
733
|
+
void Promise.allSettled(nonBlockingStops)
|
|
915
734
|
return
|
|
916
735
|
}
|
|
917
736
|
|
|
918
|
-
await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop()))
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
|
|
922
|
-
const processInstance = this.processes.get(nonBlockingStops[index].id)
|
|
923
|
-
this.logger("release non-blocking process still draining", {
|
|
924
|
-
pid: processInstance?.status().pid ?? null,
|
|
925
|
-
processId: nonBlockingStops[index].id,
|
|
926
|
-
releaseId: this.releaseId,
|
|
927
|
-
timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
|
|
928
|
-
})
|
|
929
|
-
}
|
|
930
|
-
})
|
|
931
|
-
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
|
|
932
|
-
|
|
933
|
-
// Last act of the retirement transition, before the release is recorded stopped: whatever
|
|
934
|
-
// is still alive here is the residue, and the bracket pair above can now say whether it is
|
|
935
|
-
// wedged. A generation that stops with a live process group is the state that made
|
|
936
|
-
// operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
|
|
937
|
-
// it does not end silently with the process still holding the release.
|
|
938
|
-
if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
|
|
939
|
-
|
|
737
|
+
await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop(stopOptions)))
|
|
738
|
+
await Promise.allSettled(nonBlockingStops)
|
|
739
|
+
await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop(stopOptions)))
|
|
940
740
|
this.state = "stopped"
|
|
941
741
|
this.stoppedAt = new Date().toISOString()
|
|
942
742
|
this.releasePortReservations()
|
|
943
743
|
this.abandonLocalConnections()
|
|
944
|
-
// The bracket pair is deliberately kept: a generation that stopped with a live process
|
|
945
|
-
// group stays observable, and the deploy path needs the older observation to prove the
|
|
946
|
-
// residue is still flat rather than starting a fresh window from zero.
|
|
947
744
|
}
|
|
948
745
|
|
|
949
746
|
/**
|
|
@@ -968,6 +765,7 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
968
765
|
const results = await Promise.allSettled([...this.processes.entries()]
|
|
969
766
|
.filter(([id]) => generationIds.has(id))
|
|
970
767
|
.map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
|
|
768
|
+
|
|
971
769
|
const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
|
|
972
770
|
|
|
973
771
|
if (errors.length > 0) {
|
|
@@ -978,44 +776,15 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
978
776
|
this.retirementError = undefined
|
|
979
777
|
}
|
|
980
778
|
|
|
981
|
-
/**
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
* supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
|
|
985
|
-
* is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
|
|
986
|
-
* `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
|
|
987
|
-
* the wait holds the mutation fence and every later deploy is refused (STOE-2015).
|
|
988
|
-
* @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
|
|
989
|
-
*/
|
|
990
|
-
async stop(options = {}) {
|
|
991
|
-
const {retirementTimeoutMs} = options
|
|
992
|
-
/** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
|
|
993
|
-
const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
|
|
994
|
-
const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
|
|
995
|
-
|
|
996
|
-
return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
|
|
997
|
-
})
|
|
998
|
-
|
|
999
|
-
await Promise.allSettled(stopTasks.map(({promise}) => promise))
|
|
1000
|
-
|
|
1001
|
-
// A stop that stopped waiting leaves the process running. Name it, so the operator sees the
|
|
1002
|
-
// residue instead of a command that produced nothing.
|
|
1003
|
-
for (const {bounded, id, instance} of stopTasks) {
|
|
1004
|
-
if (!bounded || !instance.retirementTimedOut) continue
|
|
1005
|
-
|
|
1006
|
-
this.logger("release stop gave up waiting for a retiring process", {
|
|
1007
|
-
pid: instance.status().pid ?? null,
|
|
1008
|
-
processId: id,
|
|
1009
|
-
releaseId: this.releaseId,
|
|
1010
|
-
timeoutMs: retirementTimeoutMs
|
|
1011
|
-
})
|
|
1012
|
-
}
|
|
779
|
+
/** @returns {Promise<void>} Stops all release-owned processes. */
|
|
780
|
+
async stop() {
|
|
781
|
+
const stopTasks = [...this.processes.values()].map((processInstance) => processInstance.stop())
|
|
1013
782
|
|
|
783
|
+
await Promise.allSettled(stopTasks)
|
|
1014
784
|
this.state = "stopped"
|
|
1015
785
|
this.stoppedAt = new Date().toISOString()
|
|
1016
786
|
this.releasePortReservations()
|
|
1017
787
|
this.abandonLocalConnections()
|
|
1018
|
-
this.clearDrainLivenessSamples()
|
|
1019
788
|
}
|
|
1020
789
|
|
|
1021
790
|
/** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
|
|
@@ -1023,171 +792,12 @@ export default class ReleaseGroup extends EventEmitter {
|
|
|
1023
792
|
await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
|
|
1024
793
|
}
|
|
1025
794
|
|
|
1026
|
-
/**
|
|
1027
|
-
* Records one liveness observation for a draining generation and classifies its drain.
|
|
1028
|
-
*
|
|
1029
|
-
* Two observations are needed to prove CPU flatness, and reading a process group means
|
|
1030
|
-
* scanning procfs, so sampling is bounded three ways: a generation that still owns live
|
|
1031
|
-
* connections is classified without touching procfs at all (connections alone settle it), a
|
|
1032
|
-
* new observation is taken only once the newest one is already a full window behind the
|
|
1033
|
-
* present read, and every other read reuses the newest observation. A group that is not
|
|
1034
|
-
* draining reports no drain.
|
|
1035
|
-
* @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
|
|
1036
|
-
*/
|
|
1037
|
-
drainStatus() {
|
|
1038
|
-
if (this.state !== "draining") return undefined
|
|
1039
|
-
|
|
1040
|
-
return this.observeRetirement({nowMs: Date.now()}).classification
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
/**
|
|
1044
|
-
* Records the newest liveness observation of this generation and classifies its retirement.
|
|
1045
|
-
* @param {object} options - Observation options.
|
|
1046
|
-
* @param {number} options.nowMs - Read time.
|
|
1047
|
-
* @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
|
|
1048
|
-
* @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
|
|
1049
|
-
*/
|
|
1050
|
-
observeRetirement({nowMs, previousSample}) {
|
|
1051
|
-
const drainBudgetMs = this.config.proxy.drainTimeoutMs
|
|
1052
|
-
const drainElapsedMs = this.drainStartedAt === undefined ? 0 : nowMs - Date.parse(this.drainStartedAt)
|
|
1053
|
-
|
|
1054
|
-
if (previousSample === undefined && this.connectionCount > 0) {
|
|
1055
|
-
const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
|
|
1056
|
-
|
|
1057
|
-
return {
|
|
1058
|
-
classification: classifyDrainProgress({
|
|
1059
|
-
connectionCount: this.connectionCount,
|
|
1060
|
-
currentSample,
|
|
1061
|
-
drainBudgetMs,
|
|
1062
|
-
drainElapsedMs,
|
|
1063
|
-
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1064
|
-
previousSample: undefined
|
|
1065
|
-
}),
|
|
1066
|
-
currentSample
|
|
1067
|
-
}
|
|
1068
|
-
}
|
|
1069
|
-
|
|
1070
|
-
const newest = this.drainNewestSample
|
|
1071
|
-
const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
|
|
1072
|
-
? this.takeLivenessSample(nowMs)
|
|
1073
|
-
: newest
|
|
1074
|
-
const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
|
|
1075
|
-
|
|
1076
|
-
if (currentSample !== newest) {
|
|
1077
|
-
this.drainPriorSample = newest
|
|
1078
|
-
this.drainNewestSample = currentSample
|
|
1079
|
-
}
|
|
1080
|
-
|
|
1081
|
-
return {
|
|
1082
|
-
classification: classifyDrainProgress({
|
|
1083
|
-
connectionCount: this.connectionCount,
|
|
1084
|
-
currentSample,
|
|
1085
|
-
drainBudgetMs,
|
|
1086
|
-
drainElapsedMs,
|
|
1087
|
-
minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
|
|
1088
|
-
previousSample: priorSample
|
|
1089
|
-
}),
|
|
1090
|
-
currentSample
|
|
1091
|
-
}
|
|
1092
|
-
}
|
|
1093
|
-
|
|
1094
|
-
/**
|
|
1095
|
-
* Applies the wedge escalation to this generation: terminates the processes a proven wedge
|
|
1096
|
-
* proves are holding nothing, and names the ones it refused to terminate.
|
|
1097
|
-
*
|
|
1098
|
-
* Both halves are required. The card this path was written for (STOE-1687) is about a
|
|
1099
|
-
* wedged worker holding a release open for four hours while deploys stack behind it; the
|
|
1100
|
-
* boundary on the same card (STOE-1077) is that no process still holding live work may ever
|
|
1101
|
-
* be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
|
|
1102
|
-
* with the pids and process groups it is holding, which is the information an operator needs
|
|
1103
|
-
* in place of guessing.
|
|
1104
|
-
* @param {object} options - Escalation context.
|
|
1105
|
-
* @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
|
|
1106
|
-
* @param {string} options.reason - Named context recorded with the decision.
|
|
1107
|
-
* @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
|
|
1108
|
-
*/
|
|
1109
|
-
async escalateWedgedGeneration({previousSample, reason}) {
|
|
1110
|
-
const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
|
|
1111
|
-
const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
|
|
1112
|
-
const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
|
|
1113
|
-
|
|
1114
|
-
if (decision.decision === "report") {
|
|
1115
|
-
if (classification.status === "wedged") {
|
|
1116
|
-
this.logger("release wedged generation left alive", {
|
|
1117
|
-
escalation: "refused",
|
|
1118
|
-
heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
|
|
1119
|
-
pids: liveObservations.map((process) => process.pid ?? null),
|
|
1120
|
-
processIds: liveObservations.map((process) => process.id),
|
|
1121
|
-
reason: decision.reason,
|
|
1122
|
-
releaseId: this.releaseId,
|
|
1123
|
-
trigger: reason
|
|
1124
|
-
})
|
|
1125
|
-
}
|
|
1126
|
-
|
|
1127
|
-
return []
|
|
1128
|
-
}
|
|
1129
|
-
|
|
1130
|
-
const escalated = []
|
|
1131
|
-
|
|
1132
|
-
for (const target of decision.targets) {
|
|
1133
|
-
const processInstance = this.processes.get(target.processId)
|
|
1134
|
-
|
|
1135
|
-
if (!processInstance) continue
|
|
1136
|
-
if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
|
|
1137
|
-
}
|
|
1138
|
-
|
|
1139
|
-
if (escalated.length > 0) {
|
|
1140
|
-
this.logger("release wedged generation escalated", {
|
|
1141
|
-
escalation: "SIGKILL",
|
|
1142
|
-
processIds: escalated,
|
|
1143
|
-
reason: decision.reason,
|
|
1144
|
-
releaseId: this.releaseId,
|
|
1145
|
-
trigger: reason
|
|
1146
|
-
})
|
|
1147
|
-
}
|
|
1148
|
-
|
|
1149
|
-
return escalated
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
/**
|
|
1153
|
-
* @param {number} nowMs - Observation time.
|
|
1154
|
-
* @returns {DrainLivenessSample} One liveness observation of every process in the generation.
|
|
1155
|
-
*/
|
|
1156
|
-
takeLivenessSample(nowMs) {
|
|
1157
|
-
const processes = [...this.processes.entries()].map(([id, processInstance]) => {
|
|
1158
|
-
const processStatus = processInstance.status()
|
|
1159
|
-
const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
|
|
1160
|
-
// A group that outlived its tracked process is still observed by group id: it holds the
|
|
1161
|
-
// generation's work even though no rollbridge record can signal it any more.
|
|
1162
|
-
const pgid = processStatus.pid ?? heldProcessGroupPgid
|
|
1163
|
-
const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
|
|
1164
|
-
|
|
1165
|
-
return {
|
|
1166
|
-
cpuMs: sample?.cpuMs,
|
|
1167
|
-
heldProcessGroupPgid,
|
|
1168
|
-
id,
|
|
1169
|
-
liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
|
|
1170
|
-
pid: processStatus.pid,
|
|
1171
|
-
retirementTimedOut: processInstance.retirementTimedOut === true
|
|
1172
|
-
}
|
|
1173
|
-
})
|
|
1174
|
-
|
|
1175
|
-
return {atMs: nowMs, processes}
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
|
-
/** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
|
|
1179
|
-
clearDrainLivenessSamples() {
|
|
1180
|
-
this.drainNewestSample = undefined
|
|
1181
|
-
this.drainPriorSample = undefined
|
|
1182
|
-
}
|
|
1183
|
-
|
|
1184
795
|
/** @returns {ReleaseStatus} Status payload. */
|
|
1185
796
|
status() {
|
|
1186
797
|
return {
|
|
1187
798
|
activatedAt: this.activatedAt,
|
|
1188
799
|
connectionCount: this.connectionCount,
|
|
1189
800
|
connections: {...this.connections},
|
|
1190
|
-
drain: this.drainStatus(),
|
|
1191
801
|
drainStartedAt: this.drainStartedAt,
|
|
1192
802
|
ports: {...this.ports},
|
|
1193
803
|
processes: [...this.processes.values()].map((processInstance) => processInstance.status()),
|
package/tensorbuzz.yml
CHANGED
|
@@ -1,23 +1,4 @@
|
|
|
1
|
-
# TensorBuzz build config for Rollbridge.
|
|
2
|
-
#
|
|
3
|
-
# `paths` (STOE-1179): this gate installs the dependency tree and runs typecheck, ESLint and the
|
|
4
|
-
# whole Velocious suite, and every one of those reads code, `examples/`, and the package/tooling
|
|
5
|
-
# manifests — no gate input is Markdown. A delivery that changes nothing but Markdown therefore
|
|
6
|
-
# has nothing for this lane to check, and TensorBuzz records it as a terminal `skipped` build
|
|
7
|
-
# instead of spending a container and ~3 minutes on it. `test/tensorbuzz-config.test.js` asserts
|
|
8
|
-
# the split against every tracked path so a later edit cannot carve a gate input out of the lane.
|
|
9
|
-
#
|
|
10
|
-
# Ordered rules, last match wins, so the negation has to come after `"**"`. Quote every negation:
|
|
11
|
-
# an unquoted `!x` is a YAML tag, and the config validator fails the whole file rather than
|
|
12
|
-
# filtering anything. The lane still runs whenever `tensorbuzz.yml` itself changes, whenever the
|
|
13
|
-
# changed-file list is missing, empty, or capped by the host, and on every manual rebuild.
|
|
14
|
-
paths:
|
|
15
|
-
- "**"
|
|
16
|
-
- "!**/*.md"
|
|
17
|
-
|
|
18
1
|
before_script:
|
|
19
2
|
- npm install
|
|
20
3
|
script:
|
|
21
|
-
|
|
22
|
-
# the TensorBuzz runner's workspace TMPDIR is deliberately long.
|
|
23
|
-
- TMPDIR=/tmp npm run all-checks
|
|
4
|
+
- npm run all-checks
|