rollbridge 0.1.69 → 0.1.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +1 -53
  2. package/TODO.md +0 -12
  3. package/docs/cli.md +3 -128
  4. package/docs/config.md +6 -177
  5. package/docs/troubleshooting.md +6 -97
  6. package/docs/velocious.md +1 -1
  7. package/docs/workers.md +0 -11
  8. package/examples/tensorbuzz.com.js +2 -3
  9. package/package.json +1 -1
  10. package/src/cli.js +2 -68
  11. package/src/config.js +8 -347
  12. package/src/daemon.js +69 -370
  13. package/src/doctor.js +0 -74
  14. package/src/guardian-client.js +8 -67
  15. package/src/health.js +8 -99
  16. package/src/managed-process.js +16 -197
  17. package/src/process-guardian.js +21 -211
  18. package/src/process-memory.js +2 -62
  19. package/src/release-group.js +37 -427
  20. package/tensorbuzz.yml +1 -20
  21. package/test/completion.test.js +2 -5
  22. package/test/config-validation.test.js +5 -135
  23. package/test/guardian-client.test.js +4 -118
  24. package/test/health.test.js +1 -74
  25. package/test/managed-process.test.js +1 -260
  26. package/test/owner-recovery.test.js +28 -132
  27. package/test/owner-replacement.test.js +11 -172
  28. package/test/process-memory.test.js +1 -47
  29. package/test/release-group.test.js +2 -408
  30. package/test/rollbridge.test.js +82 -766
  31. package/test/shutdown-completion.test.js +0 -52
  32. package/test/support/process.js +1 -3
  33. package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
  34. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
  35. package/changelog.d/20261104-compose-service-units.md +0 -21
  36. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
  37. package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
  38. package/changelog.d/20261109-path-filtered-builds.md +0 -11
  39. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
  40. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
  41. package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
  42. package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
  43. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
  44. package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
  45. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
  46. package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
  47. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
  48. package/changelog.d/20261116-guardian-version-handoff.md +0 -37
  49. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
  50. package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
  51. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
  52. package/docs/compose-services.md +0 -222
  53. package/src/compose-unit.js +0 -725
  54. package/src/drain-policy.js +0 -252
  55. package/src/process-order.js +0 -26
  56. package/test/compose-config.test.js +0 -160
  57. package/test/compose-docker.test.js +0 -297
  58. package/test/compose-unit.test.js +0 -415
  59. package/test/drain-policy.test.js +0 -328
  60. package/test/fixtures/deferred-boot-app.js +0 -106
  61. package/test/fixtures/gated-app.js +0 -50
  62. package/test/tensorbuzz-config.test.js +0 -228
@@ -2,29 +2,17 @@
2
2
 
3
3
  import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
- import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
6
5
  import ManagedProcess from "./managed-process.js"
7
6
  import {findAvailablePort} from "./port-allocator.js"
8
- import {processStartOrder} from "./process-order.js"
9
- import {sampleProcessGroup} from "./process-memory.js"
10
7
  import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
11
- import {waitForHealth, waitForListeningPort} from "./health.js"
12
-
13
- const DEPENDENCY_READY_INTERVAL_MS = 250
14
- // Shortest gap between two liveness observations that can prove a draining group's CPU is
15
- // flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
16
- // with live runner children doing work, so flatness is only evidence below this window.
17
- const DRAIN_SAMPLE_WINDOW_MS = 20000
8
+ import {waitForHealth} from "./health.js"
18
9
 
19
10
  /**
20
11
  * @typedef {import("./json.js").JsonValue} JsonValue
21
12
  * @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
22
- * @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
23
- * @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
24
13
  * @typedef {{http: number, websocket: number}} ReleaseConnections
25
- * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
26
- * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean, shouldStop?: () => boolean}} BuildProcessOptions
27
- * @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
14
+ * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
15
+ * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
28
16
  */
29
17
 
30
18
 
@@ -69,14 +57,6 @@ function envId(id) {
69
57
  return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
70
58
  }
71
59
 
72
- /**
73
- * @param {number | undefined} startedAtMs - When the measured phase began.
74
- * @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
75
- */
76
- function elapsedSince(startedAtMs) {
77
- return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
78
- }
79
-
80
60
  /**
81
61
  * @param {import("./config.js").ProcessConfig} processConfig - Process config.
82
62
  * @param {number} index - Zero-based replica index.
@@ -97,10 +77,9 @@ export default class ReleaseGroup extends EventEmitter {
97
77
  * @param {string | undefined} args.revision - Revision.
98
78
  * @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
99
79
  * @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
100
- * @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
101
80
  * @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
102
81
  */
103
- constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
82
+ constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
104
83
  super()
105
84
 
106
85
  this.config = config
@@ -122,36 +101,22 @@ export default class ReleaseGroup extends EventEmitter {
122
101
  this.servicePorts = servicePorts
123
102
  this.shouldStart = shouldStart
124
103
  this.processFactory = processFactory
125
- this.procPath = procPath
126
104
  this.portsAllocated = false
127
105
  this.drainStartedAt = /** @type {string | undefined} */ (undefined)
128
- this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
129
- this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
130
106
  this.activatedAt = /** @type {string | undefined} */ (undefined)
131
107
  this.stoppedAt = /** @type {string | undefined} */ (undefined)
132
108
  this.retirementError = /** @type {string | undefined} */ (undefined)
133
109
  this.preserveConfigOnRetirement = false
134
- this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
135
- this.readyProcessIds = /** @type {Set<string>} */ (new Set())
136
- this.startedProcessIds = /** @type {Set<string>} */ (new Set())
137
- this.startupDependencyWaitMs = 0
138
- this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
139
- this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
140
110
  }
141
111
 
142
112
  /** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
143
113
  async start() {
144
114
  this.state = "starting"
145
- this.startupStartedAtMs = Date.now()
146
115
 
147
116
  try {
148
117
  await this.allocatePorts()
149
118
 
150
119
  for (const processConfig of this.releaseProcessStartOrder()) {
151
- await this.waitForConfiguredDependencies(processConfig)
152
- /** @type {string[]} */
153
- const instanceIds = []
154
-
155
120
  for (let index = 0; index < processConfig.replicas; index += 1) {
156
121
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
157
122
 
@@ -162,17 +127,17 @@ export default class ReleaseGroup extends EventEmitter {
162
127
  if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
163
128
  if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
164
129
  await processInstance.start("deploy")
165
- this.processLaunchAtMs.set(instanceId, Date.now())
166
- this.startedProcessIds.add(instanceId)
167
- instanceIds.push(instanceId)
168
130
  }
169
131
 
170
132
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
171
- await this.waitForProxiedHealth(processConfig)
172
- }
133
+ await waitForHealth({
134
+ health: processConfig.health,
135
+ host: this.config.proxy.upstreamHost,
136
+ port: this.ports[processConfig.id]
137
+ })
173
138
 
174
- for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
175
- if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
139
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
140
+ }
176
141
  }
177
142
  } catch (error) {
178
143
  this.state = "failed"
@@ -292,132 +257,16 @@ export default class ReleaseGroup extends EventEmitter {
292
257
  }
293
258
 
294
259
  /**
295
- * Waits for the ports a process declares it depends on, on a budget that is not the
296
- * health budget. The dependency leg of a start is not app boot, and charging it to the
297
- * health deadline measures the dependency instead of the app.
298
- * @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
299
- * @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
300
- */
301
- async waitForConfiguredDependencies(processConfig) {
302
- if (processConfig.dependsOn.length === 0) return
303
-
304
- for (const dependencyId of processConfig.dependsOn) {
305
- const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
306
-
307
- if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
308
-
309
- this.logger("release process dependency wait", {
310
- dependencyPort: port,
311
- dependencyProcessId: dependencyId,
312
- processId: processConfig.id,
313
- releaseId: this.releaseId,
314
- timeoutMs: processConfig.dependencyTimeoutMs
315
- })
316
-
317
- const waitStartedAtMs = Date.now()
318
-
319
- try {
320
- const {attempts, elapsedMs} = await waitForListeningPort({
321
- host: this.config.proxy.upstreamHost,
322
- intervalMs: DEPENDENCY_READY_INTERVAL_MS,
323
- port,
324
- timeoutMs: processConfig.dependencyTimeoutMs
325
- })
326
-
327
- this.startupDependencyWaitMs += elapsedMs
328
- this.logger("release process dependency ready", {
329
- attempts,
330
- dependencyPort: port,
331
- dependencyProcessId: dependencyId,
332
- elapsedMs,
333
- processId: processConfig.id,
334
- releaseId: this.releaseId
335
- })
336
- } catch (error) {
337
- this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
338
- deadlineMs: processConfig.dependencyTimeoutMs,
339
- dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
340
- elapsedMs: elapsedSince(this.startupStartedAtMs),
341
- failedProcessId: processConfig.id,
342
- probes: undefined,
343
- stage: "dependency"
344
- })
345
- throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
346
- }
347
-
348
- if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
349
- }
350
- }
351
-
352
- /**
353
- * Health checks the proxied process on its own boot budget and records what the boot cost.
354
- * @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
355
- * @returns {Promise<void>} Resolves once the process answers its health path.
356
- */
357
- async waitForProxiedHealth(processConfig) {
358
- const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
359
- const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
360
-
361
- try {
362
- const {elapsedMs} = await waitForHealth({
363
- health,
364
- host: this.config.proxy.upstreamHost,
365
- port: /** @type {number} */ (this.ports[processConfig.id]),
366
- processId: processConfig.id
367
- })
368
-
369
- this.logger("release candidate health ready", {
370
- bootMs: Date.now() - launchedAtMs,
371
- deadlineMs: health.timeoutMs,
372
- healthGateMs: elapsedMs,
373
- path: health.path,
374
- port: this.ports[processConfig.id],
375
- processId: processConfig.id,
376
- releaseId: this.releaseId,
377
- waitedForDependenciesMs: this.startupDependencyWaitMs
378
- })
379
- } catch (error) {
380
- const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
381
-
382
- this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
383
- deadlineMs: health.timeoutMs,
384
- dependencyWaitMs: this.startupDependencyWaitMs,
385
- elapsedMs: elapsedSince(this.startupStartedAtMs),
386
- failedProcessId: processConfig.id,
387
- probes: healthCheck?.attempts,
388
- stage: "health"
389
- })
390
- throw error
391
- }
392
- }
393
-
394
- /**
395
- * Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
260
+ * Logs process diagnostics around failed startup cleanup.
396
261
  * @param {Error | string} error - Startup failure.
397
262
  * @param {{phase: string}} options - Diagnostic phase.
398
263
  * @returns {void}
399
264
  */
400
265
  logStartupFailure(error, {phase}) {
401
- const diagnostics = this.startupFailureDiagnostics
402
- const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
403
- const lastLog = failedLogs?.at(-1)
404
- const readyProcesses = [...this.readyProcessIds]
405
-
406
266
  this.logger("release startup failed", {
407
- deadlineMs: diagnostics?.deadlineMs ?? null,
408
- dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
409
- elapsedMs: diagnostics?.elapsedMs ?? null,
410
267
  error: error instanceof Error ? error.message : error,
411
- failedProcessId: diagnostics?.failedProcessId ?? null,
412
- lastOutput: lastLog?.line ?? null,
413
- lastOutputAt: lastLog?.at ?? null,
414
268
  phase,
415
- probes: diagnostics?.probes ?? null,
416
- readyProcesses,
417
- releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
418
- releaseId: this.releaseId,
419
- stage: diagnostics?.stage ?? null,
420
- unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
269
+ releaseId: this.releaseId
421
270
  })
422
271
 
423
272
  for (const processInstance of this.processes.values()) {
@@ -442,14 +291,18 @@ export default class ReleaseGroup extends EventEmitter {
442
291
  * @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
443
292
  */
444
293
  releaseProcessStartOrder() {
445
- return processStartOrder(this.config.processes)
294
+ const releaseProcesses = this.config.processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
295
+ const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
296
+ const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
297
+ const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
298
+
299
+ return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
446
300
  }
447
301
 
448
302
  /** @returns {void} Marks this release active. */
449
303
  activate() {
450
304
  this.state = "active"
451
305
  this.activatedAt = new Date().toISOString()
452
- this.clearDrainLivenessSamples()
453
306
  }
454
307
 
455
308
  /** Runs the configured release-generation activation acknowledgement, if any. */
@@ -504,16 +357,6 @@ export default class ReleaseGroup extends EventEmitter {
504
357
  this.drainStartedAt = undefined
505
358
  this.retirementError = undefined
506
359
  this.stoppedAt = undefined
507
- this.clearDrainLivenessSamples()
508
- }
509
-
510
- /** Stops only the proxied processes when a failed compensation cannot restore jobs. */
511
- async stopProxyTraffic() {
512
- const proxiedIds = new Set(this.config.processes.filter((processConfig) => processConfig.policy === "proxied").map((processConfig) => processConfig.id))
513
-
514
- await Promise.allSettled([...this.processes.entries()]
515
- .filter(([id]) => proxiedIds.has(id.replace(/#\d+$/, "")))
516
- .map(([, processInstance]) => processInstance.stop()))
517
360
  }
518
361
 
519
362
  /**
@@ -524,25 +367,24 @@ export default class ReleaseGroup extends EventEmitter {
524
367
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
525
368
  this.assertCommittedGenerationRecoverable()
526
369
  this.state = "starting"
527
- this.startupStartedAtMs = Date.now()
528
370
  try {
529
371
  for (const processConfig of this.releaseProcessStartOrder()) {
530
- await this.waitForConfiguredDependencies(processConfig)
531
372
  const instances = this.getProcesses(processConfig.id)
532
373
 
533
374
  if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
534
- for (const {id, process} of instances) {
375
+ for (const {process} of instances) {
535
376
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
536
377
  await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
537
- this.processLaunchAtMs.set(id, Date.now())
538
- this.startedProcessIds.add(id)
539
378
  }
540
379
 
541
380
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
542
- await this.waitForProxiedHealth(processConfig)
381
+ await waitForHealth({
382
+ health: processConfig.health,
383
+ host: this.config.proxy.upstreamHost,
384
+ port: this.ports[processConfig.id]
385
+ })
543
386
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
544
387
  }
545
- for (const {id} of instances) this.readyProcessIds.add(id)
546
388
  }
547
389
  } catch (error) {
548
390
  await this.abortCommittedGenerationRestart()
@@ -634,10 +476,6 @@ export default class ReleaseGroup extends EventEmitter {
634
476
  ...this.baseEnvironment(processConfig, {count, index}),
635
477
  ...renderedEnv
636
478
  }
637
- const dependencyRetired = () => processConfig.dependsOn.some((dependencyId) => {
638
- const dependency = this.processes.get(dependencyId)
639
- return dependency?.lifecycleRole === "retired" || dependency?.status().lifecycleRole === "retired"
640
- })
641
479
 
642
480
  return {
643
481
  command: renderTemplate(processConfig.command, context),
@@ -650,15 +488,7 @@ export default class ReleaseGroup extends EventEmitter {
650
488
  outputLines: processConfig.outputLines,
651
489
  restart: processConfig.restart,
652
490
  restartDelayMs: processConfig.restartDelayMs,
653
- shouldRestart: options.shouldRestart || (() => {
654
- if (this.state !== "active" && this.state !== "starting") return false
655
-
656
- // A companion whose same-generation service dependency has retired can never
657
- // satisfy its startup gate again. Treat that dependency lifecycle as terminal
658
- // for this process so an exit does not become an endless supervisor restart loop.
659
- return !dependencyRetired()
660
- }),
661
- shouldStop: options.shouldStop || dependencyRetired,
491
+ shouldRestart: options.shouldRestart || (() => this.state === "active" || this.state === "starting"),
662
492
  stopSignal: processConfig.stopSignal,
663
493
  stopTimeoutMs: processConfig.gracefulStopMs
664
494
  }
@@ -878,21 +708,10 @@ export default class ReleaseGroup extends EventEmitter {
878
708
  // their lifecycle drain runs as soon as the release is retired — in parallel with the
879
709
  // connection drain, not held until after it. The rest stop once connections have closed.
880
710
  const entries = [...this.processes.entries()]
881
- const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
882
- id,
883
- promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
884
- }))
711
+ const stopOptions = {timeoutMs: config.proxy.forceStopTimeoutMs}
712
+ const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([, processInstance]) => processInstance.stop(stopOptions))
885
713
  const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
886
714
  const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
887
- // Bracket the retirement wait with a pair of observations: this one, and the one taken when
888
- // every stop has returned. The gap between them is the wait the transition already performs
889
- // (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
890
- // itself rather than from a sampler running alongside it.
891
- const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
892
- ? this.takeLivenessSample(Date.now())
893
- : undefined
894
-
895
- if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
896
715
 
897
716
  if (this.connectionCount > 0) {
898
717
  await new Promise((resolve) => {
@@ -911,39 +730,17 @@ export default class ReleaseGroup extends EventEmitter {
911
730
  }
912
731
 
913
732
  if (this.ownerHandoffPaused) {
914
- void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
733
+ void Promise.allSettled(nonBlockingStops)
915
734
  return
916
735
  }
917
736
 
918
- await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop()))
919
- const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
920
- nonBlockingResults.forEach((result, index) => {
921
- if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
922
- const processInstance = this.processes.get(nonBlockingStops[index].id)
923
- this.logger("release non-blocking process still draining", {
924
- pid: processInstance?.status().pid ?? null,
925
- processId: nonBlockingStops[index].id,
926
- releaseId: this.releaseId,
927
- timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
928
- })
929
- }
930
- })
931
- await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop()))
932
-
933
- // Last act of the retirement transition, before the release is recorded stopped: whatever
934
- // is still alive here is the residue, and the bracket pair above can now say whether it is
935
- // wedged. A generation that stops with a live process group is the state that made
936
- // operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
937
- // it does not end silently with the process still holding the release.
938
- if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
939
-
737
+ await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop(stopOptions)))
738
+ await Promise.allSettled(nonBlockingStops)
739
+ await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop(stopOptions)))
940
740
  this.state = "stopped"
941
741
  this.stoppedAt = new Date().toISOString()
942
742
  this.releasePortReservations()
943
743
  this.abandonLocalConnections()
944
- // The bracket pair is deliberately kept: a generation that stopped with a live process
945
- // group stays observable, and the deploy path needs the older observation to prove the
946
- // residue is still flat rather than starting a fresh window from zero.
947
744
  }
948
745
 
949
746
  /**
@@ -968,6 +765,7 @@ export default class ReleaseGroup extends EventEmitter {
968
765
  const results = await Promise.allSettled([...this.processes.entries()]
969
766
  .filter(([id]) => generationIds.has(id))
970
767
  .map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
768
+
971
769
  const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
972
770
 
973
771
  if (errors.length > 0) {
@@ -978,44 +776,15 @@ export default class ReleaseGroup extends EventEmitter {
978
776
  this.retirementError = undefined
979
777
  }
980
778
 
981
- /**
982
- * Stops every process owned by the generation.
983
- * @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
984
- * supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
985
- * is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
986
- * `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
987
- * the wait holds the mutation fence and every later deploy is refused (STOE-2015).
988
- * @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
989
- */
990
- async stop(options = {}) {
991
- const {retirementTimeoutMs} = options
992
- /** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
993
- const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
994
- const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
995
-
996
- return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
997
- })
998
-
999
- await Promise.allSettled(stopTasks.map(({promise}) => promise))
1000
-
1001
- // A stop that stopped waiting leaves the process running. Name it, so the operator sees the
1002
- // residue instead of a command that produced nothing.
1003
- for (const {bounded, id, instance} of stopTasks) {
1004
- if (!bounded || !instance.retirementTimedOut) continue
1005
-
1006
- this.logger("release stop gave up waiting for a retiring process", {
1007
- pid: instance.status().pid ?? null,
1008
- processId: id,
1009
- releaseId: this.releaseId,
1010
- timeoutMs: retirementTimeoutMs
1011
- })
1012
- }
779
+ /** @returns {Promise<void>} Stops all release-owned processes. */
780
+ async stop() {
781
+ const stopTasks = [...this.processes.values()].map((processInstance) => processInstance.stop())
1013
782
 
783
+ await Promise.allSettled(stopTasks)
1014
784
  this.state = "stopped"
1015
785
  this.stoppedAt = new Date().toISOString()
1016
786
  this.releasePortReservations()
1017
787
  this.abandonLocalConnections()
1018
- this.clearDrainLivenessSamples()
1019
788
  }
1020
789
 
1021
790
  /** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
@@ -1023,171 +792,12 @@ export default class ReleaseGroup extends EventEmitter {
1023
792
  await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
1024
793
  }
1025
794
 
1026
- /**
1027
- * Records one liveness observation for a draining generation and classifies its drain.
1028
- *
1029
- * Two observations are needed to prove CPU flatness, and reading a process group means
1030
- * scanning procfs, so sampling is bounded three ways: a generation that still owns live
1031
- * connections is classified without touching procfs at all (connections alone settle it), a
1032
- * new observation is taken only once the newest one is already a full window behind the
1033
- * present read, and every other read reuses the newest observation. A group that is not
1034
- * draining reports no drain.
1035
- * @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
1036
- */
1037
- drainStatus() {
1038
- if (this.state !== "draining") return undefined
1039
-
1040
- return this.observeRetirement({nowMs: Date.now()}).classification
1041
- }
1042
-
1043
- /**
1044
- * Records the newest liveness observation of this generation and classifies its retirement.
1045
- * @param {object} options - Observation options.
1046
- * @param {number} options.nowMs - Read time.
1047
- * @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
1048
- * @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
1049
- */
1050
- observeRetirement({nowMs, previousSample}) {
1051
- const drainBudgetMs = this.config.proxy.drainTimeoutMs
1052
- const drainElapsedMs = this.drainStartedAt === undefined ? 0 : nowMs - Date.parse(this.drainStartedAt)
1053
-
1054
- if (previousSample === undefined && this.connectionCount > 0) {
1055
- const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
1056
-
1057
- return {
1058
- classification: classifyDrainProgress({
1059
- connectionCount: this.connectionCount,
1060
- currentSample,
1061
- drainBudgetMs,
1062
- drainElapsedMs,
1063
- minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1064
- previousSample: undefined
1065
- }),
1066
- currentSample
1067
- }
1068
- }
1069
-
1070
- const newest = this.drainNewestSample
1071
- const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
1072
- ? this.takeLivenessSample(nowMs)
1073
- : newest
1074
- const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
1075
-
1076
- if (currentSample !== newest) {
1077
- this.drainPriorSample = newest
1078
- this.drainNewestSample = currentSample
1079
- }
1080
-
1081
- return {
1082
- classification: classifyDrainProgress({
1083
- connectionCount: this.connectionCount,
1084
- currentSample,
1085
- drainBudgetMs,
1086
- drainElapsedMs,
1087
- minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1088
- previousSample: priorSample
1089
- }),
1090
- currentSample
1091
- }
1092
- }
1093
-
1094
- /**
1095
- * Applies the wedge escalation to this generation: terminates the processes a proven wedge
1096
- * proves are holding nothing, and names the ones it refused to terminate.
1097
- *
1098
- * Both halves are required. The card this path was written for (STOE-1687) is about a
1099
- * wedged worker holding a release open for four hours while deploys stack behind it; the
1100
- * boundary on the same card (STOE-1077) is that no process still holding live work may ever
1101
- * be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
1102
- * with the pids and process groups it is holding, which is the information an operator needs
1103
- * in place of guessing.
1104
- * @param {object} options - Escalation context.
1105
- * @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
1106
- * @param {string} options.reason - Named context recorded with the decision.
1107
- * @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
1108
- */
1109
- async escalateWedgedGeneration({previousSample, reason}) {
1110
- const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
1111
- const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
1112
- const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
1113
-
1114
- if (decision.decision === "report") {
1115
- if (classification.status === "wedged") {
1116
- this.logger("release wedged generation left alive", {
1117
- escalation: "refused",
1118
- heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
1119
- pids: liveObservations.map((process) => process.pid ?? null),
1120
- processIds: liveObservations.map((process) => process.id),
1121
- reason: decision.reason,
1122
- releaseId: this.releaseId,
1123
- trigger: reason
1124
- })
1125
- }
1126
-
1127
- return []
1128
- }
1129
-
1130
- const escalated = []
1131
-
1132
- for (const target of decision.targets) {
1133
- const processInstance = this.processes.get(target.processId)
1134
-
1135
- if (!processInstance) continue
1136
- if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
1137
- }
1138
-
1139
- if (escalated.length > 0) {
1140
- this.logger("release wedged generation escalated", {
1141
- escalation: "SIGKILL",
1142
- processIds: escalated,
1143
- reason: decision.reason,
1144
- releaseId: this.releaseId,
1145
- trigger: reason
1146
- })
1147
- }
1148
-
1149
- return escalated
1150
- }
1151
-
1152
- /**
1153
- * @param {number} nowMs - Observation time.
1154
- * @returns {DrainLivenessSample} One liveness observation of every process in the generation.
1155
- */
1156
- takeLivenessSample(nowMs) {
1157
- const processes = [...this.processes.entries()].map(([id, processInstance]) => {
1158
- const processStatus = processInstance.status()
1159
- const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
1160
- // A group that outlived its tracked process is still observed by group id: it holds the
1161
- // generation's work even though no rollbridge record can signal it any more.
1162
- const pgid = processStatus.pid ?? heldProcessGroupPgid
1163
- const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
1164
-
1165
- return {
1166
- cpuMs: sample?.cpuMs,
1167
- heldProcessGroupPgid,
1168
- id,
1169
- liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
1170
- pid: processStatus.pid,
1171
- retirementTimedOut: processInstance.retirementTimedOut === true
1172
- }
1173
- })
1174
-
1175
- return {atMs: nowMs, processes}
1176
- }
1177
-
1178
- /** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
1179
- clearDrainLivenessSamples() {
1180
- this.drainNewestSample = undefined
1181
- this.drainPriorSample = undefined
1182
- }
1183
-
1184
795
  /** @returns {ReleaseStatus} Status payload. */
1185
796
  status() {
1186
797
  return {
1187
798
  activatedAt: this.activatedAt,
1188
799
  connectionCount: this.connectionCount,
1189
800
  connections: {...this.connections},
1190
- drain: this.drainStatus(),
1191
801
  drainStartedAt: this.drainStartedAt,
1192
802
  ports: {...this.ports},
1193
803
  processes: [...this.processes.values()].map((processInstance) => processInstance.status()),
package/tensorbuzz.yml CHANGED
@@ -1,23 +1,4 @@
1
- # TensorBuzz build config for Rollbridge.
2
- #
3
- # `paths` (STOE-1179): this gate installs the dependency tree and runs typecheck, ESLint and the
4
- # whole Velocious suite, and every one of those reads code, `examples/`, and the package/tooling
5
- # manifests — no gate input is Markdown. A delivery that changes nothing but Markdown therefore
6
- # has nothing for this lane to check, and TensorBuzz records it as a terminal `skipped` build
7
- # instead of spending a container and ~3 minutes on it. `test/tensorbuzz-config.test.js` asserts
8
- # the split against every tracked path so a later edit cannot carve a gate input out of the lane.
9
- #
10
- # Ordered rules, last match wins, so the negation has to come after `"**"`. Quote every negation:
11
- # an unquoted `!x` is a YAML tag, and the config validator fails the whole file rather than
12
- # filtering anything. The lane still runs whenever `tensorbuzz.yml` itself changes, whenever the
13
- # changed-file list is missing, empty, or capped by the host, and on every manual rebuild.
14
- paths:
15
- - "**"
16
- - "!**/*.md"
17
-
18
1
  before_script:
19
2
  - npm install
20
3
  script:
21
- # Rollbridge's Unix control/guardian sockets must stay below the platform path limit;
22
- # the TensorBuzz runner's workspace TMPDIR is deliberately long.
23
- - TMPDIR=/tmp npm run all-checks
4
+ - npm run all-checks