rollbridge 0.1.70 → 0.1.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +10 -63
  2. package/TODO.md +0 -13
  3. package/docs/cli.md +3 -128
  4. package/docs/config.md +6 -177
  5. package/docs/troubleshooting.md +6 -97
  6. package/docs/velocious.md +1 -1
  7. package/docs/workers.md +0 -11
  8. package/examples/tensorbuzz.com.js +2 -3
  9. package/package.json +1 -1
  10. package/src/cli.js +2 -68
  11. package/src/config.js +8 -347
  12. package/src/daemon.js +65 -387
  13. package/src/doctor.js +0 -74
  14. package/src/guardian-client.js +8 -67
  15. package/src/health.js +8 -99
  16. package/src/managed-process.js +16 -197
  17. package/src/process-guardian.js +22 -212
  18. package/src/process-memory.js +2 -62
  19. package/src/release-group.js +37 -435
  20. package/tensorbuzz.yml +1 -20
  21. package/test/completion.test.js +2 -5
  22. package/test/config-validation.test.js +5 -135
  23. package/test/guardian-client.test.js +5 -119
  24. package/test/health.test.js +1 -74
  25. package/test/managed-process.test.js +1 -260
  26. package/test/owner-recovery.test.js +21 -125
  27. package/test/owner-replacement.test.js +14 -175
  28. package/test/process-memory.test.js +1 -47
  29. package/test/release-group.test.js +2 -408
  30. package/test/rollbridge.test.js +72 -737
  31. package/test/shutdown-completion.test.js +0 -52
  32. package/test/support/process.js +1 -3
  33. package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
  34. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
  35. package/changelog.d/20261104-compose-service-units.md +0 -21
  36. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
  37. package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
  38. package/changelog.d/20261109-path-filtered-builds.md +0 -11
  39. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
  40. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
  41. package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
  42. package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
  43. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
  44. package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
  45. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
  46. package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
  47. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
  48. package/changelog.d/20261116-guardian-version-handoff.md +0 -37
  49. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
  50. package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
  51. package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
  52. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
  53. package/docs/compose-services.md +0 -222
  54. package/src/compose-unit.js +0 -725
  55. package/src/drain-policy.js +0 -252
  56. package/src/process-order.js +0 -26
  57. package/test/compose-config.test.js +0 -160
  58. package/test/compose-docker.test.js +0 -297
  59. package/test/compose-unit.test.js +0 -415
  60. package/test/drain-policy.test.js +0 -328
  61. package/test/fixtures/deferred-boot-app.js +0 -106
  62. package/test/fixtures/gated-app.js +0 -50
  63. package/test/tensorbuzz-config.test.js +0 -228
@@ -2,29 +2,17 @@
2
2
 
3
3
  import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
- import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
6
5
  import ManagedProcess from "./managed-process.js"
7
6
  import {findAvailablePort} from "./port-allocator.js"
8
- import {processStartOrder} from "./process-order.js"
9
- import {sampleProcessGroup} from "./process-memory.js"
10
7
  import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
11
- import {waitForHealth, waitForListeningPort} from "./health.js"
12
-
13
- const DEPENDENCY_READY_INTERVAL_MS = 250
14
- // Shortest gap between two liveness observations that can prove a draining group's CPU is
15
- // flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
16
- // with live runner children doing work, so flatness is only evidence below this window.
17
- const DRAIN_SAMPLE_WINDOW_MS = 20000
8
+ import {waitForHealth} from "./health.js"
18
9
 
19
10
  /**
20
11
  * @typedef {import("./json.js").JsonValue} JsonValue
21
12
  * @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
22
- * @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
23
- * @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
24
13
  * @typedef {{http: number, websocket: number}} ReleaseConnections
25
- * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
26
- * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean, shouldStop?: () => boolean}} BuildProcessOptions
27
- * @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
14
+ * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
15
+ * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
28
16
  */
29
17
 
30
18
 
@@ -69,14 +57,6 @@ function envId(id) {
69
57
  return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
70
58
  }
71
59
 
72
- /**
73
- * @param {number | undefined} startedAtMs - When the measured phase began.
74
- * @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
75
- */
76
- function elapsedSince(startedAtMs) {
77
- return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
78
- }
79
-
80
60
  /**
81
61
  * @param {import("./config.js").ProcessConfig} processConfig - Process config.
82
62
  * @param {number} index - Zero-based replica index.
@@ -97,10 +77,9 @@ export default class ReleaseGroup extends EventEmitter {
97
77
  * @param {string | undefined} args.revision - Revision.
98
78
  * @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
99
79
  * @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
100
- * @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
101
80
  * @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
102
81
  */
103
- constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
82
+ constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
104
83
  super()
105
84
 
106
85
  this.config = config
@@ -122,36 +101,22 @@ export default class ReleaseGroup extends EventEmitter {
122
101
  this.servicePorts = servicePorts
123
102
  this.shouldStart = shouldStart
124
103
  this.processFactory = processFactory
125
- this.procPath = procPath
126
104
  this.portsAllocated = false
127
105
  this.drainStartedAt = /** @type {string | undefined} */ (undefined)
128
- this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
129
- this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
130
106
  this.activatedAt = /** @type {string | undefined} */ (undefined)
131
107
  this.stoppedAt = /** @type {string | undefined} */ (undefined)
132
108
  this.retirementError = /** @type {string | undefined} */ (undefined)
133
109
  this.preserveConfigOnRetirement = false
134
- this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
135
- this.readyProcessIds = /** @type {Set<string>} */ (new Set())
136
- this.startedProcessIds = /** @type {Set<string>} */ (new Set())
137
- this.startupDependencyWaitMs = 0
138
- this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
139
- this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
140
110
  }
141
111
 
142
112
  /** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
143
113
  async start() {
144
114
  this.state = "starting"
145
- this.startupStartedAtMs = Date.now()
146
115
 
147
116
  try {
148
117
  await this.allocatePorts()
149
118
 
150
119
  for (const processConfig of this.releaseProcessStartOrder()) {
151
- await this.waitForConfiguredDependencies(processConfig)
152
- /** @type {string[]} */
153
- const instanceIds = []
154
-
155
120
  for (let index = 0; index < processConfig.replicas; index += 1) {
156
121
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
157
122
 
@@ -162,17 +127,17 @@ export default class ReleaseGroup extends EventEmitter {
162
127
  if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
163
128
  if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
164
129
  await processInstance.start("deploy")
165
- this.processLaunchAtMs.set(instanceId, Date.now())
166
- this.startedProcessIds.add(instanceId)
167
- instanceIds.push(instanceId)
168
130
  }
169
131
 
170
132
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
171
- await this.waitForProxiedHealth(processConfig)
172
- }
133
+ await waitForHealth({
134
+ health: processConfig.health,
135
+ host: this.config.proxy.upstreamHost,
136
+ port: this.ports[processConfig.id]
137
+ })
173
138
 
174
- for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
175
- if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
139
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
140
+ }
176
141
  }
177
142
  } catch (error) {
178
143
  this.state = "failed"
@@ -292,132 +257,16 @@ export default class ReleaseGroup extends EventEmitter {
292
257
  }
293
258
 
294
259
  /**
295
- * Waits for the ports a process declares it depends on, on a budget that is not the
296
- * health budget. The dependency leg of a start is not app boot, and charging it to the
297
- * health deadline measures the dependency instead of the app.
298
- * @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
299
- * @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
300
- */
301
- async waitForConfiguredDependencies(processConfig) {
302
- if (processConfig.dependsOn.length === 0) return
303
-
304
- for (const dependencyId of processConfig.dependsOn) {
305
- const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
306
-
307
- if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
308
-
309
- this.logger("release process dependency wait", {
310
- dependencyPort: port,
311
- dependencyProcessId: dependencyId,
312
- processId: processConfig.id,
313
- releaseId: this.releaseId,
314
- timeoutMs: processConfig.dependencyTimeoutMs
315
- })
316
-
317
- const waitStartedAtMs = Date.now()
318
-
319
- try {
320
- const {attempts, elapsedMs} = await waitForListeningPort({
321
- host: this.config.proxy.upstreamHost,
322
- intervalMs: DEPENDENCY_READY_INTERVAL_MS,
323
- port,
324
- timeoutMs: processConfig.dependencyTimeoutMs
325
- })
326
-
327
- this.startupDependencyWaitMs += elapsedMs
328
- this.logger("release process dependency ready", {
329
- attempts,
330
- dependencyPort: port,
331
- dependencyProcessId: dependencyId,
332
- elapsedMs,
333
- processId: processConfig.id,
334
- releaseId: this.releaseId
335
- })
336
- } catch (error) {
337
- this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
338
- deadlineMs: processConfig.dependencyTimeoutMs,
339
- dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
340
- elapsedMs: elapsedSince(this.startupStartedAtMs),
341
- failedProcessId: processConfig.id,
342
- probes: undefined,
343
- stage: "dependency"
344
- })
345
- throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
346
- }
347
-
348
- if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
349
- }
350
- }
351
-
352
- /**
353
- * Health checks the proxied process on its own boot budget and records what the boot cost.
354
- * @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
355
- * @returns {Promise<void>} Resolves once the process answers its health path.
356
- */
357
- async waitForProxiedHealth(processConfig) {
358
- const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
359
- const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
360
-
361
- try {
362
- const {elapsedMs} = await waitForHealth({
363
- health,
364
- host: this.config.proxy.upstreamHost,
365
- port: /** @type {number} */ (this.ports[processConfig.id]),
366
- processId: processConfig.id
367
- })
368
-
369
- this.logger("release candidate health ready", {
370
- bootMs: Date.now() - launchedAtMs,
371
- deadlineMs: health.timeoutMs,
372
- healthGateMs: elapsedMs,
373
- path: health.path,
374
- port: this.ports[processConfig.id],
375
- processId: processConfig.id,
376
- releaseId: this.releaseId,
377
- waitedForDependenciesMs: this.startupDependencyWaitMs
378
- })
379
- } catch (error) {
380
- const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
381
-
382
- this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
383
- deadlineMs: health.timeoutMs,
384
- dependencyWaitMs: this.startupDependencyWaitMs,
385
- elapsedMs: elapsedSince(this.startupStartedAtMs),
386
- failedProcessId: processConfig.id,
387
- probes: healthCheck?.attempts,
388
- stage: "health"
389
- })
390
- throw error
391
- }
392
- }
393
-
394
- /**
395
- * Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
260
+ * Logs process diagnostics around failed startup cleanup.
396
261
  * @param {Error | string} error - Startup failure.
397
262
  * @param {{phase: string}} options - Diagnostic phase.
398
263
  * @returns {void}
399
264
  */
400
265
  logStartupFailure(error, {phase}) {
401
- const diagnostics = this.startupFailureDiagnostics
402
- const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
403
- const lastLog = failedLogs?.at(-1)
404
- const readyProcesses = [...this.readyProcessIds]
405
-
406
266
  this.logger("release startup failed", {
407
- deadlineMs: diagnostics?.deadlineMs ?? null,
408
- dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
409
- elapsedMs: diagnostics?.elapsedMs ?? null,
410
267
  error: error instanceof Error ? error.message : error,
411
- failedProcessId: diagnostics?.failedProcessId ?? null,
412
- lastOutput: lastLog?.line ?? null,
413
- lastOutputAt: lastLog?.at ?? null,
414
268
  phase,
415
- probes: diagnostics?.probes ?? null,
416
- readyProcesses,
417
- releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
418
- releaseId: this.releaseId,
419
- stage: diagnostics?.stage ?? null,
420
- unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
269
+ releaseId: this.releaseId
421
270
  })
422
271
 
423
272
  for (const processInstance of this.processes.values()) {
@@ -442,14 +291,18 @@ export default class ReleaseGroup extends EventEmitter {
442
291
  * @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
443
292
  */
444
293
  releaseProcessStartOrder() {
445
- return processStartOrder(this.config.processes)
294
+ const releaseProcesses = this.config.processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
295
+ const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
296
+ const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
297
+ const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
298
+
299
+ return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
446
300
  }
447
301
 
448
302
  /** @returns {void} Marks this release active. */
449
303
  activate() {
450
304
  this.state = "active"
451
305
  this.activatedAt = new Date().toISOString()
452
- this.clearDrainLivenessSamples()
453
306
  }
454
307
 
455
308
  /** Runs the configured release-generation activation acknowledgement, if any. */
@@ -504,16 +357,6 @@ export default class ReleaseGroup extends EventEmitter {
504
357
  this.drainStartedAt = undefined
505
358
  this.retirementError = undefined
506
359
  this.stoppedAt = undefined
507
- this.clearDrainLivenessSamples()
508
- }
509
-
510
- /** Stops only the proxied processes when a failed compensation cannot restore jobs. */
511
- async stopProxyTraffic() {
512
- const proxiedIds = new Set(this.config.processes.filter((processConfig) => processConfig.policy === "proxied").map((processConfig) => processConfig.id))
513
-
514
- await Promise.allSettled([...this.processes.entries()]
515
- .filter(([id]) => proxiedIds.has(id.replace(/#\d+$/, "")))
516
- .map(([, processInstance]) => processInstance.stop()))
517
360
  }
518
361
 
519
362
  /**
@@ -524,25 +367,24 @@ export default class ReleaseGroup extends EventEmitter {
524
367
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
525
368
  this.assertCommittedGenerationRecoverable()
526
369
  this.state = "starting"
527
- this.startupStartedAtMs = Date.now()
528
370
  try {
529
371
  for (const processConfig of this.releaseProcessStartOrder()) {
530
- await this.waitForConfiguredDependencies(processConfig)
531
372
  const instances = this.getProcesses(processConfig.id)
532
373
 
533
374
  if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
534
- for (const {id, process} of instances) {
375
+ for (const {process} of instances) {
535
376
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
536
377
  await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
537
- this.processLaunchAtMs.set(id, Date.now())
538
- this.startedProcessIds.add(id)
539
378
  }
540
379
 
541
380
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
542
- await this.waitForProxiedHealth(processConfig)
381
+ await waitForHealth({
382
+ health: processConfig.health,
383
+ host: this.config.proxy.upstreamHost,
384
+ port: this.ports[processConfig.id]
385
+ })
543
386
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
544
387
  }
545
- for (const {id} of instances) this.readyProcessIds.add(id)
546
388
  }
547
389
  } catch (error) {
548
390
  await this.abortCommittedGenerationRestart()
@@ -634,10 +476,6 @@ export default class ReleaseGroup extends EventEmitter {
634
476
  ...this.baseEnvironment(processConfig, {count, index}),
635
477
  ...renderedEnv
636
478
  }
637
- const dependencyRetired = () => processConfig.dependsOn.some((dependencyId) => {
638
- const dependency = this.processes.get(dependencyId)
639
- return dependency?.lifecycleRole === "retired" || dependency?.status().lifecycleRole === "retired"
640
- })
641
479
 
642
480
  return {
643
481
  command: renderTemplate(processConfig.command, context),
@@ -650,15 +488,7 @@ export default class ReleaseGroup extends EventEmitter {
650
488
  outputLines: processConfig.outputLines,
651
489
  restart: processConfig.restart,
652
490
  restartDelayMs: processConfig.restartDelayMs,
653
- shouldRestart: options.shouldRestart || (() => {
654
- if (this.state !== "active" && this.state !== "starting") return false
655
-
656
- // A companion whose same-generation service dependency has retired can never
657
- // satisfy its startup gate again. Treat that dependency lifecycle as terminal
658
- // for this process so an exit does not become an endless supervisor restart loop.
659
- return !dependencyRetired()
660
- }),
661
- shouldStop: options.shouldStop || dependencyRetired,
491
+ shouldRestart: options.shouldRestart || (() => this.state === "active" || this.state === "starting"),
662
492
  stopSignal: processConfig.stopSignal,
663
493
  stopTimeoutMs: processConfig.gracefulStopMs
664
494
  }
@@ -878,21 +708,10 @@ export default class ReleaseGroup extends EventEmitter {
878
708
  // their lifecycle drain runs as soon as the release is retired — in parallel with the
879
709
  // connection drain, not held until after it. The rest stop once connections have closed.
880
710
  const entries = [...this.processes.entries()]
881
- const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
882
- id,
883
- promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
884
- }))
711
+ const stopOptions = {timeoutMs: config.proxy.forceStopTimeoutMs}
712
+ const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([, processInstance]) => processInstance.stop(stopOptions))
885
713
  const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
886
714
  const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
887
- // Bracket the retirement wait with a pair of observations: this one, and the one taken when
888
- // every stop has returned. The gap between them is the wait the transition already performs
889
- // (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
890
- // itself rather than from a sampler running alongside it.
891
- const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
892
- ? this.takeLivenessSample(Date.now())
893
- : undefined
894
-
895
- if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
896
715
 
897
716
  if (this.connectionCount > 0) {
898
717
  await new Promise((resolve) => {
@@ -911,40 +730,17 @@ export default class ReleaseGroup extends EventEmitter {
911
730
  }
912
731
 
913
732
  if (this.ownerHandoffPaused) {
914
- void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
733
+ void Promise.allSettled(nonBlockingStops)
915
734
  return
916
735
  }
917
736
 
918
- const retirementTimeoutMs = config.proxy.nonBlockingDrainTimeoutMs
919
- await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
920
- const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
921
- nonBlockingResults.forEach((result, index) => {
922
- if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
923
- const processInstance = this.processes.get(nonBlockingStops[index].id)
924
- this.logger("release non-blocking process still draining", {
925
- pid: processInstance?.status().pid ?? null,
926
- processId: nonBlockingStops[index].id,
927
- releaseId: this.releaseId,
928
- timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
929
- })
930
- }
931
- })
932
- await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
933
-
934
- // Last act of the retirement transition, before the release is recorded stopped: whatever
935
- // is still alive here is the residue, and the bracket pair above can now say whether it is
936
- // wedged. A generation that stops with a live process group is the state that made
937
- // operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
938
- // it does not end silently with the process still holding the release.
939
- if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
940
-
737
+ await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop(stopOptions)))
738
+ await Promise.allSettled(nonBlockingStops)
739
+ await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop(stopOptions)))
941
740
  this.state = "stopped"
942
741
  this.stoppedAt = new Date().toISOString()
943
742
  this.releasePortReservations()
944
743
  this.abandonLocalConnections()
945
- // The bracket pair is deliberately kept: a generation that stopped with a live process
946
- // group stays observable, and the deploy path needs the older observation to prove the
947
- // residue is still flat rather than starting a fresh window from zero.
948
744
  }
949
745
 
950
746
  /**
@@ -964,18 +760,12 @@ export default class ReleaseGroup extends EventEmitter {
964
760
  this.drainStartedAt = new Date().toISOString()
965
761
  }
966
762
 
967
- // Refresh the live process definitions with the current retirement policy,
968
- // but retain this generation's exact config identity for owner recovery.
969
- // Replacing the stored generation config here would make a still-running
970
- // singleton from the previous release fail the guardian provenance fence.
971
- const generationConfig = this.config
972
-
973
763
  this.refreshProcessDefinitions(config)
974
- this.config = generationConfig
975
764
  const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
976
765
  const results = await Promise.allSettled([...this.processes.entries()]
977
766
  .filter(([id]) => generationIds.has(id))
978
767
  .map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
768
+
979
769
  const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
980
770
 
981
771
  if (errors.length > 0) {
@@ -986,44 +776,15 @@ export default class ReleaseGroup extends EventEmitter {
986
776
  this.retirementError = undefined
987
777
  }
988
778
 
989
- /**
990
- * Stops every process owned by the generation.
991
- * @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
992
- * supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
993
- * is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
994
- * `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
995
- * the wait holds the mutation fence and every later deploy is refused (STOE-2015).
996
- * @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
997
- */
998
- async stop(options = {}) {
999
- const {retirementTimeoutMs} = options
1000
- /** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
1001
- const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
1002
- const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
1003
-
1004
- return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
1005
- })
1006
-
1007
- await Promise.allSettled(stopTasks.map(({promise}) => promise))
1008
-
1009
- // A stop that stopped waiting leaves the process running. Name it, so the operator sees the
1010
- // residue instead of a command that produced nothing.
1011
- for (const {bounded, id, instance} of stopTasks) {
1012
- if (!bounded || !instance.retirementTimedOut) continue
1013
-
1014
- this.logger("release stop gave up waiting for a retiring process", {
1015
- pid: instance.status().pid ?? null,
1016
- processId: id,
1017
- releaseId: this.releaseId,
1018
- timeoutMs: retirementTimeoutMs
1019
- })
1020
- }
779
+ /** @returns {Promise<void>} Stops all release-owned processes. */
780
+ async stop() {
781
+ const stopTasks = [...this.processes.values()].map((processInstance) => processInstance.stop())
1021
782
 
783
+ await Promise.allSettled(stopTasks)
1022
784
  this.state = "stopped"
1023
785
  this.stoppedAt = new Date().toISOString()
1024
786
  this.releasePortReservations()
1025
787
  this.abandonLocalConnections()
1026
- this.clearDrainLivenessSamples()
1027
788
  }
1028
789
 
1029
790
  /** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
@@ -1031,171 +792,12 @@ export default class ReleaseGroup extends EventEmitter {
1031
792
  await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
1032
793
  }
1033
794
 
1034
- /**
1035
- * Records one liveness observation for a draining generation and classifies its drain.
1036
- *
1037
- * Two observations are needed to prove CPU flatness, and reading a process group means
1038
- * scanning procfs, so sampling is bounded three ways: a generation that still owns live
1039
- * connections is classified without touching procfs at all (connections alone settle it), a
1040
- * new observation is taken only once the newest one is already a full window behind the
1041
- * present read, and every other read reuses the newest observation. A group that is not
1042
- * draining reports no drain.
1043
- * @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
1044
- */
1045
- drainStatus() {
1046
- if (this.state !== "draining") return undefined
1047
-
1048
- return this.observeRetirement({nowMs: Date.now()}).classification
1049
- }
1050
-
1051
- /**
1052
- * Records the newest liveness observation of this generation and classifies its retirement.
1053
- * @param {object} options - Observation options.
1054
- * @param {number} options.nowMs - Read time.
1055
- * @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
1056
- * @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
1057
- */
1058
- observeRetirement({nowMs, previousSample}) {
1059
- const drainBudgetMs = this.config.proxy.drainTimeoutMs
1060
- const drainElapsedMs = this.drainStartedAt === undefined ? 0 : nowMs - Date.parse(this.drainStartedAt)
1061
-
1062
- if (previousSample === undefined && this.connectionCount > 0) {
1063
- const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
1064
-
1065
- return {
1066
- classification: classifyDrainProgress({
1067
- connectionCount: this.connectionCount,
1068
- currentSample,
1069
- drainBudgetMs,
1070
- drainElapsedMs,
1071
- minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1072
- previousSample: undefined
1073
- }),
1074
- currentSample
1075
- }
1076
- }
1077
-
1078
- const newest = this.drainNewestSample
1079
- const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
1080
- ? this.takeLivenessSample(nowMs)
1081
- : newest
1082
- const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
1083
-
1084
- if (currentSample !== newest) {
1085
- this.drainPriorSample = newest
1086
- this.drainNewestSample = currentSample
1087
- }
1088
-
1089
- return {
1090
- classification: classifyDrainProgress({
1091
- connectionCount: this.connectionCount,
1092
- currentSample,
1093
- drainBudgetMs,
1094
- drainElapsedMs,
1095
- minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1096
- previousSample: priorSample
1097
- }),
1098
- currentSample
1099
- }
1100
- }
1101
-
1102
- /**
1103
- * Applies the wedge escalation to this generation: terminates the processes a proven wedge
1104
- * proves are holding nothing, and names the ones it refused to terminate.
1105
- *
1106
- * Both halves are required. The card this path was written for (STOE-1687) is about a
1107
- * wedged worker holding a release open for four hours while deploys stack behind it; the
1108
- * boundary on the same card (STOE-1077) is that no process still holding live work may ever
1109
- * be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
1110
- * with the pids and process groups it is holding, which is the information an operator needs
1111
- * in place of guessing.
1112
- * @param {object} options - Escalation context.
1113
- * @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
1114
- * @param {string} options.reason - Named context recorded with the decision.
1115
- * @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
1116
- */
1117
- async escalateWedgedGeneration({previousSample, reason}) {
1118
- const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
1119
- const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
1120
- const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
1121
-
1122
- if (decision.decision === "report") {
1123
- if (classification.status === "wedged") {
1124
- this.logger("release wedged generation left alive", {
1125
- escalation: "refused",
1126
- heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
1127
- pids: liveObservations.map((process) => process.pid ?? null),
1128
- processIds: liveObservations.map((process) => process.id),
1129
- reason: decision.reason,
1130
- releaseId: this.releaseId,
1131
- trigger: reason
1132
- })
1133
- }
1134
-
1135
- return []
1136
- }
1137
-
1138
- const escalated = []
1139
-
1140
- for (const target of decision.targets) {
1141
- const processInstance = this.processes.get(target.processId)
1142
-
1143
- if (!processInstance) continue
1144
- if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
1145
- }
1146
-
1147
- if (escalated.length > 0) {
1148
- this.logger("release wedged generation escalated", {
1149
- escalation: "SIGKILL",
1150
- processIds: escalated,
1151
- reason: decision.reason,
1152
- releaseId: this.releaseId,
1153
- trigger: reason
1154
- })
1155
- }
1156
-
1157
- return escalated
1158
- }
1159
-
1160
- /**
1161
- * @param {number} nowMs - Observation time.
1162
- * @returns {DrainLivenessSample} One liveness observation of every process in the generation.
1163
- */
1164
- takeLivenessSample(nowMs) {
1165
- const processes = [...this.processes.entries()].map(([id, processInstance]) => {
1166
- const processStatus = processInstance.status()
1167
- const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
1168
- // A group that outlived its tracked process is still observed by group id: it holds the
1169
- // generation's work even though no rollbridge record can signal it any more.
1170
- const pgid = processStatus.pid ?? heldProcessGroupPgid
1171
- const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
1172
-
1173
- return {
1174
- cpuMs: sample?.cpuMs,
1175
- heldProcessGroupPgid,
1176
- id,
1177
- liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
1178
- pid: processStatus.pid,
1179
- retirementTimedOut: processInstance.retirementTimedOut === true
1180
- }
1181
- })
1182
-
1183
- return {atMs: nowMs, processes}
1184
- }
1185
-
1186
- /** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
1187
- clearDrainLivenessSamples() {
1188
- this.drainNewestSample = undefined
1189
- this.drainPriorSample = undefined
1190
- }
1191
-
1192
795
  /** @returns {ReleaseStatus} Status payload. */
1193
796
  status() {
1194
797
  return {
1195
798
  activatedAt: this.activatedAt,
1196
799
  connectionCount: this.connectionCount,
1197
800
  connections: {...this.connections},
1198
- drain: this.drainStatus(),
1199
801
  drainStartedAt: this.drainStartedAt,
1200
802
  ports: {...this.ports},
1201
803
  processes: [...this.processes.values()].map((processInstance) => processInstance.status()),