rollbridge 0.1.72 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  24. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  25. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  26. package/docs/cli.md +128 -3
  27. package/docs/compose-services.md +222 -0
  28. package/docs/config.md +177 -6
  29. package/docs/troubleshooting.md +97 -6
  30. package/docs/velocious.md +1 -1
  31. package/docs/workers.md +11 -0
  32. package/examples/tensorbuzz.com.js +3 -2
  33. package/package.json +1 -1
  34. package/src/cli.js +68 -2
  35. package/src/compose-unit.js +725 -0
  36. package/src/config.js +346 -8
  37. package/src/daemon.js +410 -73
  38. package/src/doctor.js +74 -0
  39. package/src/drain-policy.js +252 -0
  40. package/src/guardian-client.js +67 -8
  41. package/src/health.js +99 -8
  42. package/src/managed-process.js +197 -16
  43. package/src/process-guardian.js +212 -22
  44. package/src/process-memory.js +62 -2
  45. package/src/process-order.js +26 -0
  46. package/src/release-group.js +435 -37
  47. package/tensorbuzz.yml +20 -1
  48. package/test/completion.test.js +5 -2
  49. package/test/compose-config.test.js +160 -0
  50. package/test/compose-docker.test.js +297 -0
  51. package/test/compose-unit.test.js +415 -0
  52. package/test/config-validation.test.js +135 -5
  53. package/test/drain-policy.test.js +328 -0
  54. package/test/fixtures/deferred-boot-app.js +106 -0
  55. package/test/fixtures/gated-app.js +50 -0
  56. package/test/guardian-client.test.js +119 -5
  57. package/test/health.test.js +74 -1
  58. package/test/managed-process.test.js +260 -1
  59. package/test/owner-recovery.test.js +125 -21
  60. package/test/owner-replacement.test.js +175 -14
  61. package/test/process-memory.test.js +47 -1
  62. package/test/release-group.test.js +408 -2
  63. package/test/rollbridge.test.js +774 -72
  64. package/test/shutdown-completion.test.js +119 -0
  65. package/test/shutdown-retirement-bound.test.js +162 -0
  66. package/test/support/process.js +3 -1
  67. package/test/tensorbuzz-config.test.js +228 -0
@@ -2,17 +2,29 @@
2
2
 
3
3
  import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
+ import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
5
6
  import ManagedProcess from "./managed-process.js"
6
7
  import {findAvailablePort} from "./port-allocator.js"
8
+ import {processStartOrder} from "./process-order.js"
9
+ import {sampleProcessGroup} from "./process-memory.js"
7
10
  import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
8
- import {waitForHealth} from "./health.js"
11
+ import {waitForHealth, waitForListeningPort} from "./health.js"
12
+
13
+ const DEPENDENCY_READY_INTERVAL_MS = 250
14
+ // Shortest gap between two liveness observations that can prove a draining group's CPU is
15
+ // flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
16
+ // with live runner children doing work, so flatness is only evidence below this window.
17
+ const DRAIN_SAMPLE_WINDOW_MS = 20000
9
18
 
10
19
  /**
11
20
  * @typedef {import("./json.js").JsonValue} JsonValue
12
21
  * @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
22
+ * @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
23
+ * @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
13
24
  * @typedef {{http: number, websocket: number}} ReleaseConnections
14
- * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
15
- * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
25
+ * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
26
+ * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean, shouldStop?: () => boolean}} BuildProcessOptions
27
+ * @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
16
28
  */
17
29
 
18
30
 
@@ -57,6 +69,14 @@ function envId(id) {
57
69
  return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
58
70
  }
59
71
 
72
+ /**
73
+ * @param {number | undefined} startedAtMs - When the measured phase began.
74
+ * @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
75
+ */
76
+ function elapsedSince(startedAtMs) {
77
+ return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
78
+ }
79
+
60
80
  /**
61
81
  * @param {import("./config.js").ProcessConfig} processConfig - Process config.
62
82
  * @param {number} index - Zero-based replica index.
@@ -77,9 +97,10 @@ export default class ReleaseGroup extends EventEmitter {
77
97
  * @param {string | undefined} args.revision - Revision.
78
98
  * @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
79
99
  * @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
100
+ * @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
80
101
  * @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
81
102
  */
82
- constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
103
+ constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
83
104
  super()
84
105
 
85
106
  this.config = config
@@ -101,22 +122,36 @@ export default class ReleaseGroup extends EventEmitter {
101
122
  this.servicePorts = servicePorts
102
123
  this.shouldStart = shouldStart
103
124
  this.processFactory = processFactory
125
+ this.procPath = procPath
104
126
  this.portsAllocated = false
105
127
  this.drainStartedAt = /** @type {string | undefined} */ (undefined)
128
+ this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
129
+ this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
106
130
  this.activatedAt = /** @type {string | undefined} */ (undefined)
107
131
  this.stoppedAt = /** @type {string | undefined} */ (undefined)
108
132
  this.retirementError = /** @type {string | undefined} */ (undefined)
109
133
  this.preserveConfigOnRetirement = false
134
+ this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
135
+ this.readyProcessIds = /** @type {Set<string>} */ (new Set())
136
+ this.startedProcessIds = /** @type {Set<string>} */ (new Set())
137
+ this.startupDependencyWaitMs = 0
138
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
139
+ this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
110
140
  }
111
141
 
112
142
  /** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
113
143
  async start() {
114
144
  this.state = "starting"
145
+ this.startupStartedAtMs = Date.now()
115
146
 
116
147
  try {
117
148
  await this.allocatePorts()
118
149
 
119
150
  for (const processConfig of this.releaseProcessStartOrder()) {
151
+ await this.waitForConfiguredDependencies(processConfig)
152
+ /** @type {string[]} */
153
+ const instanceIds = []
154
+
120
155
  for (let index = 0; index < processConfig.replicas; index += 1) {
121
156
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
122
157
 
@@ -127,17 +162,17 @@ export default class ReleaseGroup extends EventEmitter {
127
162
  if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
128
163
  if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
129
164
  await processInstance.start("deploy")
165
+ this.processLaunchAtMs.set(instanceId, Date.now())
166
+ this.startedProcessIds.add(instanceId)
167
+ instanceIds.push(instanceId)
130
168
  }
131
169
 
132
170
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
133
- await waitForHealth({
134
- health: processConfig.health,
135
- host: this.config.proxy.upstreamHost,
136
- port: this.ports[processConfig.id]
137
- })
138
-
139
- if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
171
+ await this.waitForProxiedHealth(processConfig)
140
172
  }
173
+
174
+ for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
175
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
141
176
  }
142
177
  } catch (error) {
143
178
  this.state = "failed"
@@ -257,16 +292,132 @@ export default class ReleaseGroup extends EventEmitter {
257
292
  }
258
293
 
259
294
  /**
260
- * Logs process diagnostics around failed startup cleanup.
295
+ * Waits for the ports a process declares it depends on, on a budget that is not the
296
+ * health budget. The dependency leg of a start is not app boot, and charging it to the
297
+ * health deadline measures the dependency instead of the app.
298
+ * @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
299
+ * @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
300
+ */
301
+ async waitForConfiguredDependencies(processConfig) {
302
+ if (processConfig.dependsOn.length === 0) return
303
+
304
+ for (const dependencyId of processConfig.dependsOn) {
305
+ const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
306
+
307
+ if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
308
+
309
+ this.logger("release process dependency wait", {
310
+ dependencyPort: port,
311
+ dependencyProcessId: dependencyId,
312
+ processId: processConfig.id,
313
+ releaseId: this.releaseId,
314
+ timeoutMs: processConfig.dependencyTimeoutMs
315
+ })
316
+
317
+ const waitStartedAtMs = Date.now()
318
+
319
+ try {
320
+ const {attempts, elapsedMs} = await waitForListeningPort({
321
+ host: this.config.proxy.upstreamHost,
322
+ intervalMs: DEPENDENCY_READY_INTERVAL_MS,
323
+ port,
324
+ timeoutMs: processConfig.dependencyTimeoutMs
325
+ })
326
+
327
+ this.startupDependencyWaitMs += elapsedMs
328
+ this.logger("release process dependency ready", {
329
+ attempts,
330
+ dependencyPort: port,
331
+ dependencyProcessId: dependencyId,
332
+ elapsedMs,
333
+ processId: processConfig.id,
334
+ releaseId: this.releaseId
335
+ })
336
+ } catch (error) {
337
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
338
+ deadlineMs: processConfig.dependencyTimeoutMs,
339
+ dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
340
+ elapsedMs: elapsedSince(this.startupStartedAtMs),
341
+ failedProcessId: processConfig.id,
342
+ probes: undefined,
343
+ stage: "dependency"
344
+ })
345
+ throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
346
+ }
347
+
348
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
349
+ }
350
+ }
351
+
352
+ /**
353
+ * Health checks the proxied process on its own boot budget and records what the boot cost.
354
+ * @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
355
+ * @returns {Promise<void>} Resolves once the process answers its health path.
356
+ */
357
+ async waitForProxiedHealth(processConfig) {
358
+ const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
359
+ const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
360
+
361
+ try {
362
+ const {elapsedMs} = await waitForHealth({
363
+ health,
364
+ host: this.config.proxy.upstreamHost,
365
+ port: /** @type {number} */ (this.ports[processConfig.id]),
366
+ processId: processConfig.id
367
+ })
368
+
369
+ this.logger("release candidate health ready", {
370
+ bootMs: Date.now() - launchedAtMs,
371
+ deadlineMs: health.timeoutMs,
372
+ healthGateMs: elapsedMs,
373
+ path: health.path,
374
+ port: this.ports[processConfig.id],
375
+ processId: processConfig.id,
376
+ releaseId: this.releaseId,
377
+ waitedForDependenciesMs: this.startupDependencyWaitMs
378
+ })
379
+ } catch (error) {
380
+ const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
381
+
382
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
383
+ deadlineMs: health.timeoutMs,
384
+ dependencyWaitMs: this.startupDependencyWaitMs,
385
+ elapsedMs: elapsedSince(this.startupStartedAtMs),
386
+ failedProcessId: processConfig.id,
387
+ probes: healthCheck?.attempts,
388
+ stage: "health"
389
+ })
390
+ throw error
391
+ }
392
+ }
393
+
394
+ /**
395
+ * Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
261
396
  * @param {Error | string} error - Startup failure.
262
397
  * @param {{phase: string}} options - Diagnostic phase.
263
398
  * @returns {void}
264
399
  */
265
400
  logStartupFailure(error, {phase}) {
401
+ const diagnostics = this.startupFailureDiagnostics
402
+ const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
403
+ const lastLog = failedLogs?.at(-1)
404
+ const readyProcesses = [...this.readyProcessIds]
405
+
266
406
  this.logger("release startup failed", {
407
+ deadlineMs: diagnostics?.deadlineMs ?? null,
408
+ dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
409
+ elapsedMs: diagnostics?.elapsedMs ?? null,
267
410
  error: error instanceof Error ? error.message : error,
411
+ failedProcessId: diagnostics?.failedProcessId ?? null,
412
+ lastOutput: lastLog?.line ?? null,
413
+ lastOutputAt: lastLog?.at ?? null,
268
414
  phase,
269
- releaseId: this.releaseId
415
+ probes: diagnostics?.probes ?? null,
416
+ readyProcesses,
417
+ releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
418
+ releaseId: this.releaseId,
419
+ stage: diagnostics?.stage ?? null,
420
+ unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
270
421
  })
271
422
 
272
423
  for (const processInstance of this.processes.values()) {
@@ -291,18 +442,14 @@ export default class ReleaseGroup extends EventEmitter {
291
442
  * @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
292
443
  */
293
444
  releaseProcessStartOrder() {
294
- const releaseProcesses = this.config.processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
295
- const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
296
- const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
297
- const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
298
-
299
- return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
445
+ return processStartOrder(this.config.processes)
300
446
  }
301
447
 
302
448
  /** @returns {void} Marks this release active. */
303
449
  activate() {
304
450
  this.state = "active"
305
451
  this.activatedAt = new Date().toISOString()
452
+ this.clearDrainLivenessSamples()
306
453
  }
307
454
 
308
455
  /** Runs the configured release-generation activation acknowledgement, if any. */
@@ -357,6 +504,16 @@ export default class ReleaseGroup extends EventEmitter {
357
504
  this.drainStartedAt = undefined
358
505
  this.retirementError = undefined
359
506
  this.stoppedAt = undefined
507
+ this.clearDrainLivenessSamples()
508
+ }
509
+
510
+ /** Stops only the proxied processes when a failed compensation cannot restore jobs. */
511
+ async stopProxyTraffic() {
512
+ const proxiedIds = new Set(this.config.processes.filter((processConfig) => processConfig.policy === "proxied").map((processConfig) => processConfig.id))
513
+
514
+ await Promise.allSettled([...this.processes.entries()]
515
+ .filter(([id]) => proxiedIds.has(id.replace(/#\d+$/, "")))
516
+ .map(([, processInstance]) => processInstance.stop()))
360
517
  }
361
518
 
362
519
  /**
@@ -367,24 +524,25 @@ export default class ReleaseGroup extends EventEmitter {
367
524
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
368
525
  this.assertCommittedGenerationRecoverable()
369
526
  this.state = "starting"
527
+ this.startupStartedAtMs = Date.now()
370
528
  try {
371
529
  for (const processConfig of this.releaseProcessStartOrder()) {
530
+ await this.waitForConfiguredDependencies(processConfig)
372
531
  const instances = this.getProcesses(processConfig.id)
373
532
 
374
533
  if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
375
- for (const {process} of instances) {
534
+ for (const {id, process} of instances) {
376
535
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
377
536
  await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
537
+ this.processLaunchAtMs.set(id, Date.now())
538
+ this.startedProcessIds.add(id)
378
539
  }
379
540
 
380
541
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
381
- await waitForHealth({
382
- health: processConfig.health,
383
- host: this.config.proxy.upstreamHost,
384
- port: this.ports[processConfig.id]
385
- })
542
+ await this.waitForProxiedHealth(processConfig)
386
543
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
387
544
  }
545
+ for (const {id} of instances) this.readyProcessIds.add(id)
388
546
  }
389
547
  } catch (error) {
390
548
  await this.abortCommittedGenerationRestart()
@@ -476,6 +634,10 @@ export default class ReleaseGroup extends EventEmitter {
476
634
  ...this.baseEnvironment(processConfig, {count, index}),
477
635
  ...renderedEnv
478
636
  }
637
+ const dependencyRetired = () => processConfig.dependsOn.some((dependencyId) => {
638
+ const dependency = this.processes.get(dependencyId)
639
+ return dependency?.lifecycleRole === "retired" || dependency?.status().lifecycleRole === "retired"
640
+ })
479
641
 
480
642
  return {
481
643
  command: renderTemplate(processConfig.command, context),
@@ -488,7 +650,15 @@ export default class ReleaseGroup extends EventEmitter {
488
650
  outputLines: processConfig.outputLines,
489
651
  restart: processConfig.restart,
490
652
  restartDelayMs: processConfig.restartDelayMs,
491
- shouldRestart: options.shouldRestart || (() => this.state === "active" || this.state === "starting"),
653
+ shouldRestart: options.shouldRestart || (() => {
654
+ if (this.state !== "active" && this.state !== "starting") return false
655
+
656
+ // A companion whose same-generation service dependency has retired can never
657
+ // satisfy its startup gate again. Treat that dependency lifecycle as terminal
658
+ // for this process so an exit does not become an endless supervisor restart loop.
659
+ return !dependencyRetired()
660
+ }),
661
+ shouldStop: options.shouldStop || dependencyRetired,
492
662
  stopSignal: processConfig.stopSignal,
493
663
  stopTimeoutMs: processConfig.gracefulStopMs
494
664
  }
@@ -708,10 +878,21 @@ export default class ReleaseGroup extends EventEmitter {
708
878
  // their lifecycle drain runs as soon as the release is retired — in parallel with the
709
879
  // connection drain, not held until after it. The rest stop once connections have closed.
710
880
  const entries = [...this.processes.entries()]
711
- const stopOptions = {timeoutMs: config.proxy.forceStopTimeoutMs}
712
- const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([, processInstance]) => processInstance.stop(stopOptions))
881
+ const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
882
+ id,
883
+ promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
884
+ }))
713
885
  const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
714
886
  const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
887
+ // Bracket the retirement wait with a pair of observations: this one, and the one taken when
888
+ // every stop has returned. The gap between them is the wait the transition already performs
889
+ // (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
890
+ // itself rather than from a sampler running alongside it.
891
+ const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
892
+ ? this.takeLivenessSample(Date.now())
893
+ : undefined
894
+
895
+ if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
715
896
 
716
897
  if (this.connectionCount > 0) {
717
898
  await new Promise((resolve) => {
@@ -730,17 +911,40 @@ export default class ReleaseGroup extends EventEmitter {
730
911
  }
731
912
 
732
913
  if (this.ownerHandoffPaused) {
733
- void Promise.allSettled(nonBlockingStops)
914
+ void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
734
915
  return
735
916
  }
736
917
 
737
- await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop(stopOptions)))
738
- await Promise.allSettled(nonBlockingStops)
739
- await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop(stopOptions)))
918
+ const retirementTimeoutMs = config.proxy.nonBlockingDrainTimeoutMs
919
+ await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
920
+ const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
921
+ nonBlockingResults.forEach((result, index) => {
922
+ if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
923
+ const processInstance = this.processes.get(nonBlockingStops[index].id)
924
+ this.logger("release non-blocking process still draining", {
925
+ pid: processInstance?.status().pid ?? null,
926
+ processId: nonBlockingStops[index].id,
927
+ releaseId: this.releaseId,
928
+ timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
929
+ })
930
+ }
931
+ })
932
+ await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
933
+
934
+ // Last act of the retirement transition, before the release is recorded stopped: whatever
935
+ // is still alive here is the residue, and the bracket pair above can now say whether it is
936
+ // wedged. A generation that stops with a live process group is the state that made
937
+ // operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
938
+ // it does not end silently with the process still holding the release.
939
+ if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
940
+
740
941
  this.state = "stopped"
741
942
  this.stoppedAt = new Date().toISOString()
742
943
  this.releasePortReservations()
743
944
  this.abandonLocalConnections()
945
+ // The bracket pair is deliberately kept: a generation that stopped with a live process
946
+ // group stays observable, and the deploy path needs the older observation to prove the
947
+ // residue is still flat rather than starting a fresh window from zero.
744
948
  }
745
949
 
746
950
  /**
@@ -760,12 +964,18 @@ export default class ReleaseGroup extends EventEmitter {
760
964
  this.drainStartedAt = new Date().toISOString()
761
965
  }
762
966
 
967
+ // Refresh the live process definitions with the current retirement policy,
968
+ // but retain this generation's exact config identity for owner recovery.
969
+ // Replacing the stored generation config here would make a still-running
970
+ // singleton from the previous release fail the guardian provenance fence.
971
+ const generationConfig = this.config
972
+
763
973
  this.refreshProcessDefinitions(config)
974
+ this.config = generationConfig
764
975
  const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
765
976
  const results = await Promise.allSettled([...this.processes.entries()]
766
977
  .filter(([id]) => generationIds.has(id))
767
978
  .map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
768
-
769
979
  const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
770
980
 
771
981
  if (errors.length > 0) {
@@ -776,15 +986,44 @@ export default class ReleaseGroup extends EventEmitter {
776
986
  this.retirementError = undefined
777
987
  }
778
988
 
779
- /** @returns {Promise<void>} Stops all release-owned processes. */
780
- async stop() {
781
- const stopTasks = [...this.processes.values()].map((processInstance) => processInstance.stop())
989
+ /**
990
+ * Stops every process owned by the generation.
991
+ * @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
992
+ * supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
993
+ * is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
994
+ * `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
995
+ * the wait holds the mutation fence and every later deploy is refused (STOE-2015).
996
+ * @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
997
+ */
998
+ async stop(options = {}) {
999
+ const {retirementTimeoutMs} = options
1000
+ /** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
1001
+ const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
1002
+ const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
1003
+
1004
+ return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
1005
+ })
1006
+
1007
+ await Promise.allSettled(stopTasks.map(({promise}) => promise))
1008
+
1009
+ // A stop that stopped waiting leaves the process running. Name it, so the operator sees the
1010
+ // residue instead of a command that produced nothing.
1011
+ for (const {bounded, id, instance} of stopTasks) {
1012
+ if (!bounded || !instance.retirementTimedOut) continue
1013
+
1014
+ this.logger("release stop gave up waiting for a retiring process", {
1015
+ pid: instance.status().pid ?? null,
1016
+ processId: id,
1017
+ releaseId: this.releaseId,
1018
+ timeoutMs: retirementTimeoutMs
1019
+ })
1020
+ }
782
1021
 
783
- await Promise.allSettled(stopTasks)
784
1022
  this.state = "stopped"
785
1023
  this.stoppedAt = new Date().toISOString()
786
1024
  this.releasePortReservations()
787
1025
  this.abandonLocalConnections()
1026
+ this.clearDrainLivenessSamples()
788
1027
  }
789
1028
 
790
1029
  /** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
@@ -792,12 +1031,171 @@ export default class ReleaseGroup extends EventEmitter {
792
1031
  await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
793
1032
  }
794
1033
 
1034
+ /**
1035
+ * Records one liveness observation for a draining generation and classifies its drain.
1036
+ *
1037
+ * Two observations are needed to prove CPU flatness, and reading a process group means
1038
+ * scanning procfs, so sampling is bounded three ways: a generation that still owns live
1039
+ * connections is classified without touching procfs at all (connections alone settle it), a
1040
+ * new observation is taken only once the newest one is already a full window behind the
1041
+ * present read, and every other read reuses the newest observation. A group that is not
1042
+ * draining reports no drain.
1043
+ * @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
1044
+ */
1045
+ drainStatus() {
1046
+ if (this.state !== "draining") return undefined
1047
+
1048
+ return this.observeRetirement({nowMs: Date.now()}).classification
1049
+ }
1050
+
1051
+ /**
1052
+ * Records the newest liveness observation of this generation and classifies its retirement.
1053
+ * @param {object} options - Observation options.
1054
+ * @param {number} options.nowMs - Read time.
1055
+ * @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
1056
+ * @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
1057
+ */
1058
+ observeRetirement({nowMs, previousSample}) {
1059
+ const drainBudgetMs = this.config.proxy.drainTimeoutMs
1060
+ const drainElapsedMs = this.drainStartedAt === undefined ? 0 : nowMs - Date.parse(this.drainStartedAt)
1061
+
1062
+ if (previousSample === undefined && this.connectionCount > 0) {
1063
+ const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
1064
+
1065
+ return {
1066
+ classification: classifyDrainProgress({
1067
+ connectionCount: this.connectionCount,
1068
+ currentSample,
1069
+ drainBudgetMs,
1070
+ drainElapsedMs,
1071
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1072
+ previousSample: undefined
1073
+ }),
1074
+ currentSample
1075
+ }
1076
+ }
1077
+
1078
+ const newest = this.drainNewestSample
1079
+ const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
1080
+ ? this.takeLivenessSample(nowMs)
1081
+ : newest
1082
+ const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
1083
+
1084
+ if (currentSample !== newest) {
1085
+ this.drainPriorSample = newest
1086
+ this.drainNewestSample = currentSample
1087
+ }
1088
+
1089
+ return {
1090
+ classification: classifyDrainProgress({
1091
+ connectionCount: this.connectionCount,
1092
+ currentSample,
1093
+ drainBudgetMs,
1094
+ drainElapsedMs,
1095
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1096
+ previousSample: priorSample
1097
+ }),
1098
+ currentSample
1099
+ }
1100
+ }
1101
+
1102
+ /**
1103
+ * Applies the wedge escalation to this generation: terminates the processes a proven wedge
1104
+ * proves are holding nothing, and names the ones it refused to terminate.
1105
+ *
1106
+ * Both halves are required. The card this path was written for (STOE-1687) is about a
1107
+ * wedged worker holding a release open for four hours while deploys stack behind it; the
1108
+ * boundary on the same card (STOE-1077) is that no process still holding live work may ever
1109
+ * be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
1110
+ * with the pids and process groups it is holding, which is the information an operator needs
1111
+ * in place of guessing.
1112
+ * @param {object} options - Escalation context.
1113
+ * @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
1114
+ * @param {string} options.reason - Named context recorded with the decision.
1115
+ * @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
1116
+ */
1117
+ async escalateWedgedGeneration({previousSample, reason}) {
1118
+ const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
1119
+ const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
1120
+ const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
1121
+
1122
+ if (decision.decision === "report") {
1123
+ if (classification.status === "wedged") {
1124
+ this.logger("release wedged generation left alive", {
1125
+ escalation: "refused",
1126
+ heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
1127
+ pids: liveObservations.map((process) => process.pid ?? null),
1128
+ processIds: liveObservations.map((process) => process.id),
1129
+ reason: decision.reason,
1130
+ releaseId: this.releaseId,
1131
+ trigger: reason
1132
+ })
1133
+ }
1134
+
1135
+ return []
1136
+ }
1137
+
1138
+ const escalated = []
1139
+
1140
+ for (const target of decision.targets) {
1141
+ const processInstance = this.processes.get(target.processId)
1142
+
1143
+ if (!processInstance) continue
1144
+ if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
1145
+ }
1146
+
1147
+ if (escalated.length > 0) {
1148
+ this.logger("release wedged generation escalated", {
1149
+ escalation: "SIGKILL",
1150
+ processIds: escalated,
1151
+ reason: decision.reason,
1152
+ releaseId: this.releaseId,
1153
+ trigger: reason
1154
+ })
1155
+ }
1156
+
1157
+ return escalated
1158
+ }
1159
+
1160
+ /**
1161
+ * @param {number} nowMs - Observation time.
1162
+ * @returns {DrainLivenessSample} One liveness observation of every process in the generation.
1163
+ */
1164
+ takeLivenessSample(nowMs) {
1165
+ const processes = [...this.processes.entries()].map(([id, processInstance]) => {
1166
+ const processStatus = processInstance.status()
1167
+ const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
1168
+ // A group that outlived its tracked process is still observed by group id: it holds the
1169
+ // generation's work even though no rollbridge record can signal it any more.
1170
+ const pgid = processStatus.pid ?? heldProcessGroupPgid
1171
+ const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
1172
+
1173
+ return {
1174
+ cpuMs: sample?.cpuMs,
1175
+ heldProcessGroupPgid,
1176
+ id,
1177
+ liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
1178
+ pid: processStatus.pid,
1179
+ retirementTimedOut: processInstance.retirementTimedOut === true
1180
+ }
1181
+ })
1182
+
1183
+ return {atMs: nowMs, processes}
1184
+ }
1185
+
1186
+ /** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
1187
+ clearDrainLivenessSamples() {
1188
+ this.drainNewestSample = undefined
1189
+ this.drainPriorSample = undefined
1190
+ }
1191
+
795
1192
  /** @returns {ReleaseStatus} Status payload. */
796
1193
  status() {
797
1194
  return {
798
1195
  activatedAt: this.activatedAt,
799
1196
  connectionCount: this.connectionCount,
800
1197
  connections: {...this.connections},
1198
+ drain: this.drainStatus(),
801
1199
  drainStartedAt: this.drainStartedAt,
802
1200
  ports: {...this.ports},
803
1201
  processes: [...this.processes.values()].map((processInstance) => processInstance.status()),