rollbridge 0.1.73 → 0.1.75

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-bounded-control-client-close.md +4 -0
  24. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  25. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  26. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  27. package/docs/cli.md +128 -3
  28. package/docs/compose-services.md +222 -0
  29. package/docs/config.md +177 -6
  30. package/docs/troubleshooting.md +97 -6
  31. package/docs/velocious.md +1 -1
  32. package/docs/workers.md +11 -0
  33. package/examples/tensorbuzz.com.js +3 -2
  34. package/package.json +1 -1
  35. package/src/cli.js +69 -3
  36. package/src/compose-unit.js +725 -0
  37. package/src/config.js +346 -8
  38. package/src/control-client.js +32 -5
  39. package/src/daemon.js +448 -76
  40. package/src/doctor.js +74 -0
  41. package/src/drain-policy.js +252 -0
  42. package/src/guardian-client.js +92 -9
  43. package/src/health.js +99 -8
  44. package/src/managed-process.js +197 -16
  45. package/src/process-guardian.js +212 -22
  46. package/src/process-memory.js +62 -2
  47. package/src/process-order.js +26 -0
  48. package/src/release-group.js +452 -39
  49. package/tensorbuzz.yml +20 -1
  50. package/test/completion.test.js +5 -2
  51. package/test/compose-config.test.js +160 -0
  52. package/test/compose-docker.test.js +297 -0
  53. package/test/compose-unit.test.js +415 -0
  54. package/test/config-validation.test.js +135 -5
  55. package/test/control-protocol.test.js +22 -0
  56. package/test/drain-policy.test.js +328 -0
  57. package/test/fixtures/deferred-boot-app.js +106 -0
  58. package/test/fixtures/gated-app.js +50 -0
  59. package/test/guardian-client.test.js +149 -5
  60. package/test/health.test.js +74 -1
  61. package/test/managed-process.test.js +260 -1
  62. package/test/owner-recovery.test.js +151 -20
  63. package/test/owner-replacement.test.js +176 -15
  64. package/test/process-memory.test.js +47 -1
  65. package/test/release-group.test.js +447 -2
  66. package/test/rollbridge.test.js +774 -72
  67. package/test/shutdown-completion.test.js +119 -0
  68. package/test/shutdown-retirement-bound.test.js +162 -0
  69. package/test/support/process.js +3 -1
  70. package/test/tensorbuzz-config.test.js +228 -0
@@ -2,17 +2,29 @@
2
2
 
3
3
  import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
+ import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
5
6
  import ManagedProcess from "./managed-process.js"
6
7
  import {findAvailablePort} from "./port-allocator.js"
8
+ import {processStartOrder} from "./process-order.js"
9
+ import {sampleProcessGroup} from "./process-memory.js"
7
10
  import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
8
- import {waitForHealth} from "./health.js"
11
+ import {waitForHealth, waitForListeningPort} from "./health.js"
12
+
13
+ const DEPENDENCY_READY_INTERVAL_MS = 250
14
+ // Shortest gap between two liveness observations that can prove a draining group's CPU is
15
+ // flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
16
+ // with live runner children doing work, so flatness is only evidence below this window.
17
+ const DRAIN_SAMPLE_WINDOW_MS = 20000
9
18
 
10
19
  /**
11
20
  * @typedef {import("./json.js").JsonValue} JsonValue
12
21
  * @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
22
+ * @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
23
+ * @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
13
24
  * @typedef {{http: number, websocket: number}} ReleaseConnections
14
- * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
15
- * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
25
+ * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
26
+ * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean, shouldStop?: () => boolean}} BuildProcessOptions
27
+ * @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
16
28
  */
17
29
 
18
30
 
@@ -57,6 +69,14 @@ function envId(id) {
57
69
  return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
58
70
  }
59
71
 
72
+ /**
73
+ * @param {number | undefined} startedAtMs - When the measured phase began.
74
+ * @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
75
+ */
76
+ function elapsedSince(startedAtMs) {
77
+ return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
78
+ }
79
+
60
80
  /**
61
81
  * @param {import("./config.js").ProcessConfig} processConfig - Process config.
62
82
  * @param {number} index - Zero-based replica index.
@@ -77,9 +97,10 @@ export default class ReleaseGroup extends EventEmitter {
77
97
  * @param {string | undefined} args.revision - Revision.
78
98
  * @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
79
99
  * @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
100
+ * @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
80
101
  * @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
81
102
  */
82
- constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
103
+ constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
83
104
  super()
84
105
 
85
106
  this.config = config
@@ -93,6 +114,7 @@ export default class ReleaseGroup extends EventEmitter {
93
114
  this.transferredConnections = /** @type {ReleaseConnections} */ ({http: 0, websocket: 0})
94
115
  this.ownerHandoffPaused = false
95
116
  this.processes = /** @type {Map<string, ManagedProcess>} */ (new Map())
117
+ this.singletonProcesses = /** @type {Map<string, ManagedProcess>} */ (new Map())
96
118
  this.handoffServiceIds = /** @type {Set<string>} */ (new Set())
97
119
  this.nonBlockingDrainIds = /** @type {Set<string>} */ (new Set())
98
120
  this.ports = /** @type {Record<string, number>} */ ({})
@@ -101,22 +123,36 @@ export default class ReleaseGroup extends EventEmitter {
101
123
  this.servicePorts = servicePorts
102
124
  this.shouldStart = shouldStart
103
125
  this.processFactory = processFactory
126
+ this.procPath = procPath
104
127
  this.portsAllocated = false
105
128
  this.drainStartedAt = /** @type {string | undefined} */ (undefined)
129
+ this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
130
+ this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
106
131
  this.activatedAt = /** @type {string | undefined} */ (undefined)
107
132
  this.stoppedAt = /** @type {string | undefined} */ (undefined)
108
133
  this.retirementError = /** @type {string | undefined} */ (undefined)
109
134
  this.preserveConfigOnRetirement = false
135
+ this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
136
+ this.readyProcessIds = /** @type {Set<string>} */ (new Set())
137
+ this.startedProcessIds = /** @type {Set<string>} */ (new Set())
138
+ this.startupDependencyWaitMs = 0
139
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
140
+ this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
110
141
  }
111
142
 
112
143
  /** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
113
144
  async start() {
114
145
  this.state = "starting"
146
+ this.startupStartedAtMs = Date.now()
115
147
 
116
148
  try {
117
149
  await this.allocatePorts()
118
150
 
119
151
  for (const processConfig of this.releaseProcessStartOrder()) {
152
+ await this.waitForConfiguredDependencies(processConfig)
153
+ /** @type {string[]} */
154
+ const instanceIds = []
155
+
120
156
  for (let index = 0; index < processConfig.replicas; index += 1) {
121
157
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
122
158
 
@@ -127,17 +163,17 @@ export default class ReleaseGroup extends EventEmitter {
127
163
  if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
128
164
  if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
129
165
  await processInstance.start("deploy")
166
+ this.processLaunchAtMs.set(instanceId, Date.now())
167
+ this.startedProcessIds.add(instanceId)
168
+ instanceIds.push(instanceId)
130
169
  }
131
170
 
132
171
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
133
- await waitForHealth({
134
- health: processConfig.health,
135
- host: this.config.proxy.upstreamHost,
136
- port: this.ports[processConfig.id]
137
- })
138
-
139
- if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
172
+ await this.waitForProxiedHealth(processConfig)
140
173
  }
174
+
175
+ for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
176
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
141
177
  }
142
178
  } catch (error) {
143
179
  this.state = "failed"
@@ -262,16 +298,132 @@ export default class ReleaseGroup extends EventEmitter {
262
298
  }
263
299
 
264
300
  /**
265
- * Logs process diagnostics around failed startup cleanup.
301
+ * Waits for the ports a process declares it depends on, on a budget that is not the
302
+ * health budget. The dependency leg of a start is not app boot, and charging it to the
303
+ * health deadline measures the dependency instead of the app.
304
+ * @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
305
+ * @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
306
+ */
307
+ async waitForConfiguredDependencies(processConfig) {
308
+ if (processConfig.dependsOn.length === 0) return
309
+
310
+ for (const dependencyId of processConfig.dependsOn) {
311
+ const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
312
+
313
+ if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
314
+
315
+ this.logger("release process dependency wait", {
316
+ dependencyPort: port,
317
+ dependencyProcessId: dependencyId,
318
+ processId: processConfig.id,
319
+ releaseId: this.releaseId,
320
+ timeoutMs: processConfig.dependencyTimeoutMs
321
+ })
322
+
323
+ const waitStartedAtMs = Date.now()
324
+
325
+ try {
326
+ const {attempts, elapsedMs} = await waitForListeningPort({
327
+ host: this.config.proxy.upstreamHost,
328
+ intervalMs: DEPENDENCY_READY_INTERVAL_MS,
329
+ port,
330
+ timeoutMs: processConfig.dependencyTimeoutMs
331
+ })
332
+
333
+ this.startupDependencyWaitMs += elapsedMs
334
+ this.logger("release process dependency ready", {
335
+ attempts,
336
+ dependencyPort: port,
337
+ dependencyProcessId: dependencyId,
338
+ elapsedMs,
339
+ processId: processConfig.id,
340
+ releaseId: this.releaseId
341
+ })
342
+ } catch (error) {
343
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
344
+ deadlineMs: processConfig.dependencyTimeoutMs,
345
+ dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
346
+ elapsedMs: elapsedSince(this.startupStartedAtMs),
347
+ failedProcessId: processConfig.id,
348
+ probes: undefined,
349
+ stage: "dependency"
350
+ })
351
+ throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
352
+ }
353
+
354
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
355
+ }
356
+ }
357
+
358
+ /**
359
+ * Health checks the proxied process on its own boot budget and records what the boot cost.
360
+ * @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
361
+ * @returns {Promise<void>} Resolves once the process answers its health path.
362
+ */
363
+ async waitForProxiedHealth(processConfig) {
364
+ const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
365
+ const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
366
+
367
+ try {
368
+ const {elapsedMs} = await waitForHealth({
369
+ health,
370
+ host: this.config.proxy.upstreamHost,
371
+ port: /** @type {number} */ (this.ports[processConfig.id]),
372
+ processId: processConfig.id
373
+ })
374
+
375
+ this.logger("release candidate health ready", {
376
+ bootMs: Date.now() - launchedAtMs,
377
+ deadlineMs: health.timeoutMs,
378
+ healthGateMs: elapsedMs,
379
+ path: health.path,
380
+ port: this.ports[processConfig.id],
381
+ processId: processConfig.id,
382
+ releaseId: this.releaseId,
383
+ waitedForDependenciesMs: this.startupDependencyWaitMs
384
+ })
385
+ } catch (error) {
386
+ const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
387
+
388
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
389
+ deadlineMs: health.timeoutMs,
390
+ dependencyWaitMs: this.startupDependencyWaitMs,
391
+ elapsedMs: elapsedSince(this.startupStartedAtMs),
392
+ failedProcessId: processConfig.id,
393
+ probes: healthCheck?.attempts,
394
+ stage: "health"
395
+ })
396
+ throw error
397
+ }
398
+ }
399
+
400
+ /**
401
+ * Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
266
402
  * @param {Error | string} error - Startup failure.
267
403
  * @param {{phase: string}} options - Diagnostic phase.
268
404
  * @returns {void}
269
405
  */
270
406
  logStartupFailure(error, {phase}) {
407
+ const diagnostics = this.startupFailureDiagnostics
408
+ const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
409
+ const lastLog = failedLogs?.at(-1)
410
+ const readyProcesses = [...this.readyProcessIds]
411
+
271
412
  this.logger("release startup failed", {
413
+ deadlineMs: diagnostics?.deadlineMs ?? null,
414
+ dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
415
+ elapsedMs: diagnostics?.elapsedMs ?? null,
272
416
  error: error instanceof Error ? error.message : error,
417
+ failedProcessId: diagnostics?.failedProcessId ?? null,
418
+ lastOutput: lastLog?.line ?? null,
419
+ lastOutputAt: lastLog?.at ?? null,
273
420
  phase,
274
- releaseId: this.releaseId
421
+ probes: diagnostics?.probes ?? null,
422
+ readyProcesses,
423
+ releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
424
+ releaseId: this.releaseId,
425
+ stage: diagnostics?.stage ?? null,
426
+ unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
275
427
  })
276
428
 
277
429
  for (const processInstance of this.processes.values()) {
@@ -296,18 +448,14 @@ export default class ReleaseGroup extends EventEmitter {
296
448
  * @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
297
449
  */
298
450
  releaseProcessStartOrder() {
299
- const releaseProcesses = this.config.processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
300
- const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
301
- const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
302
- const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
303
-
304
- return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
451
+ return processStartOrder(this.config.processes)
305
452
  }
306
453
 
307
454
  /** @returns {void} Marks this release active. */
308
455
  activate() {
309
456
  this.state = "active"
310
457
  this.activatedAt = new Date().toISOString()
458
+ this.clearDrainLivenessSamples()
311
459
  }
312
460
 
313
461
  /** Runs the configured release-generation activation acknowledgement, if any. */
@@ -362,6 +510,16 @@ export default class ReleaseGroup extends EventEmitter {
362
510
  this.drainStartedAt = undefined
363
511
  this.retirementError = undefined
364
512
  this.stoppedAt = undefined
513
+ this.clearDrainLivenessSamples()
514
+ }
515
+
516
+ /** Stops only the proxied processes when a failed compensation cannot restore jobs. */
517
+ async stopProxyTraffic() {
518
+ const proxiedIds = new Set(this.config.processes.filter((processConfig) => processConfig.policy === "proxied").map((processConfig) => processConfig.id))
519
+
520
+ await Promise.allSettled([...this.processes.entries()]
521
+ .filter(([id]) => proxiedIds.has(id.replace(/#\d+$/, "")))
522
+ .map(([, processInstance]) => processInstance.stop()))
365
523
  }
366
524
 
367
525
  /**
@@ -372,24 +530,25 @@ export default class ReleaseGroup extends EventEmitter {
372
530
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
373
531
  this.assertCommittedGenerationRecoverable()
374
532
  this.state = "starting"
533
+ this.startupStartedAtMs = Date.now()
375
534
  try {
376
535
  for (const processConfig of this.releaseProcessStartOrder()) {
536
+ await this.waitForConfiguredDependencies(processConfig)
377
537
  const instances = this.getProcesses(processConfig.id)
378
538
 
379
539
  if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
380
- for (const {process} of instances) {
540
+ for (const {id, process} of instances) {
381
541
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
382
542
  await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
543
+ this.processLaunchAtMs.set(id, Date.now())
544
+ this.startedProcessIds.add(id)
383
545
  }
384
546
 
385
547
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
386
- await waitForHealth({
387
- health: processConfig.health,
388
- host: this.config.proxy.upstreamHost,
389
- port: this.ports[processConfig.id]
390
- })
548
+ await this.waitForProxiedHealth(processConfig)
391
549
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
392
550
  }
551
+ for (const {id} of instances) this.readyProcessIds.add(id)
393
552
  }
394
553
  } catch (error) {
395
554
  await this.abortCommittedGenerationRestart()
@@ -481,6 +640,10 @@ export default class ReleaseGroup extends EventEmitter {
481
640
  ...this.baseEnvironment(processConfig, {count, index}),
482
641
  ...renderedEnv
483
642
  }
643
+ const dependencyRetired = () => processConfig.dependsOn.some((dependencyId) => {
644
+ const dependency = this.processes.get(dependencyId)
645
+ return dependency?.lifecycleRole === "retired" || dependency?.status().lifecycleRole === "retired"
646
+ })
484
647
 
485
648
  return {
486
649
  command: renderTemplate(processConfig.command, context),
@@ -493,7 +656,15 @@ export default class ReleaseGroup extends EventEmitter {
493
656
  outputLines: processConfig.outputLines,
494
657
  restart: processConfig.restart,
495
658
  restartDelayMs: processConfig.restartDelayMs,
496
- shouldRestart: options.shouldRestart || (() => this.state === "active" || this.state === "starting"),
659
+ shouldRestart: options.shouldRestart || (() => {
660
+ if (this.state !== "active" && this.state !== "starting") return false
661
+
662
+ // A companion whose same-generation service dependency has retired can never
663
+ // satisfy its startup gate again. Treat that dependency lifecycle as terminal
664
+ // for this process so an exit does not become an endless supervisor restart loop.
665
+ return !dependencyRetired()
666
+ }),
667
+ shouldStop: options.shouldStop || dependencyRetired,
497
668
  stopSignal: processConfig.stopSignal,
498
669
  stopTimeoutMs: processConfig.gracefulStopMs
499
670
  }
@@ -508,10 +679,16 @@ export default class ReleaseGroup extends EventEmitter {
508
679
  buildProcess(processConfig, options = {}) {
509
680
  const definition = this.processDefinition(processConfig, options)
510
681
  const instanceId = options.instanceId ?? processConfig.id
511
-
512
- return this.processFactory
682
+ const process = this.processFactory
513
683
  ? this.processFactory(options.guardianKey || `release:${this.releaseId}:${instanceId}`, definition)
514
684
  : new ManagedProcess(definition)
685
+
686
+ // A singleton is tracked by the daemon rather than in this generation's process map, but its
687
+ // definition is still the one this generation recorded for it. Keeping the instance here is what
688
+ // lets a later retirement move that definition together with the generation's config.
689
+ if (processConfig.policy === "singleton") this.singletonProcesses.set(instanceId, process)
690
+
691
+ return process
515
692
  }
516
693
 
517
694
  /**
@@ -525,7 +702,14 @@ export default class ReleaseGroup extends EventEmitter {
525
702
  this.nonBlockingDrainIds.clear()
526
703
 
527
704
  for (const processConfig of config.processes) {
528
- const instances = this.getProcesses(processConfig.id)
705
+ const instances = [...this.getProcesses(processConfig.id)]
706
+
707
+ if (processConfig.policy === "singleton") {
708
+ for (const [instanceId, process] of this.singletonProcesses) {
709
+ if (instances.some((instance) => instance.id === instanceId)) continue
710
+ instances.push({id: instanceId, process})
711
+ }
712
+ }
529
713
 
530
714
  for (let index = 0; index < instances.length; index += 1) {
531
715
  const instance = instances[index]
@@ -713,10 +897,21 @@ export default class ReleaseGroup extends EventEmitter {
713
897
  // their lifecycle drain runs as soon as the release is retired — in parallel with the
714
898
  // connection drain, not held until after it. The rest stop once connections have closed.
715
899
  const entries = [...this.processes.entries()]
716
- const stopOptions = {timeoutMs: config.proxy.forceStopTimeoutMs}
717
- const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([, processInstance]) => processInstance.stop(stopOptions))
900
+ const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
901
+ id,
902
+ promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
903
+ }))
718
904
  const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
719
905
  const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
906
+ // Bracket the retirement wait with a pair of observations: this one, and the one taken when
907
+ // every stop has returned. The gap between them is the wait the transition already performs
908
+ // (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
909
+ // itself rather than from a sampler running alongside it.
910
+ const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
911
+ ? this.takeLivenessSample(Date.now())
912
+ : undefined
913
+
914
+ if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
720
915
 
721
916
  if (this.connectionCount > 0) {
722
917
  await new Promise((resolve) => {
@@ -735,17 +930,40 @@ export default class ReleaseGroup extends EventEmitter {
735
930
  }
736
931
 
737
932
  if (this.ownerHandoffPaused) {
738
- void Promise.allSettled(nonBlockingStops)
933
+ void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
739
934
  return
740
935
  }
741
936
 
742
- await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop(stopOptions)))
743
- await Promise.allSettled(nonBlockingStops)
744
- await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop(stopOptions)))
937
+ const retirementTimeoutMs = config.proxy.nonBlockingDrainTimeoutMs
938
+ await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
939
+ const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
940
+ nonBlockingResults.forEach((result, index) => {
941
+ if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
942
+ const processInstance = this.processes.get(nonBlockingStops[index].id)
943
+ this.logger("release non-blocking process still draining", {
944
+ pid: processInstance?.status().pid ?? null,
945
+ processId: nonBlockingStops[index].id,
946
+ releaseId: this.releaseId,
947
+ timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
948
+ })
949
+ }
950
+ })
951
+ await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
952
+
953
+ // Last act of the retirement transition, before the release is recorded stopped: whatever
954
+ // is still alive here is the residue, and the bracket pair above can now say whether it is
955
+ // wedged. A generation that stops with a live process group is the state that made
956
+ // operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
957
+ // it does not end silently with the process still holding the release.
958
+ if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
959
+
745
960
  this.state = "stopped"
746
961
  this.stoppedAt = new Date().toISOString()
747
962
  this.releasePortReservations()
748
963
  this.abandonLocalConnections()
964
+ // The bracket pair is deliberately kept: a generation that stopped with a live process
965
+ // group stays observable, and the deploy path needs the older observation to prove the
966
+ // residue is still flat rather than starting a fresh window from zero.
749
967
  }
750
968
 
751
969
  /**
@@ -765,6 +983,13 @@ export default class ReleaseGroup extends EventEmitter {
765
983
  this.drainStartedAt = new Date().toISOString()
766
984
  }
767
985
 
986
+ // The retirement policy — which processes are quiesced as one generation, and the drain
987
+ // budget — comes from the caller's config, and the live definitions are refreshed with it.
988
+ // The refreshed config has to become this generation's stored config: the guardian records the
989
+ // refreshed definition's provenance for every process it updates here, and the durable owner
990
+ // state publishes `release.config` as the definition recovery rebuilds each live process from.
991
+ // Keeping the pre-refresh config desynchronises the two, and the next daemon recovery then
992
+ // re-registers a still-live process with the stale provenance and aborts (STOE-2040).
768
993
  this.refreshProcessDefinitions(config)
769
994
  const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
770
995
  const results = await Promise.allSettled([...this.processes.entries()]
@@ -781,15 +1006,44 @@ export default class ReleaseGroup extends EventEmitter {
781
1006
  this.retirementError = undefined
782
1007
  }
783
1008
 
784
- /** @returns {Promise<void>} Stops all release-owned processes. */
785
- async stop() {
786
- const stopTasks = [...this.processes.values()].map((processInstance) => processInstance.stop())
1009
+ /**
1010
+ * Stops every process owned by the generation.
1011
+ * @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
1012
+ * supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
1013
+ * is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
1014
+ * `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
1015
+ * the wait holds the mutation fence and every later deploy is refused (STOE-2015).
1016
+ * @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
1017
+ */
1018
+ async stop(options = {}) {
1019
+ const {retirementTimeoutMs} = options
1020
+ /** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
1021
+ const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
1022
+ const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
1023
+
1024
+ return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
1025
+ })
1026
+
1027
+ await Promise.allSettled(stopTasks.map(({promise}) => promise))
1028
+
1029
+ // A stop that stopped waiting leaves the process running. Name it, so the operator sees the
1030
+ // residue instead of a command that produced nothing.
1031
+ for (const {bounded, id, instance} of stopTasks) {
1032
+ if (!bounded || !instance.retirementTimedOut) continue
1033
+
1034
+ this.logger("release stop gave up waiting for a retiring process", {
1035
+ pid: instance.status().pid ?? null,
1036
+ processId: id,
1037
+ releaseId: this.releaseId,
1038
+ timeoutMs: retirementTimeoutMs
1039
+ })
1040
+ }
787
1041
 
788
- await Promise.allSettled(stopTasks)
789
1042
  this.state = "stopped"
790
1043
  this.stoppedAt = new Date().toISOString()
791
1044
  this.releasePortReservations()
792
1045
  this.abandonLocalConnections()
1046
+ this.clearDrainLivenessSamples()
793
1047
  }
794
1048
 
795
1049
  /** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
@@ -797,12 +1051,171 @@ export default class ReleaseGroup extends EventEmitter {
797
1051
  await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
798
1052
  }
799
1053
 
1054
+ /**
1055
+ * Records one liveness observation for a draining generation and classifies its drain.
1056
+ *
1057
+ * Two observations are needed to prove CPU flatness, and reading a process group means
1058
+ * scanning procfs, so sampling is bounded three ways: a generation that still owns live
1059
+ * connections is classified without touching procfs at all (connections alone settle it), a
1060
+ * new observation is taken only once the newest one is already a full window behind the
1061
+ * present read, and every other read reuses the newest observation. A group that is not
1062
+ * draining reports no drain.
1063
+ * @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
1064
+ */
1065
+ drainStatus() {
1066
+ if (this.state !== "draining") return undefined
1067
+
1068
+ return this.observeRetirement({nowMs: Date.now()}).classification
1069
+ }
1070
+
1071
+ /**
1072
+ * Records the newest liveness observation of this generation and classifies its retirement.
1073
+ * @param {object} options - Observation options.
1074
+ * @param {number} options.nowMs - Read time.
1075
+ * @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
1076
+ * @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
1077
+ */
1078
+ observeRetirement({nowMs, previousSample}) {
1079
+ const drainBudgetMs = this.config.proxy.drainTimeoutMs
1080
+ const drainElapsedMs = this.drainStartedAt === undefined ? 0 : nowMs - Date.parse(this.drainStartedAt)
1081
+
1082
+ if (previousSample === undefined && this.connectionCount > 0) {
1083
+ const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
1084
+
1085
+ return {
1086
+ classification: classifyDrainProgress({
1087
+ connectionCount: this.connectionCount,
1088
+ currentSample,
1089
+ drainBudgetMs,
1090
+ drainElapsedMs,
1091
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1092
+ previousSample: undefined
1093
+ }),
1094
+ currentSample
1095
+ }
1096
+ }
1097
+
1098
+ const newest = this.drainNewestSample
1099
+ const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
1100
+ ? this.takeLivenessSample(nowMs)
1101
+ : newest
1102
+ const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
1103
+
1104
+ if (currentSample !== newest) {
1105
+ this.drainPriorSample = newest
1106
+ this.drainNewestSample = currentSample
1107
+ }
1108
+
1109
+ return {
1110
+ classification: classifyDrainProgress({
1111
+ connectionCount: this.connectionCount,
1112
+ currentSample,
1113
+ drainBudgetMs,
1114
+ drainElapsedMs,
1115
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1116
+ previousSample: priorSample
1117
+ }),
1118
+ currentSample
1119
+ }
1120
+ }
1121
+
1122
+ /**
1123
+ * Applies the wedge escalation to this generation: terminates the processes a proven wedge
1124
+ * proves are holding nothing, and names the ones it refused to terminate.
1125
+ *
1126
+ * Both halves are required. The card this path was written for (STOE-1687) is about a
1127
+ * wedged worker holding a release open for four hours while deploys stack behind it; the
1128
+ * boundary on the same card (STOE-1077) is that no process still holding live work may ever
1129
+ * be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
1130
+ * with the pids and process groups it is holding, which is the information an operator needs
1131
+ * in place of guessing.
1132
+ * @param {object} options - Escalation context.
1133
+ * @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
1134
+ * @param {string} options.reason - Named context recorded with the decision.
1135
+ * @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
1136
+ */
1137
+ async escalateWedgedGeneration({previousSample, reason}) {
1138
+ const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
1139
+ const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
1140
+ const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
1141
+
1142
+ if (decision.decision === "report") {
1143
+ if (classification.status === "wedged") {
1144
+ this.logger("release wedged generation left alive", {
1145
+ escalation: "refused",
1146
+ heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
1147
+ pids: liveObservations.map((process) => process.pid ?? null),
1148
+ processIds: liveObservations.map((process) => process.id),
1149
+ reason: decision.reason,
1150
+ releaseId: this.releaseId,
1151
+ trigger: reason
1152
+ })
1153
+ }
1154
+
1155
+ return []
1156
+ }
1157
+
1158
+ const escalated = []
1159
+
1160
+ for (const target of decision.targets) {
1161
+ const processInstance = this.processes.get(target.processId)
1162
+
1163
+ if (!processInstance) continue
1164
+ if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
1165
+ }
1166
+
1167
+ if (escalated.length > 0) {
1168
+ this.logger("release wedged generation escalated", {
1169
+ escalation: "SIGKILL",
1170
+ processIds: escalated,
1171
+ reason: decision.reason,
1172
+ releaseId: this.releaseId,
1173
+ trigger: reason
1174
+ })
1175
+ }
1176
+
1177
+ return escalated
1178
+ }
1179
+
1180
+ /**
1181
+ * @param {number} nowMs - Observation time.
1182
+ * @returns {DrainLivenessSample} One liveness observation of every process in the generation.
1183
+ */
1184
+ takeLivenessSample(nowMs) {
1185
+ const processes = [...this.processes.entries()].map(([id, processInstance]) => {
1186
+ const processStatus = processInstance.status()
1187
+ const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
1188
+ // A group that outlived its tracked process is still observed by group id: it holds the
1189
+ // generation's work even though no rollbridge record can signal it any more.
1190
+ const pgid = processStatus.pid ?? heldProcessGroupPgid
1191
+ const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
1192
+
1193
+ return {
1194
+ cpuMs: sample?.cpuMs,
1195
+ heldProcessGroupPgid,
1196
+ id,
1197
+ liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
1198
+ pid: processStatus.pid,
1199
+ retirementTimedOut: processInstance.retirementTimedOut === true
1200
+ }
1201
+ })
1202
+
1203
+ return {atMs: nowMs, processes}
1204
+ }
1205
+
1206
+ /** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
1207
+ clearDrainLivenessSamples() {
1208
+ this.drainNewestSample = undefined
1209
+ this.drainPriorSample = undefined
1210
+ }
1211
+
800
1212
  /** @returns {ReleaseStatus} Status payload. */
801
1213
  status() {
802
1214
  return {
803
1215
  activatedAt: this.activatedAt,
804
1216
  connectionCount: this.connectionCount,
805
1217
  connections: {...this.connections},
1218
+ drain: this.drainStatus(),
806
1219
  drainStartedAt: this.drainStartedAt,
807
1220
  ports: {...this.ports},
808
1221
  processes: [...this.processes.values()].map((processInstance) => processInstance.status()),