rollbridge 0.1.73 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  24. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  25. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  26. package/docs/cli.md +128 -3
  27. package/docs/compose-services.md +222 -0
  28. package/docs/config.md +177 -6
  29. package/docs/troubleshooting.md +97 -6
  30. package/docs/velocious.md +1 -1
  31. package/docs/workers.md +11 -0
  32. package/examples/tensorbuzz.com.js +3 -2
  33. package/package.json +1 -1
  34. package/src/cli.js +68 -2
  35. package/src/compose-unit.js +725 -0
  36. package/src/config.js +346 -8
  37. package/src/daemon.js +411 -78
  38. package/src/doctor.js +74 -0
  39. package/src/drain-policy.js +252 -0
  40. package/src/guardian-client.js +68 -23
  41. package/src/health.js +99 -8
  42. package/src/managed-process.js +198 -45
  43. package/src/process-guardian.js +212 -37
  44. package/src/process-memory.js +58 -23
  45. package/src/process-order.js +26 -0
  46. package/src/release-group.js +438 -45
  47. package/tensorbuzz.yml +20 -1
  48. package/test/completion.test.js +5 -2
  49. package/test/compose-config.test.js +160 -0
  50. package/test/compose-docker.test.js +297 -0
  51. package/test/compose-unit.test.js +415 -0
  52. package/test/config-validation.test.js +135 -5
  53. package/test/drain-policy.test.js +328 -0
  54. package/test/fixtures/deferred-boot-app.js +106 -0
  55. package/test/fixtures/gated-app.js +50 -0
  56. package/test/guardian-client.test.js +119 -5
  57. package/test/health.test.js +74 -1
  58. package/test/managed-process.test.js +260 -24
  59. package/test/owner-recovery.test.js +125 -21
  60. package/test/owner-replacement.test.js +175 -14
  61. package/test/process-memory.test.js +47 -1
  62. package/test/release-group.test.js +408 -2
  63. package/test/rollbridge.test.js +774 -72
  64. package/test/shutdown-completion.test.js +119 -0
  65. package/test/shutdown-retirement-bound.test.js +162 -0
  66. package/test/support/process.js +3 -1
  67. package/test/tensorbuzz-config.test.js +228 -0
@@ -2,17 +2,29 @@
2
2
 
3
3
  import {spawn} from "node:child_process"
4
4
  import {EventEmitter} from "node:events"
5
+ import {classifyDrainProgress, decideWedgedEscalation} from "./drain-policy.js"
5
6
  import ManagedProcess from "./managed-process.js"
6
7
  import {findAvailablePort} from "./port-allocator.js"
8
+ import {processStartOrder} from "./process-order.js"
9
+ import {sampleProcessGroup} from "./process-memory.js"
7
10
  import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
8
- import {waitForHealth} from "./health.js"
11
+ import {waitForHealth, waitForListeningPort} from "./health.js"
12
+
13
+ const DEPENDENCY_READY_INTERVAL_MS = 250
14
+ // Shortest gap between two liveness observations that can prove a draining group's CPU is
15
+ // flat. Measured on TensorBuzz production (2026-11-14/15): honest drains ran 18-114 minutes
16
+ // with live runner children doing work, so flatness is only evidence below this window.
17
+ const DRAIN_SAMPLE_WINDOW_MS = 20000
9
18
 
10
19
  /**
11
20
  * @typedef {import("./json.js").JsonValue} JsonValue
12
21
  * @typedef {"starting" | "active" | "draining" | "stopped" | "failed"} ReleaseState
22
+ * @typedef {import("./drain-policy.js").DrainLivenessSample} DrainLivenessSample
23
+ * @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: import("./drain-policy.js").DrainProgress}} ReleaseDrainStatus
13
24
  * @typedef {{http: number, websocket: number}} ReleaseConnections
14
- * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
15
- * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean}} BuildProcessOptions
25
+ * @typedef {{activatedAt: string | undefined, connectionCount: number, connections: ReleaseConnections, drain: ReleaseDrainStatus | undefined, drainStartedAt: string | undefined, ports: Record<string, number>, processes: import("./managed-process.js").ManagedProcessStatus[], releaseId: string, releasePath: string, retirementError: string | undefined, revision: string, state: ReleaseState, stoppedAt: string | undefined}} ReleaseStatus
26
+ * @typedef {{count?: number, guardianKey?: string, index?: number, instanceId?: string, shouldRestart?: () => boolean, shouldStop?: () => boolean}} BuildProcessOptions
27
+ * @typedef {{deadlineMs: number, dependencyWaitMs: number, elapsedMs: number | undefined, failedProcessId: string, probes: number | undefined, stage: "dependency" | "health"}} StartupFailureDiagnostics
16
28
  */
17
29
 
18
30
 
@@ -57,6 +69,14 @@ function envId(id) {
57
69
  return id.replace(/[^a-zA-Z0-9]+/g, "_").toUpperCase()
58
70
  }
59
71
 
72
+ /**
73
+ * @param {number | undefined} startedAtMs - When the measured phase began.
74
+ * @returns {number | undefined} Milliseconds elapsed since then, or undefined when it never began.
75
+ */
76
+ function elapsedSince(startedAtMs) {
77
+ return startedAtMs === undefined ? undefined : Date.now() - startedAtMs
78
+ }
79
+
60
80
  /**
61
81
  * @param {import("./config.js").ProcessConfig} processConfig - Process config.
62
82
  * @param {number} index - Zero-based replica index.
@@ -77,9 +97,10 @@ export default class ReleaseGroup extends EventEmitter {
77
97
  * @param {string | undefined} args.revision - Revision.
78
98
  * @param {Record<string, number>} [args.servicePorts] - Ports already owned by daemon-wide services.
79
99
  * @param {() => boolean} [args.shouldStart] - Whether bootstrap may create another process.
100
+ * @param {string} [args.procPath] - Procfs root for drain liveness observations (overridable for deterministic tests).
80
101
  * @param {(key: string, definition: ConstructorParameters<typeof ManagedProcess>[0]) => ManagedProcess} [args.processFactory] - Durable process factory.
81
102
  */
82
- constructor({config, logger, portReservations = new Set(), processFactory, releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
103
+ constructor({config, logger, portReservations = new Set(), processFactory, procPath = "/proc", releaseId, releasePath, revision, servicePorts = {}, shouldStart = () => true}) {
83
104
  super()
84
105
 
85
106
  this.config = config
@@ -101,22 +122,36 @@ export default class ReleaseGroup extends EventEmitter {
101
122
  this.servicePorts = servicePorts
102
123
  this.shouldStart = shouldStart
103
124
  this.processFactory = processFactory
125
+ this.procPath = procPath
104
126
  this.portsAllocated = false
105
127
  this.drainStartedAt = /** @type {string | undefined} */ (undefined)
128
+ this.drainNewestSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
129
+ this.drainPriorSample = /** @type {DrainLivenessSample | undefined} */ (undefined)
106
130
  this.activatedAt = /** @type {string | undefined} */ (undefined)
107
131
  this.stoppedAt = /** @type {string | undefined} */ (undefined)
108
132
  this.retirementError = /** @type {string | undefined} */ (undefined)
109
133
  this.preserveConfigOnRetirement = false
134
+ this.processLaunchAtMs = /** @type {Map<string, number>} */ (new Map())
135
+ this.readyProcessIds = /** @type {Set<string>} */ (new Set())
136
+ this.startedProcessIds = /** @type {Set<string>} */ (new Set())
137
+ this.startupDependencyWaitMs = 0
138
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics | undefined} */ (undefined)
139
+ this.startupStartedAtMs = /** @type {number | undefined} */ (undefined)
110
140
  }
111
141
 
112
142
  /** @returns {Promise<void>} Starts release-owned processes and health checks the proxied process. */
113
143
  async start() {
114
144
  this.state = "starting"
145
+ this.startupStartedAtMs = Date.now()
115
146
 
116
147
  try {
117
148
  await this.allocatePorts()
118
149
 
119
150
  for (const processConfig of this.releaseProcessStartOrder()) {
151
+ await this.waitForConfiguredDependencies(processConfig)
152
+ /** @type {string[]} */
153
+ const instanceIds = []
154
+
120
155
  for (let index = 0; index < processConfig.replicas; index += 1) {
121
156
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
122
157
 
@@ -127,17 +162,17 @@ export default class ReleaseGroup extends EventEmitter {
127
162
  if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(instanceId)
128
163
  if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(instanceId)
129
164
  await processInstance.start("deploy")
165
+ this.processLaunchAtMs.set(instanceId, Date.now())
166
+ this.startedProcessIds.add(instanceId)
167
+ instanceIds.push(instanceId)
130
168
  }
131
169
 
132
170
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
133
- await waitForHealth({
134
- health: processConfig.health,
135
- host: this.config.proxy.upstreamHost,
136
- port: this.ports[processConfig.id]
137
- })
138
-
139
- if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
171
+ await this.waitForProxiedHealth(processConfig)
140
172
  }
173
+
174
+ for (const instanceId of instanceIds) this.readyProcessIds.add(instanceId)
175
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
141
176
  }
142
177
  } catch (error) {
143
178
  this.state = "failed"
@@ -151,9 +186,9 @@ export default class ReleaseGroup extends EventEmitter {
151
186
  /**
152
187
  * Reconstructs this release around processes still owned by the durable guardian.
153
188
  * @param {ReleaseStatus} snapshot - Persisted release snapshot.
154
- * @param {{adoptCommittedGeneration?: boolean, configDigest?: string, synchronizeLifecycleRole?: boolean}} [options] - Guardian role synchronization options.
189
+ * @param {{synchronizeLifecycleRole?: boolean}} [options] - Guardian role synchronization options.
155
190
  */
156
- async restore(snapshot, {adoptCommittedGeneration = false, configDigest, synchronizeLifecycleRole = true} = {}) {
191
+ async restore(snapshot, {synchronizeLifecycleRole = true} = {}) {
157
192
  if (!snapshot || snapshot.releaseId !== this.releaseId || snapshot.releasePath !== this.releasePath || snapshot.revision !== this.revision) {
158
193
  throw new Error(`Persisted release identity mismatch for ${this.releaseId}`)
159
194
  }
@@ -214,12 +249,7 @@ export default class ReleaseGroup extends EventEmitter {
214
249
  this.processes.set(processStatus.id, processInstance)
215
250
  if (processConfig.policy === "service" && processConfig.deployStrategy === "handoff") this.handoffServiceIds.add(processStatus.id)
216
251
  if (processConfig.nonBlockingDrain) this.nonBlockingDrainIds.add(processStatus.id)
217
- if ("recover" in processInstance && typeof processInstance.recover === "function") {
218
- const adopt = adoptCommittedGeneration && processStatus.state === "stopped" && processStatus.pid && processStatus.processStartTime && configDigest
219
- ? {pid: processStatus.pid, processStartTime: processStatus.processStartTime, configDigest}
220
- : undefined
221
- await processInstance.recover(adopt ? {adopt} : undefined)
222
- }
252
+ if ("recover" in processInstance && typeof processInstance.recover === "function") await processInstance.recover()
223
253
  }
224
254
  if (synchronizeLifecycleRole) await this.synchronizeLifecycleRoles()
225
255
  }
@@ -262,16 +292,132 @@ export default class ReleaseGroup extends EventEmitter {
262
292
  }
263
293
 
264
294
  /**
265
- * Logs process diagnostics around failed startup cleanup.
295
+ * Waits for the ports a process declares it depends on, on a budget that is not the
296
+ * health budget. The dependency leg of a start is not app boot, and charging it to the
297
+ * health deadline measures the dependency instead of the app.
298
+ * @param {import("./config.js").ProcessConfig} processConfig - Process about to start.
299
+ * @returns {Promise<void>} Resolves once every dependency endpoint accepts a connection.
300
+ */
301
+ async waitForConfiguredDependencies(processConfig) {
302
+ if (processConfig.dependsOn.length === 0) return
303
+
304
+ for (const dependencyId of processConfig.dependsOn) {
305
+ const port = this.ports[dependencyId] ?? this.servicePorts[dependencyId]
306
+
307
+ if (port === undefined) throw new Error(`Process ${processConfig.id} depends on ${dependencyId}, which has no allocated port for release ${this.releaseId}`)
308
+
309
+ this.logger("release process dependency wait", {
310
+ dependencyPort: port,
311
+ dependencyProcessId: dependencyId,
312
+ processId: processConfig.id,
313
+ releaseId: this.releaseId,
314
+ timeoutMs: processConfig.dependencyTimeoutMs
315
+ })
316
+
317
+ const waitStartedAtMs = Date.now()
318
+
319
+ try {
320
+ const {attempts, elapsedMs} = await waitForListeningPort({
321
+ host: this.config.proxy.upstreamHost,
322
+ intervalMs: DEPENDENCY_READY_INTERVAL_MS,
323
+ port,
324
+ timeoutMs: processConfig.dependencyTimeoutMs
325
+ })
326
+
327
+ this.startupDependencyWaitMs += elapsedMs
328
+ this.logger("release process dependency ready", {
329
+ attempts,
330
+ dependencyPort: port,
331
+ dependencyProcessId: dependencyId,
332
+ elapsedMs,
333
+ processId: processConfig.id,
334
+ releaseId: this.releaseId
335
+ })
336
+ } catch (error) {
337
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
338
+ deadlineMs: processConfig.dependencyTimeoutMs,
339
+ dependencyWaitMs: this.startupDependencyWaitMs + (Date.now() - waitStartedAtMs),
340
+ elapsedMs: elapsedSince(this.startupStartedAtMs),
341
+ failedProcessId: processConfig.id,
342
+ probes: undefined,
343
+ stage: "dependency"
344
+ })
345
+ throw new Error(`Process ${processConfig.id} could not start because dependency ${dependencyId} was not ready: ${error instanceof Error ? error.message : String(error)}`, {cause: error})
346
+ }
347
+
348
+ if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
349
+ }
350
+ }
351
+
352
+ /**
353
+ * Health checks the proxied process on its own boot budget and records what the boot cost.
354
+ * @param {import("./config.js").ProcessConfig} processConfig - Proxied process config.
355
+ * @returns {Promise<void>} Resolves once the process answers its health path.
356
+ */
357
+ async waitForProxiedHealth(processConfig) {
358
+ const launchedAtMs = this.processLaunchAtMs.get(processConfig.id) ?? Date.now()
359
+ const health = /** @type {import("./config.js").HealthConfig} */ (processConfig.health)
360
+
361
+ try {
362
+ const {elapsedMs} = await waitForHealth({
363
+ health,
364
+ host: this.config.proxy.upstreamHost,
365
+ port: /** @type {number} */ (this.ports[processConfig.id]),
366
+ processId: processConfig.id
367
+ })
368
+
369
+ this.logger("release candidate health ready", {
370
+ bootMs: Date.now() - launchedAtMs,
371
+ deadlineMs: health.timeoutMs,
372
+ healthGateMs: elapsedMs,
373
+ path: health.path,
374
+ port: this.ports[processConfig.id],
375
+ processId: processConfig.id,
376
+ releaseId: this.releaseId,
377
+ waitedForDependenciesMs: this.startupDependencyWaitMs
378
+ })
379
+ } catch (error) {
380
+ const healthCheck = error && typeof error === "object" && "healthCheck" in error ? /** @type {import("./health.js").HealthCheckError} */ (error).healthCheck : undefined
381
+
382
+ this.startupFailureDiagnostics = /** @type {StartupFailureDiagnostics} */ ({
383
+ deadlineMs: health.timeoutMs,
384
+ dependencyWaitMs: this.startupDependencyWaitMs,
385
+ elapsedMs: elapsedSince(this.startupStartedAtMs),
386
+ failedProcessId: processConfig.id,
387
+ probes: healthCheck?.attempts,
388
+ stage: "health"
389
+ })
390
+ throw error
391
+ }
392
+ }
393
+
394
+ /**
395
+ * Logs the startup budget that ran out, then process diagnostics around failed startup cleanup.
266
396
  * @param {Error | string} error - Startup failure.
267
397
  * @param {{phase: string}} options - Diagnostic phase.
268
398
  * @returns {void}
269
399
  */
270
400
  logStartupFailure(error, {phase}) {
401
+ const diagnostics = this.startupFailureDiagnostics
402
+ const failedLogs = diagnostics ? this.processes.get(diagnostics.failedProcessId)?.status().logs : undefined
403
+ const lastLog = failedLogs?.at(-1)
404
+ const readyProcesses = [...this.readyProcessIds]
405
+
271
406
  this.logger("release startup failed", {
407
+ deadlineMs: diagnostics?.deadlineMs ?? null,
408
+ dependencyWaitMs: diagnostics?.dependencyWaitMs ?? null,
409
+ elapsedMs: diagnostics?.elapsedMs ?? null,
272
410
  error: error instanceof Error ? error.message : error,
411
+ failedProcessId: diagnostics?.failedProcessId ?? null,
412
+ lastOutput: lastLog?.line ?? null,
413
+ lastOutputAt: lastLog?.at ?? null,
273
414
  phase,
274
- releaseId: this.releaseId
415
+ probes: diagnostics?.probes ?? null,
416
+ readyProcesses,
417
+ releaseElapsedMs: elapsedSince(this.startupStartedAtMs),
418
+ releaseId: this.releaseId,
419
+ stage: diagnostics?.stage ?? null,
420
+ unreadyProcesses: [...this.startedProcessIds].filter((instanceId) => !this.readyProcessIds.has(instanceId))
275
421
  })
276
422
 
277
423
  for (const processInstance of this.processes.values()) {
@@ -296,18 +442,14 @@ export default class ReleaseGroup extends EventEmitter {
296
442
  * @returns {import("./config.js").ProcessConfig[]} Ordered process configs.
297
443
  */
298
444
  releaseProcessStartOrder() {
299
- const releaseProcesses = this.config.processes.filter((processConfig) => processConfig.policy !== "singleton" && (processConfig.policy !== "service" || processConfig.deployStrategy === "handoff"))
300
- const serviceProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "service")
301
- const companionProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "companion")
302
- const proxiedProcesses = releaseProcesses.filter((processConfig) => processConfig.policy === "proxied")
303
-
304
- return [...serviceProcesses, ...companionProcesses, ...proxiedProcesses]
445
+ return processStartOrder(this.config.processes)
305
446
  }
306
447
 
307
448
  /** @returns {void} Marks this release active. */
308
449
  activate() {
309
450
  this.state = "active"
310
451
  this.activatedAt = new Date().toISOString()
452
+ this.clearDrainLivenessSamples()
311
453
  }
312
454
 
313
455
  /** Runs the configured release-generation activation acknowledgement, if any. */
@@ -362,6 +504,16 @@ export default class ReleaseGroup extends EventEmitter {
362
504
  this.drainStartedAt = undefined
363
505
  this.retirementError = undefined
364
506
  this.stoppedAt = undefined
507
+ this.clearDrainLivenessSamples()
508
+ }
509
+
510
+ /** Stops only the proxied processes when a failed compensation cannot restore jobs. */
511
+ async stopProxyTraffic() {
512
+ const proxiedIds = new Set(this.config.processes.filter((processConfig) => processConfig.policy === "proxied").map((processConfig) => processConfig.id))
513
+
514
+ await Promise.allSettled([...this.processes.entries()]
515
+ .filter(([id]) => proxiedIds.has(id.replace(/#\d+$/, "")))
516
+ .map(([, processInstance]) => processInstance.stop()))
365
517
  }
366
518
 
367
519
  /**
@@ -372,24 +524,25 @@ export default class ReleaseGroup extends EventEmitter {
372
524
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
373
525
  this.assertCommittedGenerationRecoverable()
374
526
  this.state = "starting"
527
+ this.startupStartedAtMs = Date.now()
375
528
  try {
376
529
  for (const processConfig of this.releaseProcessStartOrder()) {
530
+ await this.waitForConfiguredDependencies(processConfig)
377
531
  const instances = this.getProcesses(processConfig.id)
378
532
 
379
533
  if (instances.length !== processConfig.replicas) throw new Error(`Committed generation ${this.releaseId} is missing process ${processConfig.id}`)
380
- for (const {process} of instances) {
534
+ for (const {id, process} of instances) {
381
535
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
382
536
  await process.start("deploy", processConfig.lifecycle.activateCommand ? "candidate" : undefined)
537
+ this.processLaunchAtMs.set(id, Date.now())
538
+ this.startedProcessIds.add(id)
383
539
  }
384
540
 
385
541
  if (processConfig.policy === "proxied" && processConfig.port && processConfig.health) {
386
- await waitForHealth({
387
- health: processConfig.health,
388
- host: this.config.proxy.upstreamHost,
389
- port: this.ports[processConfig.id]
390
- })
542
+ await this.waitForProxiedHealth(processConfig)
391
543
  if (!this.shouldStart()) throw new Error("Rollbridge is shutting down")
392
544
  }
545
+ for (const {id} of instances) this.readyProcessIds.add(id)
393
546
  }
394
547
  } catch (error) {
395
548
  await this.abortCommittedGenerationRestart()
@@ -481,6 +634,10 @@ export default class ReleaseGroup extends EventEmitter {
481
634
  ...this.baseEnvironment(processConfig, {count, index}),
482
635
  ...renderedEnv
483
636
  }
637
+ const dependencyRetired = () => processConfig.dependsOn.some((dependencyId) => {
638
+ const dependency = this.processes.get(dependencyId)
639
+ return dependency?.lifecycleRole === "retired" || dependency?.status().lifecycleRole === "retired"
640
+ })
484
641
 
485
642
  return {
486
643
  command: renderTemplate(processConfig.command, context),
@@ -493,7 +650,15 @@ export default class ReleaseGroup extends EventEmitter {
493
650
  outputLines: processConfig.outputLines,
494
651
  restart: processConfig.restart,
495
652
  restartDelayMs: processConfig.restartDelayMs,
496
- shouldRestart: options.shouldRestart || (() => this.state === "active" || this.state === "starting"),
653
+ shouldRestart: options.shouldRestart || (() => {
654
+ if (this.state !== "active" && this.state !== "starting") return false
655
+
656
+ // A companion whose same-generation service dependency has retired can never
657
+ // satisfy its startup gate again. Treat that dependency lifecycle as terminal
658
+ // for this process so an exit does not become an endless supervisor restart loop.
659
+ return !dependencyRetired()
660
+ }),
661
+ shouldStop: options.shouldStop || dependencyRetired,
497
662
  stopSignal: processConfig.stopSignal,
498
663
  stopTimeoutMs: processConfig.gracefulStopMs
499
664
  }
@@ -713,10 +878,21 @@ export default class ReleaseGroup extends EventEmitter {
713
878
  // their lifecycle drain runs as soon as the release is retired — in parallel with the
714
879
  // connection drain, not held until after it. The rest stop once connections have closed.
715
880
  const entries = [...this.processes.entries()]
716
- const stopOptions = {timeoutMs: config.proxy.forceStopTimeoutMs}
717
- const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([, processInstance]) => processInstance.stop(stopOptions))
881
+ const nonBlockingStops = entries.filter(([id]) => this.nonBlockingDrainIds.has(id)).map(([id, processInstance]) => ({
882
+ id,
883
+ promise: processInstance.stop({retirementTimeoutMs: config.proxy.nonBlockingDrainTimeoutMs})
884
+ }))
718
885
  const handoffServices = entries.filter(([id]) => this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
719
886
  const connectionDependent = entries.filter(([id]) => !this.nonBlockingDrainIds.has(id) && !this.handoffServiceIds.has(id)).map(([, processInstance]) => processInstance)
887
+ // Bracket the retirement wait with a pair of observations: this one, and the one taken when
888
+ // every stop has returned. The gap between them is the wait the transition already performs
889
+ // (nonBlockingDrainTimeoutMs at minimum), so CPU flatness can be proven from the transition
890
+ // itself rather than from a sampler running alongside it.
891
+ const retirementBaselineSample = config.releaseRetention.escalateWedgedGenerations && nonBlockingStops.length > 0
892
+ ? this.takeLivenessSample(Date.now())
893
+ : undefined
894
+
895
+ if (retirementBaselineSample) this.drainNewestSample = retirementBaselineSample
720
896
 
721
897
  if (this.connectionCount > 0) {
722
898
  await new Promise((resolve) => {
@@ -735,17 +911,40 @@ export default class ReleaseGroup extends EventEmitter {
735
911
  }
736
912
 
737
913
  if (this.ownerHandoffPaused) {
738
- void Promise.allSettled(nonBlockingStops)
914
+ void Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
739
915
  return
740
916
  }
741
917
 
742
- await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop(stopOptions)))
743
- await Promise.allSettled(nonBlockingStops)
744
- await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop(stopOptions)))
918
+ const retirementTimeoutMs = config.proxy.nonBlockingDrainTimeoutMs
919
+ await Promise.allSettled(connectionDependent.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
920
+ const nonBlockingResults = await Promise.allSettled(nonBlockingStops.map(({promise}) => promise))
921
+ nonBlockingResults.forEach((result, index) => {
922
+ if (result.status === "fulfilled" && this.processes.get(nonBlockingStops[index].id)?.retirementTimedOut) {
923
+ const processInstance = this.processes.get(nonBlockingStops[index].id)
924
+ this.logger("release non-blocking process still draining", {
925
+ pid: processInstance?.status().pid ?? null,
926
+ processId: nonBlockingStops[index].id,
927
+ releaseId: this.releaseId,
928
+ timeoutMs: config.proxy.nonBlockingDrainTimeoutMs
929
+ })
930
+ }
931
+ })
932
+ await Promise.allSettled(handoffServices.map((processInstance) => processInstance.stop({retirementTimeoutMs})))
933
+
934
+ // Last act of the retirement transition, before the release is recorded stopped: whatever
935
+ // is still alive here is the residue, and the bracket pair above can now say whether it is
936
+ // wedged. A generation that stops with a live process group is the state that made
937
+ // operators signal PIDs by hand (STOE-1687), so the transition decides it or reports it —
938
+ // it does not end silently with the process still holding the release.
939
+ if (retirementBaselineSample) await this.escalateWedgedGeneration({previousSample: retirementBaselineSample, reason: "release retirement"})
940
+
745
941
  this.state = "stopped"
746
942
  this.stoppedAt = new Date().toISOString()
747
943
  this.releasePortReservations()
748
944
  this.abandonLocalConnections()
945
+ // The bracket pair is deliberately kept: a generation that stopped with a live process
946
+ // group stays observable, and the deploy path needs the older observation to prove the
947
+ // residue is still flat rather than starting a fresh window from zero.
749
948
  }
750
949
 
751
950
  /**
@@ -765,12 +964,18 @@ export default class ReleaseGroup extends EventEmitter {
765
964
  this.drainStartedAt = new Date().toISOString()
766
965
  }
767
966
 
967
+ // Refresh the live process definitions with the current retirement policy,
968
+ // but retain this generation's exact config identity for owner recovery.
969
+ // Replacing the stored generation config here would make a still-running
970
+ // singleton from the previous release fail the guardian provenance fence.
971
+ const generationConfig = this.config
972
+
768
973
  this.refreshProcessDefinitions(config)
974
+ this.config = generationConfig
769
975
  const generationIds = new Set([...this.handoffServiceIds, ...this.nonBlockingDrainIds])
770
976
  const results = await Promise.allSettled([...this.processes.entries()]
771
977
  .filter(([id]) => generationIds.has(id))
772
978
  .map(([, processInstance]) => retry ? processInstance.requiesceStrict() : processInstance.quiesceStrict()))
773
-
774
979
  const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason)
775
980
 
776
981
  if (errors.length > 0) {
@@ -781,15 +986,44 @@ export default class ReleaseGroup extends EventEmitter {
781
986
  this.retirementError = undefined
782
987
  }
783
988
 
784
- /** @returns {Promise<void>} Stops all release-owned processes. */
785
- async stop() {
786
- const stopTasks = [...this.processes.values()].map((processInstance) => processInstance.stop())
989
+ /**
990
+ * Stops every process owned by the generation.
991
+ * @param {{retirementTimeoutMs?: number}} [options] - Retirement budget for the processes the
992
+ * supervisor must not kill: a non-blocking-drain process, and any process whose graceful window
993
+ * is `"indefinite"`. Without it those stops have no deadline at all, so an explicit
994
+ * `rollbridge stop` can wait forever — and because that command runs inside an owner mutation,
995
+ * the wait holds the mutation fence and every later deploy is refused (STOE-2015).
996
+ * @returns {Promise<void>} Resolves when stopped, or when retirement stops waiting.
997
+ */
998
+ async stop(options = {}) {
999
+ const {retirementTimeoutMs} = options
1000
+ /** @type {{bounded: boolean, id: string, instance: ManagedProcess, promise: Promise<void>}[]} */
1001
+ const stopTasks = [...this.processes.entries()].map(([id, processInstance]) => {
1002
+ const bounded = retirementTimeoutMs !== undefined && (this.nonBlockingDrainIds.has(id) || processInstance.hasUnboundedGracefulWindow())
1003
+
1004
+ return {bounded, id, instance: processInstance, promise: processInstance.stop(bounded ? {retirementTimeoutMs} : {})}
1005
+ })
1006
+
1007
+ await Promise.allSettled(stopTasks.map(({promise}) => promise))
1008
+
1009
+ // A stop that stopped waiting leaves the process running. Name it, so the operator sees the
1010
+ // residue instead of a command that produced nothing.
1011
+ for (const {bounded, id, instance} of stopTasks) {
1012
+ if (!bounded || !instance.retirementTimedOut) continue
1013
+
1014
+ this.logger("release stop gave up waiting for a retiring process", {
1015
+ pid: instance.status().pid ?? null,
1016
+ processId: id,
1017
+ releaseId: this.releaseId,
1018
+ timeoutMs: retirementTimeoutMs
1019
+ })
1020
+ }
787
1021
 
788
- await Promise.allSettled(stopTasks)
789
1022
  this.state = "stopped"
790
1023
  this.stoppedAt = new Date().toISOString()
791
1024
  this.releasePortReservations()
792
1025
  this.abandonLocalConnections()
1026
+ this.clearDrainLivenessSamples()
793
1027
  }
794
1028
 
795
1029
  /** @returns {Promise<void>} Quiesces every release process without waiting for its drain. */
@@ -797,12 +1031,171 @@ export default class ReleaseGroup extends EventEmitter {
797
1031
  await Promise.all([...this.processes.values()].map((processInstance) => processInstance.quiesce()))
798
1032
  }
799
1033
 
1034
+ /**
1035
+ * Records one liveness observation for a draining generation and classifies its drain.
1036
+ *
1037
+ * Two observations are needed to prove CPU flatness, and reading a process group means
1038
+ * scanning procfs, so sampling is bounded three ways: a generation that still owns live
1039
+ * connections is classified without touching procfs at all (connections alone settle it), a
1040
+ * new observation is taken only once the newest one is already a full window behind the
1041
+ * present read, and every other read reuses the newest observation. A group that is not
1042
+ * draining reports no drain.
1043
+ * @returns {ReleaseDrainStatus | undefined} The classification, or undefined when not draining.
1044
+ */
1045
+ drainStatus() {
1046
+ if (this.state !== "draining") return undefined
1047
+
1048
+ return this.observeRetirement({nowMs: Date.now()}).classification
1049
+ }
1050
+
1051
+ /**
1052
+ * Records the newest liveness observation of this generation and classifies its retirement.
1053
+ * @param {object} options - Observation options.
1054
+ * @param {number} options.nowMs - Read time.
1055
+ * @param {DrainLivenessSample | undefined} [options.previousSample] - Explicit older observation to classify against. The retirement transition brackets its own wait with one instead of reusing a stored pair.
1056
+ * @returns {{classification: ReleaseDrainStatus, currentSample: DrainLivenessSample}} The observation the decision acts on, and what the pair says.
1057
+ */
1058
+ observeRetirement({nowMs, previousSample}) {
1059
+ const drainBudgetMs = this.config.proxy.drainTimeoutMs
1060
+ const drainElapsedMs = this.drainStartedAt === undefined ? 0 : nowMs - Date.parse(this.drainStartedAt)
1061
+
1062
+ if (previousSample === undefined && this.connectionCount > 0) {
1063
+ const currentSample = this.drainNewestSample ?? {atMs: nowMs, processes: []}
1064
+
1065
+ return {
1066
+ classification: classifyDrainProgress({
1067
+ connectionCount: this.connectionCount,
1068
+ currentSample,
1069
+ drainBudgetMs,
1070
+ drainElapsedMs,
1071
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1072
+ previousSample: undefined
1073
+ }),
1074
+ currentSample
1075
+ }
1076
+ }
1077
+
1078
+ const newest = this.drainNewestSample
1079
+ const currentSample = previousSample !== undefined || newest === undefined || nowMs - newest.atMs >= DRAIN_SAMPLE_WINDOW_MS
1080
+ ? this.takeLivenessSample(nowMs)
1081
+ : newest
1082
+ const priorSample = previousSample ?? (currentSample === newest ? this.drainPriorSample : newest)
1083
+
1084
+ if (currentSample !== newest) {
1085
+ this.drainPriorSample = newest
1086
+ this.drainNewestSample = currentSample
1087
+ }
1088
+
1089
+ return {
1090
+ classification: classifyDrainProgress({
1091
+ connectionCount: this.connectionCount,
1092
+ currentSample,
1093
+ drainBudgetMs,
1094
+ drainElapsedMs,
1095
+ minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS,
1096
+ previousSample: priorSample
1097
+ }),
1098
+ currentSample
1099
+ }
1100
+ }
1101
+
1102
+ /**
1103
+ * Applies the wedge escalation to this generation: terminates the processes a proven wedge
1104
+ * proves are holding nothing, and names the ones it refused to terminate.
1105
+ *
1106
+ * Both halves are required. The card this path was written for (STOE-1687) is about a
1107
+ * wedged worker holding a release open for four hours while deploys stack behind it; the
1108
+ * boundary on the same card (STOE-1077) is that no process still holding live work may ever
1109
+ * be SIGKILLed. So a wedge that is not kill-safe is not silently skipped — it is reported
1110
+ * with the pids and process groups it is holding, which is the information an operator needs
1111
+ * in place of guessing.
1112
+ * @param {object} options - Escalation context.
1113
+ * @param {DrainLivenessSample | undefined} [options.previousSample] - Older observation bracketing a wait this caller already performed.
1114
+ * @param {string} options.reason - Named context recorded with the decision.
1115
+ * @returns {Promise<string[]>} Instance ids of the processes the SIGKILL was delivered to.
1116
+ */
1117
+ async escalateWedgedGeneration({previousSample, reason}) {
1118
+ const {classification, currentSample} = this.observeRetirement({nowMs: Date.now(), previousSample})
1119
+ const decision = decideWedgedEscalation({classification, currentSample, minSampleWindowMs: DRAIN_SAMPLE_WINDOW_MS})
1120
+ const liveObservations = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
1121
+
1122
+ if (decision.decision === "report") {
1123
+ if (classification.status === "wedged") {
1124
+ this.logger("release wedged generation left alive", {
1125
+ escalation: "refused",
1126
+ heldProcessGroupPgids: liveObservations.map((process) => process.heldProcessGroupPgid ?? null),
1127
+ pids: liveObservations.map((process) => process.pid ?? null),
1128
+ processIds: liveObservations.map((process) => process.id),
1129
+ reason: decision.reason,
1130
+ releaseId: this.releaseId,
1131
+ trigger: reason
1132
+ })
1133
+ }
1134
+
1135
+ return []
1136
+ }
1137
+
1138
+ const escalated = []
1139
+
1140
+ for (const target of decision.targets) {
1141
+ const processInstance = this.processes.get(target.processId)
1142
+
1143
+ if (!processInstance) continue
1144
+ if (await processInstance.escalateRetirement({reason: `${reason}: ${decision.reason}`})) escalated.push(target.processId)
1145
+ }
1146
+
1147
+ if (escalated.length > 0) {
1148
+ this.logger("release wedged generation escalated", {
1149
+ escalation: "SIGKILL",
1150
+ processIds: escalated,
1151
+ reason: decision.reason,
1152
+ releaseId: this.releaseId,
1153
+ trigger: reason
1154
+ })
1155
+ }
1156
+
1157
+ return escalated
1158
+ }
1159
+
1160
+ /**
1161
+ * @param {number} nowMs - Observation time.
1162
+ * @returns {DrainLivenessSample} One liveness observation of every process in the generation.
1163
+ */
1164
+ takeLivenessSample(nowMs) {
1165
+ const processes = [...this.processes.entries()].map(([id, processInstance]) => {
1166
+ const processStatus = processInstance.status()
1167
+ const heldProcessGroupPgid = processStatus.heldProcessGroup?.pgid
1168
+ // A group that outlived its tracked process is still observed by group id: it holds the
1169
+ // generation's work even though no rollbridge record can signal it any more.
1170
+ const pgid = processStatus.pid ?? heldProcessGroupPgid
1171
+ const sample = pgid === undefined ? undefined : sampleProcessGroup(pgid, this.procPath)
1172
+
1173
+ return {
1174
+ cpuMs: sample?.cpuMs,
1175
+ heldProcessGroupPgid,
1176
+ id,
1177
+ liveChildCount: sample === undefined ? undefined : Math.max(0, sample.liveMemberCount - 1),
1178
+ pid: processStatus.pid,
1179
+ retirementTimedOut: processInstance.retirementTimedOut === true
1180
+ }
1181
+ })
1182
+
1183
+ return {atMs: nowMs, processes}
1184
+ }
1185
+
1186
+ /** @returns {void} Drops the liveness observations of a generation that is no longer draining. */
1187
+ clearDrainLivenessSamples() {
1188
+ this.drainNewestSample = undefined
1189
+ this.drainPriorSample = undefined
1190
+ }
1191
+
800
1192
  /** @returns {ReleaseStatus} Status payload. */
801
1193
  status() {
802
1194
  return {
803
1195
  activatedAt: this.activatedAt,
804
1196
  connectionCount: this.connectionCount,
805
1197
  connections: {...this.connections},
1198
+ drain: this.drainStatus(),
806
1199
  drainStartedAt: this.drainStartedAt,
807
1200
  ports: {...this.ports},
808
1201
  processes: [...this.processes.values()].map((processInstance) => processInstance.status()),