rollbridge 0.1.73 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +63 -10
  2. package/TODO.md +13 -0
  3. package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
  4. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
  5. package/changelog.d/20261104-compose-service-units.md +21 -0
  6. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
  7. package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
  8. package/changelog.d/20261109-path-filtered-builds.md +11 -0
  9. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
  10. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
  11. package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
  12. package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
  13. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
  14. package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
  15. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
  16. package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
  17. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
  18. package/changelog.d/20261116-guardian-version-handoff.md +37 -0
  19. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
  20. package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
  21. package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
  22. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
  23. package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
  24. package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
  25. package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
  26. package/docs/cli.md +128 -3
  27. package/docs/compose-services.md +222 -0
  28. package/docs/config.md +177 -6
  29. package/docs/troubleshooting.md +97 -6
  30. package/docs/velocious.md +1 -1
  31. package/docs/workers.md +11 -0
  32. package/examples/tensorbuzz.com.js +3 -2
  33. package/package.json +1 -1
  34. package/src/cli.js +68 -2
  35. package/src/compose-unit.js +725 -0
  36. package/src/config.js +346 -8
  37. package/src/daemon.js +411 -78
  38. package/src/doctor.js +74 -0
  39. package/src/drain-policy.js +252 -0
  40. package/src/guardian-client.js +68 -23
  41. package/src/health.js +99 -8
  42. package/src/managed-process.js +198 -45
  43. package/src/process-guardian.js +212 -37
  44. package/src/process-memory.js +58 -23
  45. package/src/process-order.js +26 -0
  46. package/src/release-group.js +438 -45
  47. package/tensorbuzz.yml +20 -1
  48. package/test/completion.test.js +5 -2
  49. package/test/compose-config.test.js +160 -0
  50. package/test/compose-docker.test.js +297 -0
  51. package/test/compose-unit.test.js +415 -0
  52. package/test/config-validation.test.js +135 -5
  53. package/test/drain-policy.test.js +328 -0
  54. package/test/fixtures/deferred-boot-app.js +106 -0
  55. package/test/fixtures/gated-app.js +50 -0
  56. package/test/guardian-client.test.js +119 -5
  57. package/test/health.test.js +74 -1
  58. package/test/managed-process.test.js +260 -24
  59. package/test/owner-recovery.test.js +125 -21
  60. package/test/owner-replacement.test.js +175 -14
  61. package/test/process-memory.test.js +47 -1
  62. package/test/release-group.test.js +408 -2
  63. package/test/rollbridge.test.js +774 -72
  64. package/test/shutdown-completion.test.js +119 -0
  65. package/test/shutdown-retirement-bound.test.js +162 -0
  66. package/test/support/process.js +3 -1
  67. package/test/tensorbuzz-config.test.js +228 -0
@@ -2,10 +2,17 @@
2
2
 
3
3
  import {EventEmitter} from "node:events"
4
4
  import {spawn} from "node:child_process"
5
- import {processGroupHasLiveMembers, processGroupMembers, processStartTime} from "./process-memory.js"
5
+ import {processGroupHasLiveMembers, processGroupMembers, sampleProcessGroup} from "./process-memory.js"
6
6
 
7
7
  const DEFAULT_ACTIVATION_HOOK_TIMEOUT_MS = 30000
8
8
  const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
9
+ // Reading a process group means scanning procfs, and status() is read continuously. A held
10
+ // group is re-verified on this interval instead of on every read.
11
+ const HELD_GROUP_RECHECK_MS = 5000
12
+ // How long an escalated SIGKILL is given to clear the group before the group is recorded as
13
+ // outliving it again. Bounded so the escalation itself can never become the unbounded wait it
14
+ // replaced.
15
+ const ESCALATION_KILL_CONFIRM_MS = 5000
9
16
 
10
17
  /**
11
18
  * @typedef {import("./json.js").JsonValue} JsonValue
@@ -15,8 +22,9 @@ const MAX_BUFFERED_OUTPUT_CHARACTERS = 64 * 1024
15
22
  * @typedef {import("node:child_process").ChildProcess["signalCode"]} ProcessExitSignal
16
23
  * @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
17
24
  * @typedef {import("./config.js").StopTimeoutMs} StopTimeoutMs
18
- * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
19
- * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, processStartTime?: string, restarts: number, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
25
+ * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, shouldStop?: () => boolean, stopSignal: string, stopTimeoutMs: StopTimeoutMs}} ManagedProcessDefinition
26
+ * @typedef {{liveMemberCount: number | undefined, pgid: number}} HeldProcessGroup
27
+ * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, heldProcessGroup: HeldProcessGroup | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, lifecycleRole?: LifecycleRole, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, retirementTimedOut: boolean, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
20
28
  */
21
29
 
22
30
  export default class ManagedProcess extends EventEmitter {
@@ -33,10 +41,11 @@ export default class ManagedProcess extends EventEmitter {
33
41
  * @param {import("./config.js").RestartConfig} [args.restart] - Restart policy (defaults to unlimited restarts with a constant delay).
34
42
  * @param {number} args.restartDelayMs - Restart delay.
35
43
  * @param {() => boolean} args.shouldRestart - Restart policy callback.
44
+ * @param {() => boolean} [args.shouldStop] - Whether an unexpected exit is terminal because its dependency retired.
36
45
  * @param {string} [args.stopSignal] - Signal sent to gracefully stop the process (default "SIGTERM").
37
46
  * @param {StopTimeoutMs} args.stopTimeoutMs - Stop timeout.
38
47
  */
39
- constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, stopSignal = "SIGTERM", stopTimeoutMs}) {
48
+ constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, shouldStop = () => false, stopSignal = "SIGTERM", stopTimeoutMs}) {
40
49
  super()
41
50
 
42
51
  this.command = command
@@ -50,6 +59,7 @@ export default class ManagedProcess extends EventEmitter {
50
59
  this.restart = restart
51
60
  this.restartDelayMs = restartDelayMs
52
61
  this.shouldRestart = shouldRestart
62
+ this.shouldStop = shouldStop
53
63
  this.stopSignal = stopSignal
54
64
  this.stopTimeoutMs = stopTimeoutMs
55
65
  this.state = /** @type {ManagedProcessState} */ ("stopped")
@@ -73,12 +83,15 @@ export default class ManagedProcess extends EventEmitter {
73
83
  this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
74
84
  this.quiesceError = /** @type {Error | undefined} */ (undefined)
75
85
  this.stopPromise = /** @type {Promise<void> | undefined} */ (undefined)
86
+ this.retirementTimedOut = false
87
+ this.groupOutlivedStop = /** @type {number | undefined} */ (undefined)
88
+ this.heldProcessGroupCheckedAtMs = /** @type {number | undefined} */ (undefined)
89
+ this.heldProcessGroupCount = /** @type {number | undefined} */ (undefined)
76
90
  this.operationRevision = 0
77
91
  this.restartTimer = undefined
78
92
  this.child = undefined
79
93
  this.exitPromise = undefined
80
94
  this.pid = undefined
81
- this.processStartTime = undefined
82
95
  this.exitCode = undefined
83
96
  this.exitSignal = undefined
84
97
  }
@@ -101,6 +114,10 @@ export default class ManagedProcess extends EventEmitter {
101
114
  this.quiescePromise = undefined
102
115
  this.quiesceError = undefined
103
116
  this.stopPromise = undefined
117
+ this.retirementTimedOut = false
118
+ this.groupOutlivedStop = undefined
119
+ this.heldProcessGroupCheckedAtMs = undefined
120
+ this.heldProcessGroupCount = undefined
104
121
  this.exitCode = undefined
105
122
  this.exitSignal = undefined
106
123
  this.state = "starting"
@@ -127,7 +144,6 @@ export default class ManagedProcess extends EventEmitter {
127
144
  child.once("spawn", () => {
128
145
  void (async () => {
129
146
  this.startedAtMs = Date.now()
130
- this.processStartTime = child.pid ? processStartTime(child.pid) : undefined
131
147
  this.lastStartReason = reason
132
148
  const lifecycleRestore = this.restoreLifecycleRole()
133
149
  const lifecycleRestoreBarrier = lifecycleRestore.then(() => undefined, () => undefined)
@@ -163,7 +179,6 @@ export default class ManagedProcess extends EventEmitter {
163
179
  if (this.child === child) {
164
180
  this.child = undefined
165
181
  this.pid = undefined
166
- this.processStartTime = undefined
167
182
  this.exitPromise = undefined
168
183
  }
169
184
  reject(error)
@@ -177,29 +192,6 @@ export default class ManagedProcess extends EventEmitter {
177
192
  })
178
193
  }
179
194
 
180
- /**
181
- * Re-adopts an already-live process after a guardian/version handoff.
182
- * @param {{pid: number, processStartTime: string}} identity - Exact process identity.
183
- * @param {string} [_configDigest] - Owner digest accepted by guardian-backed implementations.
184
- * @returns {Promise<void>}
185
- */
186
- async adopt({pid, processStartTime: expectedStartTime}, _configDigest) {
187
- if (!Number.isInteger(pid) || pid <= 0 || !/^\d+$/.test(expectedStartTime)) throw new Error(`Process ${this.id} adoption identity is invalid`)
188
- const actualStartTime = processStartTime(pid)
189
-
190
- if (actualStartTime !== expectedStartTime) throw new Error(`Process ${this.id} adoption rejected: PID ${pid} start time does not match the committed identity`)
191
- if (this.child) throw new Error(`Process ${this.id} adoption rejected while the process is already supervised`)
192
-
193
- this.pid = pid
194
- this.processStartTime = actualStartTime
195
- this.state = "running"
196
- this.intentionalStop = false
197
- this.intentionalStopSignal = undefined
198
- this.exitCode = undefined
199
- this.exitSignal = undefined
200
- this.lastStartReason = "deploy"
201
- }
202
-
203
195
  /**
204
196
  * Updates the command template used for future restarts without touching the currently running child.
205
197
  * @param {ManagedProcessDefinition} definition - Replacement process definition.
@@ -217,6 +209,7 @@ export default class ManagedProcess extends EventEmitter {
217
209
  this.restart = definition.restart
218
210
  this.restartDelayMs = definition.restartDelayMs
219
211
  this.shouldRestart = definition.shouldRestart
212
+ this.shouldStop = definition.shouldStop ?? (() => false)
220
213
  this.stopSignal = definition.stopSignal
221
214
  this.stopTimeoutMs = definition.stopTimeoutMs
222
215
  }
@@ -271,20 +264,84 @@ export default class ManagedProcess extends EventEmitter {
271
264
  this.exitSignal = signal ?? this.intentionalStopSignal
272
265
  this.child = undefined
273
266
  this.pid = undefined
274
- this.processStartTime = undefined
275
267
  this.exitPromise = undefined
276
268
  this.rssBytes = undefined
277
269
  this.children = []
278
270
  this.clearMemoryMonitor()
279
- this.state = wasIntentional ? "stopped" : "failed"
271
+ const terminalExit = !wasIntentional && this.shouldStop()
272
+ this.state = wasIntentional || terminalExit ? "stopped" : "failed"
273
+
280
274
  this.logger("process exited", {code, id: this.id, signal})
281
275
  this.emit("exit", {code, signal})
282
276
 
283
- if (!wasIntentional && this.shouldRestart()) {
277
+ if (!wasIntentional && !terminalExit && this.shouldRestart()) {
284
278
  this.scheduleRestart()
285
279
  }
286
280
  }
287
281
 
282
+ /**
283
+ * Records that the owned process group was still alive when the bounded stop gave up.
284
+ *
285
+ * Every process is spawned `detached: true`, so the tracked pid is the group leader and a
286
+ * member that survives its leader — a `sh` wrapper killed by the stop signal while the app
287
+ * it launched keeps running, or a runner child that outlives its parent — keeps holding the
288
+ * group id while the tracked ChildProcess record is already terminal. Nothing in rollbridge
289
+ * can wait for or signal that group again, yet the group still holds the release's work.
290
+ *
291
+ * Only the stop path can know this, and it already observed the group while waiting for it to
292
+ * leave, so the group id is recorded there instead of rediscovered in `onExit`: that hook
293
+ * fires for every retired process, and a procfs scan costs ~200 ms per exit on a busy host.
294
+ * @param {number} pgid - Process group id the tracked process led.
295
+ * @returns {void}
296
+ */
297
+ markGroupOutlivedStop(pgid) {
298
+ if (this.groupOutlivedStop === pgid) return
299
+
300
+ this.groupOutlivedStop = pgid
301
+ this.heldProcessGroupCheckedAtMs = undefined
302
+ this.heldProcessGroupCount = undefined
303
+ this.logger("process group outlived its tracked process", {id: this.id, pgid})
304
+ }
305
+
306
+ /**
307
+ * Reports the process group left behind by a bounded stop, as observed right now.
308
+ *
309
+ * The record is re-verified rather than trusted forever: once the orphaned members exit, the
310
+ * group is gone and stops holding the key. Re-verification is rate-limited to
311
+ * `HELD_GROUP_RECHECK_MS` because it costs a procfs scan, and when procfs cannot answer at
312
+ * all the last known state is reported instead of guessing that the group is gone.
313
+ * @returns {HeldProcessGroup | undefined} The held group, or undefined when nothing is held.
314
+ */
315
+ currentHeldProcessGroup() {
316
+ const pgid = this.groupOutlivedStop
317
+
318
+ if (pgid === undefined) return undefined
319
+ if (this.state !== "stopped" && this.state !== "failed") return undefined
320
+
321
+ const nowMs = Date.now()
322
+
323
+ if (this.heldProcessGroupCount !== undefined && this.heldProcessGroupCheckedAtMs !== undefined && nowMs - this.heldProcessGroupCheckedAtMs < HELD_GROUP_RECHECK_MS) {
324
+ return {liveMemberCount: this.heldProcessGroupCount, pgid}
325
+ }
326
+
327
+ const sample = sampleProcessGroup(pgid)
328
+
329
+ this.heldProcessGroupCheckedAtMs = nowMs
330
+
331
+ if (sample === undefined) return {liveMemberCount: this.heldProcessGroupCount, pgid}
332
+
333
+ if (sample.memberCount === 0) {
334
+ this.groupOutlivedStop = undefined
335
+ this.heldProcessGroupCount = undefined
336
+
337
+ return undefined
338
+ }
339
+
340
+ this.heldProcessGroupCount = sample.liveMemberCount
341
+
342
+ return {liveMemberCount: sample.liveMemberCount, pgid}
343
+ }
344
+
288
345
  /**
289
346
  * Schedules an automatic restart per the restart policy, or gives up once the policy's limit is reached.
290
347
  * @returns {void}
@@ -438,8 +495,8 @@ export default class ManagedProcess extends EventEmitter {
438
495
  }
439
496
 
440
497
  /**
441
- * @param {{timeoutMs?: number}} [options] - Stop options.
442
- * @returns {Promise<void>} Resolves when stopped.
498
+ * @param {{retirementTimeoutMs?: number, timeoutMs?: number}} [options] - Stop options.
499
+ * @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
443
500
  */
444
501
  async stop(options = {}) {
445
502
  this.operationRevision += 1
@@ -448,12 +505,13 @@ export default class ManagedProcess extends EventEmitter {
448
505
  }
449
506
 
450
507
  /**
451
- * @param {{timeoutMs?: number}} options - Stop options.
452
- * @returns {Promise<void>} Resolves when stopped.
508
+ * @param {{retirementTimeoutMs?: number, timeoutMs?: number}} options - Stop options.
509
+ * @returns {Promise<void>} Resolves when stopped or when retirement stops waiting.
453
510
  */
454
511
  async performStop(options) {
455
512
  const pgid = this.child?.pid ?? this.pid
456
513
  const exitPromise = this.exitPromise
514
+ const retirementDeadline = options.retirementTimeoutMs === undefined ? undefined : Date.now() + options.retirementTimeoutMs
457
515
  await this.quiesce()
458
516
 
459
517
  if (!pgid) {
@@ -475,23 +533,43 @@ export default class ManagedProcess extends EventEmitter {
475
533
  // 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
476
534
  if (this.processGroupExists(pgid)) {
477
535
  const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
478
- let gracefulDeadline
536
+ const gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
537
+ // A retirement budget is the caller's decision that this process must not be waited for and
538
+ // must not be killed, so it caps the graceful window rather than only standing in for a
539
+ // missing one. Read the other way round, a finite `gracefulStopMs` outvotes the retirement
540
+ // bound and the wait runs for the whole window: TensorBuzz production moved its worker from
541
+ // `"indefinite"` to 4,5 h and the stop would still have held the control plane for 4,5 h.
542
+ const retirementBound = retirementDeadline !== undefined && (gracefulDeadline === undefined || retirementDeadline < gracefulDeadline)
543
+ const waitDeadline = retirementBound ? retirementDeadline : gracefulDeadline
479
544
 
480
545
  if (stopCommand) {
481
546
  await this.runHook(stopCommand, hookTimeoutMs, "stop command", pgid)
482
- gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
483
547
  } else {
484
548
  this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
485
- gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
486
- await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline)
549
+ await this.signalProcessGroup(this.stopSignal, pgid, waitDeadline)
487
550
  }
488
551
 
489
- if (!(await this.waitForProcessGroupExit(pgid, gracefulDeadline))) {
552
+ const exited = await this.waitForProcessGroupExit(pgid, waitDeadline)
553
+
554
+ if (!exited && retirementBound) {
555
+ this.retirementTimedOut = true
556
+ this.markGroupOutlivedStop(pgid)
557
+ // The tracked process has completed its supervised stop even though its process group is
558
+ // still draining. Keep the residue in heldProcessGroup rather than leaving the process
559
+ // permanently quiesced, which would make a drained generation look active to restore
560
+ // admission and keep the owner state from describing the actual stopped generation.
561
+ this.state = "stopped"
562
+ this.logger("process retirement wait timed out; leaving process alive", {id: this.id, pid: pgid, timeoutMs: options.retirementTimeoutMs})
563
+ return
564
+ }
565
+
566
+ if (!exited) {
490
567
  this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: pgid})
491
568
  const killDeadline = Date.now() + 5000
492
569
 
493
570
  await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
494
- await this.waitForProcessGroupExit(pgid, killDeadline)
571
+
572
+ if (!await this.waitForProcessGroupExit(pgid, killDeadline)) this.markGroupOutlivedStop(pgid)
495
573
  }
496
574
  }
497
575
 
@@ -500,6 +578,49 @@ export default class ManagedProcess extends EventEmitter {
500
578
  this.state = "stopped"
501
579
  }
502
580
 
581
+ /**
582
+ * Delivers the SIGKILL that retirement proved it must not wait for.
583
+ *
584
+ * Retirement deliberately leaves a non-blocking process alive: STOE-1077 measured what a
585
+ * stop-timeout kill costs when the process still holds live lanes (67 builds, 27.3
586
+ * build-hours). That is right while the process can still drain and wrong once it cannot — a
587
+ * wedged generation keeps its release open, keeps its port reservation, and makes every
588
+ * later deploy wait on it until an operator signals the exact PIDs by hand. This is the
589
+ * acting half of that decision, and it re-checks the two invariants this class owns before
590
+ * sending anything: the process must already be recorded as past its retirement budget, and
591
+ * its group must still have a member that can be signalled.
592
+ * @param {{reason: string}} options - Named evidence behind the escalation.
593
+ * @returns {Promise<boolean>} True when a live process group was signalled.
594
+ */
595
+ async escalateRetirement({reason}) {
596
+ const pgid = this.pid ?? this.groupOutlivedStop
597
+
598
+ if (pgid === undefined) return false
599
+
600
+ if (!this.retirementTimedOut) {
601
+ this.logger("retirement escalation refused; process is not past its retirement budget", {id: this.id, pid: pgid})
602
+
603
+ return false
604
+ }
605
+
606
+ if (!this.processGroupExists(pgid)) {
607
+ this.groupOutlivedStop = undefined
608
+ this.heldProcessGroupCount = undefined
609
+
610
+ return false
611
+ }
612
+
613
+ this.logger("process retirement escalated to SIGKILL", {id: this.id, pid: pgid, reason})
614
+
615
+ const killDeadline = Date.now() + ESCALATION_KILL_CONFIRM_MS
616
+
617
+ await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
618
+
619
+ if (!await this.waitForProcessGroupExit(pgid, killDeadline)) this.markGroupOutlivedStop(pgid)
620
+
621
+ return true
622
+ }
623
+
503
624
  /** @returns {Promise<void>} Stops restarts and waits until the process has stopped accepting new work. */
504
625
  async quiesce() {
505
626
  if (this.quiescePromise) return await this.quiescePromise
@@ -511,7 +632,7 @@ export default class ManagedProcess extends EventEmitter {
511
632
  this.restartTimer = undefined
512
633
  }
513
634
  if (this.lifecycleRestoreBarrier) await this.lifecycleRestoreBarrier
514
- if (!this.child?.pid && !this.pid) {
635
+ if (!this.child?.pid) {
515
636
  this.state = "stopped"
516
637
  if (this.hasRestorableLifecycle()) this.lifecycleRole = "retired"
517
638
  return
@@ -642,6 +763,19 @@ export default class ManagedProcess extends EventEmitter {
642
763
  return this.stopTimeoutMs
643
764
  }
644
765
 
766
+ /**
767
+ * Reports whether this process's graceful window has no upper bound.
768
+ *
769
+ * `"indefinite"` is the consumer's statement that the supervisor must never kill this process;
770
+ * it is not a statement that a caller may wait for it forever. A stop that has no retirement
771
+ * budget for such a process can only hang, and when that stop runs inside an owner mutation it
772
+ * locks out every later mutation (STOE-2015).
773
+ * @returns {boolean} True when no deadline can ever end the wait.
774
+ */
775
+ hasUnboundedGracefulWindow() {
776
+ return this.stopTimeoutMs === "indefinite"
777
+ }
778
+
645
779
  /**
646
780
  * Runs a lifecycle hook command, bounded by a timeout so a hung hook can never block stop().
647
781
  * Failures are logged and returned so generation retirement can surface a failed quiet hook;
@@ -844,6 +978,7 @@ export default class ManagedProcess extends EventEmitter {
844
978
  cwd: this.cwd,
845
979
  exitCode: this.exitCode,
846
980
  exitSignal: this.exitSignal,
981
+ heldProcessGroup: this.currentHeldProcessGroup(),
847
982
  id: this.id,
848
983
  lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
849
984
  lastStartReason: this.lastStartReason,
@@ -851,8 +986,8 @@ export default class ManagedProcess extends EventEmitter {
851
986
  logs: this.logs.slice(-this.outputLines),
852
987
  memoryRestarts: this.memoryRestarts,
853
988
  pid: this.pid,
854
- processStartTime: this.processStartTime,
855
989
  restarts: this.restarts,
990
+ retirementTimedOut: this.retirementTimedOut,
856
991
  rssBytes: this.rssBytes,
857
992
  startedAt: this.startedAtMs === undefined ? undefined : new Date(this.startedAtMs).toISOString(),
858
993
  state: this.state,
@@ -860,3 +995,21 @@ export default class ManagedProcess extends EventEmitter {
860
995
  }
861
996
  }
862
997
  }
998
+
999
+ /**
1000
+ * The guardian's provenance fence, in one place: a registered child stops fencing a later
1001
+ * claim on its key only when its record is terminal *and* it holds no pid, and a process
1002
+ * group that outlived the tracked process is not gone either.
1003
+ *
1004
+ * Both the guardian (re-registering a key under a new provenance) and the daemon (restoring a
1005
+ * release that is recorded as drained) must agree on this, or a restore reaches the guardian
1006
+ * and fails there with a bare `Guardian provenance mismatch` naming a process the operator
1007
+ * cannot see in `status`.
1008
+ * @param {Pick<ManagedProcessStatus, "heldProcessGroup" | "pid" | "state">} status - Process status record.
1009
+ * @returns {boolean} True when the child is conclusively gone.
1010
+ */
1011
+ export function childRecordIsGone(status) {
1012
+ if (status.heldProcessGroup) return false
1013
+
1014
+ return (status.state === "stopped" || status.state === "failed") && status.pid === undefined
1015
+ }