rollbridge 0.1.21 → 0.1.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rollbridge",
-  "version": "0.1.21",
+  "version": "0.1.23",
   "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
   "keywords": [
     "deploy",

package/src/cli.js

@@ -65,9 +65,19 @@ export async function runCli(argv) {
           } else {
             await daemon.exposeControl()
           }
-        } catch {
-          daemon.logger("bootstrap activation failed", {releaseId: bootstrap.releaseId, status: "error"})
-          if (!options.takeoverOwner) await daemon.shutdown()
+        } catch (error) {
+          const failure = error instanceof Error ? error : String(error)
+
+          daemon.logger("bootstrap activation failed", {releaseId: bootstrap.releaseId, status: "error", ...errorLogData(failure)})
+
+          try {
+            await daemon.shutdown()
+          } catch (shutdownError) {
+            const shutdownFailure = shutdownError instanceof Error ? shutdownError : String(shutdownError)
+
+            daemon.logger("bootstrap shutdown failed", {releaseId: bootstrap.releaseId, status: "error", ...errorLogData(shutdownFailure)})
+          }
+
           process.exitCode = 1
           return
         }
@@ -960,3 +970,18 @@ function isMissingDaemonError(error) {
 
   return error.code === "ENOENT" || error.code === "ECONNREFUSED"
 }
+
+/**
+ * Converts any thrown value into JSON-safe diagnostics without losing an Error stack.
+ * @param {Error | string} error - Thrown value.
+ * @returns {{error: string, stack: string}} Safe structured log fields.
+ */
+function errorLogData(error) {
+  if (error instanceof Error) {
+    return {error: error.message, stack: error.stack || `${error.name}: ${error.message}`}
+  }
+
+  const message = String(error)
+
+  return {error: message, stack: message}
+}

package/src/daemon.js

@@ -56,6 +56,7 @@ export default class RollbridgeDaemon {
     this.proxy = httpProxy.createProxyServer({ws: true, xfwd: true})
     this.proxyServer = /** @type {http.Server | undefined} */ (undefined)
     this.controlServer = /** @type {net.Server | undefined} */ (undefined)
+    this.controlSocketOwned = false
     this.controlSockets = /** @type {Set<net.Socket>} */ (new Set())
     this.proxyPort = /** @type {number | undefined} */ (undefined)
     this.stopping = false
@@ -63,6 +64,7 @@ export default class RollbridgeDaemon {
     this.persistTimer = /** @type {ReturnType<typeof setInterval> | undefined} */ (undefined)
     this.persistenceEnabled = false
     this.pendingWrite = /** @type {Promise<void> | undefined} */ (undefined)
+    this.stateCleanupEnabled = false
     this.shutdownPromise = /** @type {Promise<void> | undefined} */ (undefined)
     this.retirementPromise = /** @type {Promise<void> | undefined} */ (undefined)
     this.controlClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
@@ -128,6 +130,7 @@ export default class RollbridgeDaemon {
     await new Promise((resolve, reject) => {
       server.once("error", reject)
       server.listen(this.config.control.path, () => {
+        this.controlSocketOwned = true
         this.logger("control socket listening", {path: this.config.control.path})
         resolve(undefined)
       })
@@ -741,6 +744,7 @@ export default class RollbridgeDaemon {
   startStatePersistence() {
     if (!this.statePath) return
 
+    this.stateCleanupEnabled = true
     this.persistenceEnabled = true
     this.persistState()
     this.persistTimer = setInterval(() => this.persistState(), STATE_PERSIST_INTERVAL_MS)
@@ -780,6 +784,7 @@ export default class RollbridgeDaemon {
   async reportOrphans() {
     if (!this.statePath) return
 
+    this.stateCleanupEnabled = true
     const orphans = liveProcesses(await readState(this.statePath))
 
     // Keep them for status() so `rollbridge status` reflects still-running children after a
@@ -832,6 +837,7 @@ export default class RollbridgeDaemon {
     }
     this.persistenceEnabled = false
     if (this.pendingWrite) await this.pendingWrite
+    this.stateCleanupEnabled = false
     this.controlClosePromise = this.closeServer(this.controlServer)
     for (const socket of this.controlSockets) if (socket !== completionSocket) socket.destroy()
     await Promise.all([
@@ -888,7 +894,7 @@ export default class RollbridgeDaemon {
     // new writes start: stopping is set and the persist timer is cleared above). Prior-daemon
     // orphans are not owned by this daemon, so retain their records until they are confirmed gone.
     await captureShutdownError(cleanupErrors, "persistent state cleanup", async () => {
-      if (!this.statePath) return
+      if (!this.statePath || !this.stateCleanupEnabled) return
       if (this.pendingWrite) await this.pendingWrite
       const orphans = this.orphans.filter((orphan) => isProcessAlive(orphan.pid))
 
@@ -910,7 +916,10 @@ export default class RollbridgeDaemon {
 
   /** @returns {Promise<void>} Removes the configured control socket path. */
   async removeControlSocket() {
+    if (!this.controlSocketOwned) return
+
     await fs.rm(this.config.control.path, {force: true})
+    this.controlSocketOwned = false
   }
 
   /**

package/src/managed-process.js

@@ -62,6 +62,7 @@ export default class ManagedProcess extends EventEmitter {
     this.memoryWarned = false
     this.startedAtMs = /** @type {number | undefined} */ (undefined)
     this.intentionalStop = false
+    this.intentionalStopSignal = /** @type {ProcessExitSignal | undefined} */ (undefined)
     this.quiescePromise = /** @type {Promise<void> | undefined} */ (undefined)
     this.restartTimer = undefined
     this.child = undefined
@@ -79,6 +80,7 @@ export default class ManagedProcess extends EventEmitter {
     if (this.child) return
 
     this.intentionalStop = false
+    this.intentionalStopSignal = undefined
     this.quiescePromise = undefined
     this.exitCode = undefined
     this.exitSignal = undefined
@@ -167,8 +169,8 @@ export default class ManagedProcess extends EventEmitter {
   onExit(code, signal) {
     const wasIntentional = this.intentionalStop
 
-    this.exitCode = code
-    this.exitSignal = signal
+    this.exitCode = this.intentionalStopSignal ? null : code
+    this.exitSignal = signal ?? this.intentionalStopSignal
     this.child = undefined
     this.pid = undefined
     this.exitPromise = undefined
@@ -362,15 +364,24 @@ export default class ManagedProcess extends EventEmitter {
 
     // 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
     if (this.processGroupExists(pgid)) {
-      if (stopCommand) await this.runHook(stopCommand, hookTimeoutMs, "stop command", pgid)
-      else this.killProcessGroup(this.stopSignal, pgid)
-
       const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
+      let gracefulDeadline
+
+      if (stopCommand) {
+        await this.runHook(stopCommand, hookTimeoutMs, "stop command", pgid)
+        gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
+      } else {
+        this.intentionalStopSignal = /** @type {ProcessExitSignal} */ (this.stopSignal)
+        gracefulDeadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
+        await this.signalProcessGroup(this.stopSignal, pgid, gracefulDeadline)
+      }
 
-      if (!(await this.waitForProcessGroupExit(pgid, timeoutMs))) {
+      if (!(await this.waitForProcessGroupExit(pgid, gracefulDeadline))) {
         this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: pgid})
-        this.killProcessGroup("SIGKILL", pgid)
-        await this.waitForProcessGroupExit(pgid, 5000)
+        const killDeadline = Date.now() + 5000
+
+        await this.signalProcessGroup("SIGKILL", pgid, killDeadline)
+        await this.waitForProcessGroupExit(pgid, killDeadline)
       }
     }
 
@@ -489,6 +500,49 @@ export default class ManagedProcess extends EventEmitter {
     }
   }
 
+  /**
+   * Signals descendants before their shell leader so the leader can reap them before it exits.
+   * Falls back to the portable group signal when procfs cannot identify group members.
+   * @param {string} signal - Signal name.
+   * @param {number} pgid - Process group id.
+   * @param {number | undefined} deadline - Absolute graceful deadline, or undefined to wait indefinitely.
+   * @returns {Promise<void>} Resolves after the leader has been signalled.
+   */
+  async signalProcessGroup(signal, pgid, deadline) {
+    const members = processGroupMembers(pgid)
+    const descendants = members.filter((member) => member.pid !== pgid)
+
+    if (descendants.length === 0) {
+      this.killProcessGroup(signal, pgid)
+      return
+    }
+
+    for (const descendant of descendants) this.killProcess(descendant.pid, signal)
+    while (processGroupMembers(pgid).some((member) => member.pid !== pgid)) {
+      if (deadline !== undefined && Date.now() >= deadline) break
+      const waitMs = deadline === undefined ? 25 : Math.min(25, deadline - Date.now())
+
+      await new Promise((resolve) => setTimeout(resolve, waitMs))
+    }
+
+    this.killProcess(pgid, signal)
+  }
+
+  /**
+   * Signals one verified member of the owned process group.
+   * @param {number} pid - Process id.
+   * @param {string} signal - Signal name.
+   * @returns {void}
+   */
+  killProcess(pid, signal) {
+    try {
+      process.kill(pid, signal)
+    } catch (error) {
+      if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") return
+      throw error
+    }
+  }
+
   /**
    * @param {number} pgid - Process group id.
    * @returns {boolean} True until the process group no longer exists.
@@ -510,15 +564,15 @@ export default class ManagedProcess extends EventEmitter {
 
   /**
    * @param {number} pgid - Process group id.
-   * @param {StopTimeoutMs} timeoutMs - Timeout.
+   * @param {number | undefined} deadline - Absolute deadline, or undefined to wait indefinitely.
    * @returns {Promise<boolean>} True once the process group no longer exists.
    */
-  async waitForProcessGroupExit(pgid, timeoutMs) {
-    const deadline = timeoutMs === "indefinite" ? undefined : Date.now() + timeoutMs
-
+  async waitForProcessGroupExit(pgid, deadline) {
     while (this.processGroupExists(pgid)) {
       if (deadline !== undefined && Date.now() >= deadline) return false
-      await new Promise((resolve) => setTimeout(resolve, 10))
+      const waitMs = deadline === undefined ? 10 : Math.min(10, deadline - Date.now())
+
+      await new Promise((resolve) => setTimeout(resolve, waitMs))
     }
 
     return true

package/test/daemon-bootstrap.test.js

@@ -4,6 +4,7 @@ import assert from "node:assert/strict"
 import {spawn} from "node:child_process"
 import {once} from "node:events"
 import fs from "node:fs/promises"
+import net from "node:net"
 import os from "node:os"
 import path from "node:path"
 import test from "node:test"
@@ -14,9 +15,12 @@ import {isProcessAlive, liveProcesses, readState, writeState} from "../src/state
 const currentDir = path.dirname(fileURLToPath(import.meta.url))
 const binPath = path.join(currentDir, "..", "bin", "rollbridge")
 const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
+const ownedChildPath = path.join(currentDir, "fixtures", "owned-child.js")
 const firstAttestation = `sha256:${"a".repeat(64)}`
 const secondAttestation = `sha256:${"b".repeat(64)}`
 
+/** @typedef {{data?: Record<string, import("../src/json.js").JsonValue>, message?: string}} StructuredRecord */
+
 test("daemon bootstrap requires complete, safe, absolute inputs before binding listeners", async (t) => {
   const cases = [
     {args: ["--config", "CONFIG", "--release-path", "RELEASE", "--release-id", "v1"], message: /must be provided together/},
@@ -86,11 +90,12 @@ test("daemon bootstrap activates the exact release through the foreground daemon
 })
 
 test("failed takeover bootstrap preserves the previously accepted owner", async () => {
-  const fixture = await createFixture()
+  const fixture = await createFixture({persistState: true})
   const accepted = spawnDaemon(fixture, {attestation: firstAttestation, releaseId: "accepted", revision: "accepted123"})
 
   try {
     await waitForLog(accepted, "control socket listening")
+    await waitForFile(fixture.statePath)
     const badConfig = JSON.parse((await fs.readFile(fixture.configPath, "utf8")).replace(/^module\.exports = /, ""))
     badConfig.processes[0].health.path = "/never-ready"
     badConfig.processes[0].health.timeoutMs = 100
@@ -106,10 +111,20 @@ test("failed takeover bootstrap preserves the previously accepted owner", async
     ])
 
     assert.notEqual(result.code, 0)
+    const records = parseStructuredOutput(result.output)
+    const failure = records.find((record) => record.message === "bootstrap activation failed")
+    const candidatePid = Number(await fs.readFile(fixture.startedPath, "utf8"))
+
+    assert.match(String(failure?.data?.error), /Health check failed/)
+    assert.match(String(failure?.data?.stack), /Error: Health check failed/)
+    assert.equal(isProcessAlive(candidatePid), false, "the failed candidate process must be stopped")
     const status = await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})
     assert.equal(status.activeReleaseId, "accepted")
     assert.ok(status.bootstrap && typeof status.bootstrap === "object" && !Array.isArray(status.bootstrap))
     assert.equal(status.bootstrap.attestation, firstAttestation)
+    const priorState = await readState(fixture.statePath)
+
+    assert.ok(priorState && typeof priorState === "object" && !Array.isArray(priorState) && priorState.activeReleaseId === "accepted", "candidate cleanup must preserve the prior owner's state")
   } finally {
     accepted.kill("SIGTERM")
     if (accepted.exitCode === null) await once(accepted, "exit")
@@ -267,24 +282,56 @@ test("SIGTERM during multi-process bootstrap owns every process started after sh
   }
 })
 
-test("failed daemon bootstrap reports a structured failure, cleans its processes, and exits non-zero", async () => {
-  const fixture = await createFixture({healthPath: "/never-ready", healthTimeoutMs: 100})
+test("failed ordinary bootstrap completely shuts down attempt-owned resources and exposes the cause", async () => {
+  const fixture = await createFixture({attemptOwnedProcesses: true, fixedPorts: true, missingControlParent: true})
 
   try {
     const result = await runDaemon([
       "--config", fixture.configPath,
       "--release-path", fixture.root,
-      "--release-id", "bad-release",
-      "--revision", "bad123"
+      "--release-id", "ordinary-failure",
+      "--revision", "ordinary123"
     ])
-    const records = result.output.split("\n").filter(Boolean).map((line) => JSON.parse(line))
+    const records = parseStructuredOutput(result.output)
+    const failure = records.find((record) => record.message === "bootstrap activation failed")
+
+    assert.notEqual(result.code, 0)
+    assert.equal(failure?.data?.releaseId, "ordinary-failure")
+    assert.equal(failure?.data?.status, "error")
+    assert.match(String(failure?.data?.error), /listen (?:EACCES|ENOENT)/)
+    assert.match(String(failure?.data?.stack), /Error: listen (?:EACCES|ENOENT)/)
+    await assertAttemptResourcesStopped(fixture, records)
+    await assert.rejects(() => fs.stat(fixture.socketPath), {code: "ENOENT"})
+  } finally {
+    await killAttemptProcesses(fixture.lifecyclePath)
+    await fs.rm(fixture.root, {force: true, recursive: true})
+  }
+})
+
+test("failed takeover retirement completely shuts down and exits non-zero instead of lingering", async () => {
+  const fixture = await createFixture({attemptOwnedProcesses: true, fixedPorts: true})
+
+  try {
+    const result = await runDaemon([
+      "--config", fixture.configPath,
+      "--release-path", fixture.root,
+      "--release-id", "orphaned-candidate",
+      "--revision", "candidate123",
+      "--boot-attestation", secondAttestation,
+      "--takeover-owner"
+    ], {timeoutMs: 2000})
+    const records = parseStructuredOutput(result.output)
     const failure = records.find((record) => record.message === "bootstrap activation failed")
 
     assert.notEqual(result.code, 0)
-    assert.deepEqual(failure?.data, {releaseId: "bad-release", status: "error"})
-    assert.ok(records.some((record) => record.message === "release startup process status" && record.data?.phase === "after cleanup" && record.data?.state === "stopped"))
+    assert.equal(failure?.data?.releaseId, "orphaned-candidate")
+    assert.equal(failure?.data?.status, "error")
+    assert.match(String(failure?.data?.error), /connect ENOENT/)
+    assert.match(String(failure?.data?.stack), /Error: connect ENOENT/)
+    await assertAttemptResourcesStopped(fixture, records)
     await assert.rejects(() => fs.stat(fixture.socketPath), {code: "ENOENT"})
   } finally {
+    await killAttemptProcesses(fixture.lifecyclePath)
     await fs.rm(fixture.root, {force: true, recursive: true})
   }
 })
@@ -350,12 +397,12 @@ test("failed daemon bootstrap preserves prior live process records in statePath"
 })
 
 /**
- * @param {{healthGate?: boolean, healthPath?: string, healthTimeoutMs?: number, multiProcessSignal?: boolean, persistState?: boolean}} [options] - Fixture options.
- * @returns {Promise<{configPath: string, gatePath: string, healthGatePath: string, lifecyclePath: string, root: string, socketPath: string, startedPath: string, statePath: string, stoppedPath: string}>} Fixture paths.
+ * @param {{attemptOwnedProcesses?: boolean, fixedPorts?: boolean, healthGate?: boolean, healthPath?: string, healthTimeoutMs?: number, missingControlParent?: boolean, multiProcessSignal?: boolean, persistState?: boolean}} [options] - Fixture options.
+ * @returns {Promise<{configPath: string, gatePath: string, healthGatePath: string, lifecyclePath: string, processPort: number, proxyPort: number, root: string, socketPath: string, startedPath: string, statePath: string, stoppedPath: string}>} Fixture paths.
  */
-async function createFixture({healthGate = false, healthPath = "/ping", healthTimeoutMs = 1000, multiProcessSignal = false, persistState = false} = {}) {
+async function createFixture({attemptOwnedProcesses = false, fixedPorts = false, healthGate = false, healthPath = "/ping", healthTimeoutMs = 1000, missingControlParent = false, multiProcessSignal = false, persistState = false} = {}) {
   const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-bootstrap-"))
-  const socketPath = path.join(root, "control.sock")
+  const socketPath = missingControlParent ? path.join(root, "missing", "control.sock") : path.join(root, "control.sock")
   const statePath = path.join(root, "state.json")
   const startedPath = path.join(root, "started.pid")
   const stoppedPath = path.join(root, "stopped.pid")
@@ -364,8 +411,12 @@ async function createFixture({healthGate = false, healthPath = "/ping", healthTi
   const healthGatePath = path.join(root, "health-ready")
   const configPath = path.join(root, "rollbridge.js")
   const command = `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`
+  const ownedCommand = `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(ownedChildPath)}`
+  const ownedWebCommand = `exec ${command}`
+  const [processPort, proxyPort] = fixedPorts ? await availablePorts(2) : [0, 0]
   const lifecycleEnv = {ROLLBRIDGE_TEST_LIFECYCLE_PATH: lifecyclePath}
   const webEnv = {
+    ...(attemptOwnedProcesses ? lifecycleEnv : {}),
     ROLLBRIDGE_TEST_STARTED_PATH: startedPath,
     ROLLBRIDGE_TEST_STOPPED_PATH: stoppedPath,
     ...(healthGate ? {ROLLBRIDGE_TEST_HEALTH_GATE_PATH: healthGatePath} : {})
@@ -377,8 +428,13 @@ async function createFixture({healthGate = false, healthPath = "/ping", healthTi
       {command: `trap '' TERM; printf '%s\\n' '{"event":"shutdown"}' >> ${JSON.stringify(lifecyclePath)}; kill -TERM "$ROLLBRIDGE_TEST_DAEMON_PID"; printf '{"event":"started","pid":%s,"processId":"database","replicaIndex":"0"}\\n' "$$" >> ${JSON.stringify(lifecyclePath)}; read ignored < ${JSON.stringify(gatePath)}`, env: lifecycleEnv, id: "database", policy: "service"},
       {command: `exec ${command}`, env: lifecycleEnv, id: "worker", policy: "companion", replicas: 2},
       {command: `exec ${command}`, env: lifecycleEnv, health: {path: healthPath, timeoutMs: healthTimeoutMs}, id: "web", policy: "proxied", port: {from: 0, to: 0}}
+    ] : attemptOwnedProcesses ? [
+      {command: ownedCommand, env: lifecycleEnv, id: "database", policy: "service"},
+      {command: ownedCommand, env: lifecycleEnv, id: "worker", policy: "companion"},
+      {command: ownedCommand, env: lifecycleEnv, id: "scheduler", policy: "singleton"},
+      {command: ownedWebCommand, env: webEnv, health: {path: healthPath, timeoutMs: healthTimeoutMs}, id: "web", policy: "proxied", port: {from: processPort, to: processPort}}
     ] : [{command, env: webEnv, health: {path: healthPath, timeoutMs: healthTimeoutMs}, id: "web", policy: "proxied", port: {from: 0, to: 0}}],
-    proxy: {host: "127.0.0.1", port: 0},
+    proxy: {host: "127.0.0.1", port: proxyPort},
     ...(persistState ? {statePath} : {})
   }
 
@@ -392,7 +448,7 @@ async function createFixture({healthGate = false, healthPath = "/ping", healthTi
   }
 
   await fs.writeFile(configPath, `${setup}module.exports = ${JSON.stringify(config, null, 2)}\n`)
-  return {configPath, gatePath, healthGatePath, lifecyclePath, root, socketPath, startedPath, statePath, stoppedPath}
+  return {configPath, gatePath, healthGatePath, lifecyclePath, processPort, proxyPort, root, socketPath, startedPath, statePath, stoppedPath}
 }
 
 /**
@@ -427,6 +483,12 @@ async function waitForFile(filePath) {
   const watcher = fs.watch(path.dirname(filePath))
 
   try {
+    try {
+      return await fs.readFile(filePath, "utf8")
+    } catch (error) {
+      if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
+    }
+
     for await (const event of watcher) {
       if (event.filename === path.basename(filePath)) return await fs.readFile(filePath, "utf8")
     }
@@ -452,26 +514,143 @@ function spawnDaemon(fixture, release) {
 
 /**
  * @param {string[]} args - Daemon arguments.
+ * @param {{timeoutMs?: number}} [options] - Process execution options.
  * @returns {Promise<{code: number | null, output: string, stderr: string}>} Completed process result.
  */
-async function runDaemon(args) {
-  return await runRollbridge(["daemon", ...args])
+async function runDaemon(args, options) {
+  return await runRollbridge(["daemon", ...args], options)
 }
 
 /**
  * @param {string[]} args - Rollbridge command and arguments.
+ * @param {{timeoutMs?: number}} [options] - Process execution options.
  * @returns {Promise<{code: number | null, output: string, stderr: string}>} Completed process result.
  */
-async function runRollbridge(args) {
+async function runRollbridge(args, {timeoutMs} = {}) {
   const child = spawn(process.execPath, [binPath, ...args], {stdio: ["ignore", "pipe", "pipe"]})
   let output = ""
   let stderr = ""
 
   child.stdout.setEncoding("utf8").on("data", (chunk) => { output += chunk })
   child.stderr.setEncoding("utf8").on("data", (chunk) => { stderr += chunk })
-  const [code] = await once(child, "exit")
+  let timer
+
+  const exited = once(child, "exit")
+  const result = timeoutMs === undefined ? await exited : await Promise.race([
+    exited,
+    new Promise((resolve) => {
+      timer = setTimeout(() => {
+        child.kill("SIGKILL")
+        resolve(["timeout"])
+      }, timeoutMs)
+    })
+  ])
+
+  if (timer) clearTimeout(timer)
+  if (result[0] === "timeout") {
+    await exited
+    throw new Error(`Rollbridge did not terminate within ${timeoutMs}ms`)
+  }
+
+  const [code] = result
+
+  return {code: typeof code === "number" || code === null ? code : null, output, stderr}
+}
+
+/**
+ * @param {string} output - JSON-lines daemon output.
+ * @returns {StructuredRecord[]} Parsed records.
+ */
+function parseStructuredOutput(output) {
+  return output.split("\n").filter(Boolean).map((line) => JSON.parse(line))
+}
+
+/**
+ * @param {{lifecyclePath: string, processPort: number, proxyPort: number}} fixture - Attempt fixture.
+ * @param {StructuredRecord[]} records - Structured daemon records.
+ * @returns {Promise<void>} Resolves after all owned resources are verified stopped.
+ */
+async function assertAttemptResourcesStopped(fixture, records) {
+  const events = await readLifecycleEvents(fixture.lifecyclePath)
+  const started = events.filter((event) => event.event === "started")
+  const stoppedPids = new Set(events.filter((event) => event.event === "stopped").map((event) => event.pid))
+  const expectedProcessIds = new Set(["database", "scheduler", "web", "worker"])
+  const managedStarts = new Set(records.filter((record) => record.message === "process started").map((record) => record.data?.processId))
+  const managedExits = new Set(records.filter((record) => record.message === "process exited").map((record) => record.data?.processId))
+
+  assert.deepEqual(managedStarts, expectedProcessIds)
+  assert.deepEqual(managedExits, expectedProcessIds)
+  for (const event of started) {
+    assert.equal(stoppedPids.has(event.pid), true, `${event.processId} must receive graceful shutdown`)
+    assert.equal(isProcessAlive(Number(event.pid)), false, `${event.processId} pid ${event.pid} must be gone before daemon exit`)
+  }
+
+  await assertPortAvailable(fixture.processPort)
+  await assertPortAvailable(fixture.proxyPort)
+}
+
+/**
+ * @param {string} lifecyclePath - Fixture lifecycle log.
+ * @returns {Promise<Record<string, import("../src/json.js").JsonValue>[]>} Parsed lifecycle events.
+ */
+async function readLifecycleEvents(lifecyclePath) {
+  return (await fs.readFile(lifecyclePath, "utf8")).trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))
+}
+
+/**
+ * Best-effort cleanup for a deliberately failing lingering-process regression.
+ * @param {string} lifecyclePath - Fixture lifecycle log.
+ * @returns {Promise<void>} Resolves after known fixture process groups are stopped.
+ */
+async function killAttemptProcesses(lifecyclePath) {
+  try {
+    const events = await readLifecycleEvents(lifecyclePath)
+
+    for (const event of events) {
+      const pid = Number(event.pid)
+
+      if (event.event === "started" && isProcessAlive(pid)) process.kill(-pid, "SIGKILL")
+    }
+  } catch {
+    // The attempt can fail before creating the lifecycle file.
+  }
+}
+
+/**
+ * @param {number} port - Local port expected to be free.
+ * @returns {Promise<void>} Resolves after binding and releasing the port.
+ */
+async function assertPortAvailable(port) {
+  const server = net.createServer()
+
+  await new Promise((resolve, reject) => {
+    server.once("error", reject)
+    server.listen(port, "127.0.0.1", () => resolve(undefined))
+  })
+  await new Promise((resolve) => server.close(() => resolve(undefined)))
+}
+
+/**
+ * Reserves distinct ephemeral ports until all have been observed, then releases them.
+ * @param {number} count - Number of ports.
+ * @returns {Promise<number[]>} Available port numbers.
+ */
+async function availablePorts(count) {
+  const servers = Array.from({length: count}, () => net.createServer())
+
+  await Promise.all(servers.map((server) => new Promise((resolve, reject) => {
+    server.once("error", reject)
+    server.listen(0, "127.0.0.1", () => resolve(undefined))
+  })))
+  const ports = servers.map((server) => {
+    const address = server.address()
+
+    if (!address || typeof address === "string") throw new Error("Expected a TCP server address")
+    return address.port
+  })
 
-  return {code, output, stderr}
+  await Promise.all(servers.map((server) => new Promise((resolve) => server.close(() => resolve(undefined)))))
+  return ports
 }
 
 /**

package/test/fixtures/owned-child.js

@@ -0,0 +1,19 @@
+// @ts-check
+
+import fs from "node:fs"
+
+const lifecyclePath = process.env.ROLLBRIDGE_TEST_LIFECYCLE_PATH
+
+if (!lifecyclePath) throw new Error("ROLLBRIDGE_TEST_LIFECYCLE_PATH is required")
+
+/** @param {string} event - Lifecycle event. */
+const record = (event) => {
+  fs.appendFileSync(lifecyclePath, `${JSON.stringify({event, pid: process.pid, processId: process.env.ROLLBRIDGE_PROCESS_ID})}\n`)
+}
+
+record("started")
+process.on("SIGTERM", () => {
+  record("stopped")
+  process.exit(0)
+})
+setInterval(() => {}, 1000)

package/test/managed-process.test.js

@@ -386,8 +386,10 @@ test("a hanging lifecycle hook is bounded so stop still completes", async () =>
 })
 
 test("sends the configured stopSignal as the graceful stop signal", async () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-stop-signal-"))
+  const readyPath = path.join(dir, "ready")
   const managed = new ManagedProcess({
-    command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
+    command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => {}, 1000)`)}`,
     cwd: undefined,
     env: {},
     id: "worker",
@@ -401,19 +403,25 @@ test("sends the configured stopSignal as the graceful stop signal", async () =>
 
   /** @type {string[]} */
   const signals = []
-  const killProcessGroup = managed.killProcessGroup.bind(managed)
+  const killProcess = managed.killProcess.bind(managed)
 
-  managed.killProcessGroup = (signal) => {
+  managed.killProcess = (pid, signal) => {
     signals.push(signal)
-    killProcessGroup(signal)
+    killProcess(pid, signal)
   }
 
-  await managed.start()
-  await managed.stop()
+  try {
+    await managed.start()
+    await waitFor(() => fs.existsSync(readyPath))
+    await managed.stop()
 
-  // The graceful stop uses the configured signal (a SIGKILL fallback, if any, comes after).
-  assert.equal(signals[0], "SIGINT")
-  assert.equal(managed.status().state, "stopped")
+    // The graceful stop reaches the ready descendant and its shell leader without SIGKILL.
+    assert.deepEqual(signals, ["SIGINT", "SIGINT"])
+    assert.equal(managed.status().state, "stopped")
+  } finally {
+    await managed.stop()
+    fs.rmSync(dir, {force: true, recursive: true})
+  }
 })
 
 test("indefinite stop waits for the process to exit without SIGKILL", async () => {
@@ -431,11 +439,11 @@ test("indefinite stop waits for the process to exit without SIGKILL", async () =
   })
   /** @type {string[]} */
   const signals = []
-  const killProcessGroup = managed.killProcessGroup.bind(managed)
+  const killProcess = managed.killProcess.bind(managed)
 
-  managed.killProcessGroup = (signal) => {
+  managed.killProcess = (pid, signal) => {
     signals.push(signal)
-    killProcessGroup(signal)
+    killProcess(pid, signal)
   }
 
   try {
@@ -443,7 +451,7 @@ test("indefinite stop waits for the process to exit without SIGKILL", async () =
     await managed.stop()
 
     assert.equal(managed.status().state, "stopped")
-    assert.deepEqual(signals, ["SIGTERM"])
+    assert.deepEqual(signals, ["SIGTERM", "SIGTERM"])
   } finally {
     await managed.stop()
   }
@@ -491,6 +499,75 @@ test("stop waits for process group descendants after the detached shell exits",
   }
 })
 
+test("stop does not return while a gracefully stopped descendant remains unreaped", async () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-process-group-"))
+  const pidPath = path.join(dir, "child.pid")
+  const child = [
+    "const fs = require('node:fs')",
+    `fs.writeFileSync(${JSON.stringify(pidPath)}, String(process.pid))`,
+    "process.on('SIGTERM', () => setTimeout(() => process.exit(0), 50))",
+    "setInterval(() => {}, 1000)"
+  ].join("; ")
+  const managed = new ManagedProcess({
+    command: `trap 'exit 0' TERM; ${JSON.stringify(process.execPath)} -e ${JSON.stringify(child)} & wait`,
+    cwd: undefined,
+    env: {},
+    id: "worker",
+    logger: () => {},
+    outputLines: 50,
+    restartDelayMs: 10,
+    shouldRestart: () => false,
+    stopSignal: "SIGTERM",
+    stopTimeoutMs: 2000
+  })
+
+  try {
+    await managed.start()
+    await waitFor(() => fs.existsSync(pidPath))
+    const childPid = Number(fs.readFileSync(pidPath, "utf8"))
+
+    await managed.stop()
+
+    assert.throws(() => process.kill(childPid, 0), {code: "ESRCH"})
+  } finally {
+    await managed.stop()
+    fs.rmSync(dir, {force: true, recursive: true})
+  }
+})
+
+test("descendant reaping and leader shutdown share one graceful deadline", async () => {
+  const managed = buildProcess(50)
+  const originalNow = Date.now
+  let now = 1000
+  /** @type {{deadline: number | undefined, signal?: string}[]} */
+  const calls = []
+
+  managed.pid = 123
+  managed.exitPromise = Promise.resolve()
+  managed.quiesce = async () => {}
+  managed.processGroupExists = () => true
+  managed.signalProcessGroup = async (signal, _pgid, deadline) => {
+    calls.push({deadline, signal})
+    now += signal === "SIGTERM" ? 100 : 0
+  }
+  managed.waitForProcessGroupExit = async (_pgid, deadline) => {
+    calls.push({deadline})
+    return calls.length > 2
+  }
+  Date.now = () => now
+
+  try {
+    await managed.stop({timeoutMs: 150})
+
+    assert.deepEqual(calls.slice(0, 2), [
+      {deadline: 1150, signal: "SIGTERM"},
+      {deadline: 1150}
+    ])
+  } finally {
+    Date.now = originalNow
+  }
+})
+
 test("a memory restart respawns and is counted when the supervisor still wants the process", async () => {
   const managed = buildLongLived(() => true)
 

package/test/release-runtime-retention.test.js

@@ -214,12 +214,15 @@ test("concurrent startup loser re-attests the winner before sending deploy", asy
       proxy: {drainTimeoutMs: 100, forceStopTimeoutMs: 100, host: "127.0.0.1", port: 0}
     }, null, 2)}\n`)
 
-    const loserDeploy = runReleaseCli(loserRelease, [
-      "deploy", "--ensure-daemon", "--config", configPath,
-      "--release-path", loserRelease, "--release-id", "loser",
-      "--daemon-log-path", path.join(root, "loser.log"), "--daemon-pid-path", loserPidPath,
-      "--daemon-runtime-path", path.join(root, "loser-runtime")
-    ])
+    const loserRejected = assert.rejects(
+      runReleaseCli(loserRelease, [
+        "deploy", "--ensure-daemon", "--config", configPath,
+        "--release-path", loserRelease, "--release-id", "loser",
+        "--daemon-log-path", path.join(root, "loser.log"), "--daemon-pid-path", loserPidPath,
+        "--daemon-runtime-path", path.join(root, "loser-runtime")
+      ]),
+      /legacy or mismatched runtime.*deploy was not sent/s
+    )
 
     await waitForFile(loserPausedPath)
     await runReleaseCli(winnerRelease, [
@@ -229,7 +232,7 @@ test("concurrent startup loser re-attests the winner before sending deploy", asy
       "--daemon-runtime-path", path.join(root, "winner-runtime")
     ])
 
-    await assert.rejects(loserDeploy, /legacy or mismatched runtime.*deploy was not sent/s)
+    await loserRejected
     const status = await sendControlCommand({command: {command: "status"}, path: socketPath})
 
     assert.equal(status.activeReleaseId, "winner")

package/test/shutdown-completion.test.js

@@ -12,7 +12,7 @@ import {fileURLToPath} from "node:url"
 import {normalizeConfig} from "../src/config.js"
 import {sendControlCommand} from "../src/control-client.js"
 import RollbridgeDaemon from "../src/daemon.js"
-import {isProcessAlive} from "../src/state-store.js"
+import {isProcessAlive, readState} from "../src/state-store.js"
 
 const dummyAppPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "dummy-app.js")
 
@@ -152,6 +152,41 @@ test("external-owner retirement releases listeners before a long-draining compan
   }
 })
 
+test("a retired owner cannot clear replacement state during late shutdown", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-retired-owner-state-"))
+  const socketPath = path.join(root, "rollbridge.sock")
+  const statePath = path.join(root, "state.json")
+  const config = normalizeConfig({...rawConfig(socketPath), statePath})
+  const retired = new RollbridgeDaemon({config, logger: () => {}})
+  let replacement
+
+  try {
+    await retired.start()
+    await retired.deploy({releaseId: "retired", releasePath: root, revision: "retired"})
+    if (retired.pendingWrite) await retired.pendingWrite
+    await retired.retireOwner({attestation: `sha256:${"a".repeat(64)}`})
+
+    replacement = new RollbridgeDaemon({config, logger: () => {}})
+    await replacement.start({reportOrphans: false})
+    await replacement.deploy({releaseId: "replacement", releasePath: root, revision: "replacement"})
+    if (replacement.pendingWrite) await replacement.pendingWrite
+
+    const replacementState = /** @type {{activeReleaseId: string} | undefined} */ (await readState(statePath))
+
+    assert.equal(replacementState?.activeReleaseId, "replacement")
+
+    await retired.shutdown()
+
+    const stateAfterRetiredShutdown = /** @type {{activeReleaseId: string} | undefined} */ (await readState(statePath))
+
+    assert.equal(stateAfterRetiredShutdown?.activeReleaseId, "replacement", "late shutdown of the retired owner must preserve replacement state")
+  } finally {
+    if (replacement) await replacement.shutdown()
+    await retired.shutdown()
+    await fs.rm(root, {force: true, recursive: true})
+  }
+})
+
 test("control socket unlink failure is reported only after owned cleanup completes", async () => {
   const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-shutdown-unlink-failure-"))
   const socketPath = path.join(root, "control.sock")