rollbridge 0.1.18 → 0.1.19

package/README.md

@@ -509,6 +509,12 @@ Shut down the daemon and managed processes:
 rollbridge shutdown --config rollbridge.js
 ```
 
+A successful shutdown response is emitted only after the targeted control
+endpoint has stopped accepting connections and been removed, owned processes
+and the proxy have stopped, and persistent state cleanup has finished. It is
+therefore safe to start or ensure a replacement daemon immediately, without a
+delay or retry loop. Cleanup failure or an already-missing daemon exits non-zero.
+
 Prepare a first Rollbridge deploy by recovering Rollbridge-managed orphans and
 stopping configured legacy processes:
 

package/docs/cli.md

@@ -292,7 +292,15 @@ rollbridge shutdown [--config <path>]
 
 Stops all managed processes (services, singletons, and releases), closes the
 proxy and control socket, removes the socket file, and prints
-`{"status": "success", "message": "shutdown"}`.
+`{"status": "success", "message": "shutdown"}`. The success response is a
+completion signal, not an early acknowledgement: before sending it, Rollbridge
+stops accepting new control connections, removes the targeted socket, finishes
+owned-process and proxy cleanup, and finalizes persistent state. A caller may
+immediately start or ensure a replacement daemon after the command returns.
+
+If cleanup fails, the command exits non-zero with the daemon's error instead of
+reporting success. Calling `shutdown` when no daemon owns the configured control
+socket also remains an explicit connection error.
 
 ## `validate`
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rollbridge",
-  "version": "0.1.18",
+  "version": "0.1.19",
   "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
   "keywords": [
     "deploy",

package/src/daemon.js

@@ -56,11 +56,14 @@ export default class RollbridgeDaemon {
     this.proxy = httpProxy.createProxyServer({ws: true, xfwd: true})
     this.proxyServer = /** @type {http.Server | undefined} */ (undefined)
     this.controlServer = /** @type {net.Server | undefined} */ (undefined)
+    this.controlSockets = /** @type {Set<net.Socket>} */ (new Set())
     this.proxyPort = /** @type {number | undefined} */ (undefined)
     this.stopping = false
     this.statePath = config.statePath
     this.persistTimer = /** @type {ReturnType<typeof setInterval> | undefined} */ (undefined)
     this.pendingWrite = /** @type {Promise<void> | undefined} */ (undefined)
+    this.shutdownPromise = /** @type {Promise<void> | undefined} */ (undefined)
+    this.controlClosePromise = /** @type {Promise<void> | undefined} */ (undefined)
     this.startingReleases = /** @type {Set<ReleaseGroup>} */ (new Set())
     // Still-alive managed processes left by a previous daemon (from statePath), captured at
     // startup and surfaced in status(). The daemon cannot re-manage them, only report them.
@@ -247,9 +250,17 @@ export default class RollbridgeDaemon {
    * @returns {void}
    */
   handleControlSocket(socket) {
+    this.controlSockets.add(socket)
     socket.setEncoding("utf8")
     let buffer = ""
 
+    socket.once("close", () => this.controlSockets.delete(socket))
+    socket.on("error", (error) => {
+      const code = error && typeof error === "object" && "code" in error && typeof error.code === "string" ? error.code : null
+
+      this.logger("control connection error", {code, error: error.message})
+    })
+
     socket.on("data", (chunk) => {
       buffer += chunk
       let newlineIndex = buffer.indexOf("\n")
@@ -269,22 +280,34 @@ export default class RollbridgeDaemon {
    * @returns {void}
    */
   handleControlLine(line, socket) {
-    this.executeControlLine(line)
-      .then((response) => socket.write(`${JSON.stringify({status: "success", ...response})}\n`))
+    const closesConnection = isShutdownControlLine(line)
+    const respond = (/** @type {Record<string, JsonValue>} */ response) => {
+      const payload = `${JSON.stringify(response)}\n`
+
+      if (closesConnection) {
+        socket.end(payload, () => socket.destroy())
+      } else if (!socket.destroyed) {
+        socket.write(payload)
+      }
+    }
+
+    this.executeControlLine(line, socket)
+      .then((response) => respond({status: "success", ...response}))
       .catch((error) => {
         this.logger("command failed", {error: error instanceof Error ? error.message : String(error)})
-        socket.write(`${JSON.stringify({
+        respond({
           error: error instanceof Error ? error.message : String(error),
           status: "error"
-        })}\n`)
+        })
       })
   }
 
   /**
    * @param {string} line - JSON command line.
+   * @param {net.Socket} [controlSocket] - Requesting control connection, used only for shutdown completion.
    * @returns {Promise<Record<string, JsonValue>>} Command response.
    */
-  async executeControlLine(line) {
+  async executeControlLine(line, controlSocket) {
     const command = JSON.parse(line)
 
     if (!command || typeof command !== "object") {
@@ -327,11 +350,10 @@ export default class RollbridgeDaemon {
     }
 
     if (commandName === "shutdown") {
-      setImmediate(() => {
-        this.shutdown().catch((error) => {
-          this.logger("shutdown failed", {error: error instanceof Error ? error.message : String(error)})
-        })
-      })
+      // Stop accepting new control connections before cleanup, but keep this requesting
+      // connection open as the completion channel. Waiting for all control connections here
+      // would deadlock: server.close() includes the socket awaiting this response.
+      await this.shutdown({completionSocket: controlSocket, waitForControlConnections: false})
 
       return {message: "shutdown"}
     }
@@ -763,30 +785,56 @@ export default class RollbridgeDaemon {
     }
   }
 
-  /** @returns {Promise<void>} Stops proxy, control socket, and child processes. */
-  async shutdown() {
-    if (this.stopping) return
+  /**
+   * Stops proxy, control socket, and child processes.
+   * @param {{completionSocket?: net.Socket, waitForControlConnections?: boolean}} [options] - Shutdown connection behavior.
+   * @returns {Promise<void>} Resolves when owned resources are stopped (and, by default, control connections close).
+   */
+  async shutdown({completionSocket, waitForControlConnections = true} = {}) {
+    if (!this.shutdownPromise) this.shutdownPromise = this.performShutdown(completionSocket)
+
+    await this.shutdownPromise
+    if (waitForControlConnections && this.controlClosePromise) await this.controlClosePromise
+  }
 
+  /**
+   * @param {net.Socket | undefined} completionSocket - Requester retained for the final response.
+   * @returns {Promise<void>} Retires listeners and cleans up every daemon-owned resource.
+   */
+  async performShutdown(completionSocket) {
     this.stopping = true
+    const cleanupErrors = /** @type {Error[]} */ ([])
+
+    // server.close() stops new connections synchronously. Unlink immediately afterward so a
+    // replacement can bind as soon as cleanup completes; existing connections remain usable for
+    // the shutdown completion/error response.
+    this.controlClosePromise = this.closeServer(this.controlServer)
+
+    for (const socket of this.controlSockets) {
+      if (socket !== completionSocket) socket.destroy()
+    }
+
+    await captureShutdownError(cleanupErrors, "control socket unlink", () => this.removeControlSocket())
 
     if (this.persistTimer) {
       clearInterval(this.persistTimer)
       this.persistTimer = undefined
     }
 
-    this.proxy.close()
-    await Promise.allSettled([...this.services.values()].map((processInstance) => processInstance.stop()))
-    await Promise.allSettled([...this.singletons.values()].map((processInstance) => processInstance.stop()))
-    await Promise.allSettled([...this.startingReleases].map((release) => release.stop()))
-    await Promise.allSettled([...this.releases.values()].map((release) => release.stop()))
-    await this.closeServer(this.proxyServer)
-    await this.closeServer(this.controlServer)
-    await fs.rm(this.config.control.path, {force: true})
+    await captureShutdownError(cleanupErrors, "proxy close", async () => this.proxy.close())
+    const stopResults = await Promise.allSettled([
+      ...[...this.services.values()].map((processInstance) => processInstance.stop()),
+      ...[...this.singletons.values()].map((processInstance) => processInstance.stop()),
+      ...[...this.startingReleases].map((release) => release.stop()),
+      ...[...this.releases.values()].map((release) => release.stop())
+    ])
+    await captureShutdownError(cleanupErrors, "proxy server close", () => this.closeServer(this.proxyServer))
 
     // Wait for any in-flight write first so it can't recreate or overwrite the final state (no
     // new writes start: stopping is set and the persist timer is cleared above). Prior-daemon
     // orphans are not owned by this daemon, so retain their records until they are confirmed gone.
-    if (this.statePath) {
+    await captureShutdownError(cleanupErrors, "persistent state cleanup", async () => {
+      if (!this.statePath) return
       if (this.pendingWrite) await this.pendingWrite
       const orphans = this.orphans.filter((orphan) => isProcessAlive(orphan.pid))
 
@@ -795,7 +843,20 @@ export default class RollbridgeDaemon {
       } else {
         await clearState(this.statePath)
       }
+    })
+
+    const stopErrors = stopResults.filter((result) => result.status === "rejected").map((result) => result.reason)
+
+    if (stopErrors.length > 0) {
+      cleanupErrors.push(new AggregateError(stopErrors, `Shutdown failed to stop ${stopErrors.length} owned resource${stopErrors.length === 1 ? "" : "s"}.`))
     }
+
+    if (cleanupErrors.length > 0) throw new AggregateError(cleanupErrors, cleanupErrors.map((error) => error.message).join("; "))
+  }
+
+  /** @returns {Promise<void>} Removes the configured control socket path. */
+  async removeControlSocket() {
+    await fs.rm(this.config.control.path, {force: true})
   }
 
   /**
@@ -856,6 +917,37 @@ function stringOrUndefined(value) {
   return value
 }
 
+/**
+ * @param {string} line - Raw control line.
+ * @returns {boolean} Whether the line requests shutdown and needs a terminal response connection.
+ */
+function isShutdownControlLine(line) {
+  try {
+    const command = JSON.parse(line)
+
+    return Boolean(command && typeof command === "object" && command.command === "shutdown")
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Runs one shutdown cleanup step and records a labeled failure without skipping later cleanup.
+ * @param {Error[]} errors - Accumulated cleanup errors.
+ * @param {string} label - Non-secret cleanup step name.
+ * @param {() => Promise<void>} operation - Cleanup operation.
+ * @returns {Promise<void>} Resolves after the operation succeeds or its failure is recorded.
+ */
+async function captureShutdownError(errors, label, operation) {
+  try {
+    await operation()
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error)
+
+    errors.push(new Error(`${label} failed: ${reason}`, {cause: error}))
+  }
+}
+
 const SECRET_BEARING_STATE_KEYS = new Set(["children", "command", "cwd", "env", "environment", "logs", "output"])
 
 /**

package/test/shutdown-completion.test.js

@@ -0,0 +1,290 @@
+// @ts-check
+
+import assert from "node:assert/strict"
+import {spawn} from "node:child_process"
+import {once} from "node:events"
+import fs from "node:fs/promises"
+import net from "node:net"
+import os from "node:os"
+import path from "node:path"
+import test from "node:test"
+import {fileURLToPath} from "node:url"
+import {normalizeConfig} from "../src/config.js"
+import {sendControlCommand} from "../src/control-client.js"
+import RollbridgeDaemon from "../src/daemon.js"
+import {isProcessAlive} from "../src/state-store.js"
+
+const dummyAppPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "dummy-app.js")
+
+test("shutdown response waits for endpoint and owned-process cleanup before immediate replacement", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-shutdown-completion-"))
+  const socketPath = path.join(root, "control.sock")
+  const unrelatedSocketPath = path.join(root, "unrelated.sock")
+  const gatePath = path.join(root, "shutdown.fifo")
+  const stoppingPath = path.join(root, "stopping")
+  const gate = spawn("mkfifo", [gatePath])
+
+  assert.equal((await once(gate, "exit"))[0], 0)
+
+  const config = buildConfig(socketPath, {
+    companion: {
+      command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
+      id: "worker",
+      lifecycle: {drainTimeoutMs: 0, quietCommand: `printf stopping > ${JSON.stringify(stoppingPath)}; read released < ${JSON.stringify(gatePath)}`},
+      policy: "companion"
+    }
+  })
+  const unrelatedConfig = buildConfig(unrelatedSocketPath)
+  const daemon = new RollbridgeDaemon({config, logger: () => {}})
+  const unrelated = new RollbridgeDaemon({config: unrelatedConfig, logger: () => {}})
+  let idleTarget = /** @type {net.Socket | undefined} */ (undefined)
+  let idleUnrelated = /** @type {net.Socket | undefined} */ (undefined)
+  let replacement
+  let gateReleased = false
+
+  try {
+    await daemon.start()
+    await unrelated.start()
+    idleTarget = net.createConnection(socketPath)
+    idleUnrelated = net.createConnection(unrelatedSocketPath)
+    await Promise.all([once(idleTarget, "connect"), once(idleUnrelated, "connect")])
+    await daemon.deploy({releaseId: "v1", releasePath: root, revision: "v1"})
+
+    const workerPid = daemon.activeRelease?.getProcess("worker")?.pid
+
+    assert.equal(typeof workerPid, "number")
+
+    const stopping = waitForFile(stoppingPath)
+    let shutdownResolved = false
+    const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
+      .then((response) => {
+        shutdownResolved = true
+        return response
+      })
+
+    await stopping
+
+    let oldEndpointAccepted = true
+
+    try {
+      await sendControlCommand({command: {command: "status"}, path: socketPath})
+    } catch {
+      oldEndpointAccepted = false
+    }
+
+    const resolvedDuringStop = shutdownResolved
+    const processAliveDuringStop = isProcessAlive(/** @type {number} */ (workerPid))
+    const idleTargetClosedDuringStop = idleTarget.destroyed
+    const idleUnrelatedClosedDuringStop = idleUnrelated.destroyed
+
+    // Ensure the RED path cannot leave an idle client handle blocking test cleanup.
+    idleTarget.destroy()
+
+    await fs.writeFile(gatePath, "continue\n")
+    gateReleased = true
+
+    const response = await shutdown
+
+    assert.equal(shutdownResolved, true)
+    assert.equal(resolvedDuringStop, false, "shutdown must not acknowledge while an owned process is still stopping")
+    assert.equal(oldEndpointAccepted, false, "the targeted endpoint must stop accepting new commands before cleanup")
+    assert.equal(processAliveDuringStop, true, "the fixture must hold shutdown while its owned process is alive")
+    assert.equal(idleTargetClosedDuringStop, true, "an idle accepted client must be closed when the targeted endpoint retires")
+    assert.equal(idleUnrelatedClosedDuringStop, false, "an unrelated daemon's accepted clients must remain untouched")
+    assert.deepEqual(response, {message: "shutdown", status: "success"})
+    await assert.rejects(() => fs.stat(socketPath), {code: "ENOENT"})
+    assert.equal(isProcessAlive(/** @type {number} */ (workerPid)), false)
+
+    // A different daemon remains reachable; shutdown is scoped to the targeted control endpoint.
+    assert.equal((await sendControlCommand({command: {command: "status"}, path: unrelatedSocketPath})).application, "shutdown-unrelated")
+
+    // Replacement starts immediately, with no polling or retry between truthful ACK and bind.
+    replacement = new RollbridgeDaemon({config, logger: () => {}})
+    await replacement.start()
+    assert.equal((await sendControlCommand({command: {command: "status"}, path: socketPath})).application, "shutdown-target")
+  } finally {
+    if (!gateReleased) {
+      await fs.writeFile(gatePath, "continue\n").catch(() => {})
+    }
+    idleTarget?.destroy()
+    idleUnrelated?.destroy()
+    if (replacement) await replacement.shutdown()
+    await daemon.shutdown()
+    await unrelated.shutdown()
+    await fs.rm(root, {force: true, recursive: true})
+  }
+})
+
+test("control socket unlink failure is reported only after owned cleanup completes", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-shutdown-unlink-failure-"))
+  const socketPath = path.join(root, "control.sock")
+  const statePath = path.join(root, "state.json")
+  const config = normalizeConfig({...rawConfig(socketPath), statePath})
+  const daemon = new RollbridgeDaemon({config, logger: () => {}})
+
+  try {
+    await daemon.start()
+    await daemon.deploy({releaseId: "v1", releasePath: root, revision: "v1"})
+    if (daemon.pendingWrite) await daemon.pendingWrite
+
+    const webPid = daemon.activeRelease?.getProcess("web")?.pid
+    const proxyPort = daemon.getProxyPort()
+
+    assert.equal(typeof webPid, "number")
+    assert.equal(typeof proxyPort, "number")
+
+    daemon.removeControlSocket = async () => { throw new Error("injected unlink failure") }
+
+    await assert.rejects(
+      () => sendControlCommand({command: {command: "shutdown"}, path: socketPath}),
+      /control socket unlink failed: injected unlink failure/
+    )
+
+    assert.equal(isProcessAlive(/** @type {number} */ (webPid)), false, "unlink failure must not strand an owned process")
+    await assert.rejects(() => fetch(`http://127.0.0.1:${proxyPort}/ping`))
+    await assert.rejects(() => fs.stat(statePath), {code: "ENOENT"})
+  } finally {
+    await fs.rm(root, {force: true, recursive: true})
+  }
+})
+
+test("direct shutdown closes idle accepted clients and converges", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-direct-shutdown-idle-"))
+  const socketPath = path.join(root, "control.sock")
+  const daemon = new RollbridgeDaemon({config: buildConfig(socketPath), logger: () => {}})
+  let idle = /** @type {net.Socket | undefined} */ (undefined)
+
+  try {
+    await daemon.start()
+    idle = net.createConnection(socketPath)
+    await once(idle, "connect")
+    const idleClosed = once(idle, "close")
+
+    await daemon.shutdown()
+    await idleClosed
+
+    assert.equal(idle.destroyed, true)
+    await assert.rejects(() => fs.stat(socketPath), {code: "ENOENT"})
+  } finally {
+    idle?.destroy()
+    await fs.rm(root, {force: true, recursive: true})
+  }
+})
+
+test("shutdown reports cleanup failure and still retires the targeted endpoint", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-shutdown-failure-"))
+  const socketPath = path.join(root, "control.sock")
+  const statePath = path.join(root, "state-directory")
+
+  await fs.mkdir(statePath)
+
+  const config = normalizeConfig({
+    ...rawConfig(socketPath),
+    statePath
+  })
+  const daemon = new RollbridgeDaemon({config, logger: () => {}})
+
+  try {
+    await daemon.start()
+
+    await assert.rejects(
+      () => sendControlCommand({command: {command: "shutdown"}, path: socketPath}),
+      /directory|EISDIR/i
+    )
+    await assert.rejects(() => fs.stat(socketPath), {code: "ENOENT"})
+  } finally {
+    await fs.rm(root, {force: true, recursive: true})
+  }
+})
+
+test("shutdown does not turn an owned-resource stop rejection into success", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-shutdown-stop-failure-"))
+  const socketPath = path.join(root, "control.sock")
+  const config = buildConfig(socketPath)
+  const daemon = new RollbridgeDaemon({config, logger: () => {}})
+  let restoreStop
+
+  try {
+    await daemon.start()
+    await daemon.deploy({releaseId: "v1", releasePath: root, revision: "v1"})
+
+    const release = daemon.activeRelease
+
+    assert.ok(release)
+    const originalStop = release.stop.bind(release)
+
+    restoreStop = originalStop
+    release.stop = async () => { throw new Error("owned release stop failed") }
+
+    await assert.rejects(
+      () => sendControlCommand({command: {command: "shutdown"}, path: socketPath}),
+      /Shutdown failed to stop 1 owned resource/
+    )
+  } finally {
+    if (restoreStop) await restoreStop()
+    await fs.rm(root, {force: true, recursive: true})
+  }
+})
+
+test("shutdown of an already-stopped endpoint fails explicitly", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-shutdown-missing-"))
+  const socketPath = path.join(root, "missing.sock")
+
+  try {
+    await assert.rejects(
+      () => sendControlCommand({command: {command: "shutdown"}, path: socketPath}),
+      (error) => Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT")
+    )
+  } finally {
+    await fs.rm(root, {force: true, recursive: true})
+  }
+})
+
+/**
+ * @param {string} socketPath - Control socket path.
+ * @param {{companion?: Record<string, import("../src/json.js").JsonValue>}} [options] - Optional companion process.
+ * @returns {import("../src/config.js").RollbridgeConfig} Normalized config.
+ */
+function buildConfig(socketPath, {companion} = {}) {
+  return normalizeConfig({
+    ...rawConfig(socketPath),
+    ...(companion ? {processes: [companion, ...rawConfig(socketPath).processes]} : {})
+  })
+}
+
+/**
+ * @param {string} socketPath - Control socket path.
+ * @returns {{application: string, control: {path: string}, processes: Record<string, import("../src/json.js").JsonValue>[], proxy: {forceStopTimeoutMs: number, host: string, port: number}}} Raw config.
+ */
+function rawConfig(socketPath) {
+  return {
+    application: socketPath.endsWith("unrelated.sock") ? "shutdown-unrelated" : "shutdown-target",
+    control: {path: socketPath},
+    processes: [{
+      command: `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`,
+      health: {intervalMs: 25, path: "/ping", timeoutMs: 3000},
+      id: "web",
+      policy: "proxied",
+      port: {from: 0, to: 0}
+    }],
+    proxy: {forceStopTimeoutMs: 1000, host: "127.0.0.1", port: 0}
+  }
+}
+
+/**
+ * @param {string} filePath - File to await without polling.
+ * @returns {Promise<void>} Resolves when the file appears.
+ */
+async function waitForFile(filePath) {
+  const watcher = fs.watch(path.dirname(filePath))
+
+  try {
+    for await (const event of watcher) {
+      if (event.filename === path.basename(filePath)) return
+    }
+  } finally {
+    await watcher.return?.()
+  }
+
+  throw new Error(`Watcher ended before ${filePath} appeared`)
+}