rollbridge 0.1.4 → 0.1.6

package/src/managed-process.js

@@ -2,14 +2,16 @@
 
 import {EventEmitter} from "node:events"
 import {spawn} from "node:child_process"
+import {processGroupMembers} from "./process-memory.js"
 
 /**
  * @typedef {import("./json.js").JsonValue} JsonValue
  * @typedef {"starting" | "running" | "stopping" | "stopped" | "failed"} ManagedProcessState
+ * @typedef {"deploy" | "crash" | "manual" | "memory"} ManagedProcessStartReason
  * @typedef {import("node:child_process").ChildProcess["signalCode"]} ProcessExitSignal
  * @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
- * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, outputLines: number, restartDelayMs: number, shouldRestart: () => boolean, stopTimeoutMs: number}} ManagedProcessDefinition
- * @typedef {{command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, logs: ManagedProcessLog[], pid: number | undefined, restarts: number, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
+ * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: number}} ManagedProcessDefinition
+ * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
  */
 
 export default class ManagedProcess extends EventEmitter {
@@ -20,26 +22,43 @@ export default class ManagedProcess extends EventEmitter {
    * @param {Record<string, string | undefined>} args.env - Environment.
    * @param {string} args.id - Process id.
    * @param {(message: string, data?: Record<string, JsonValue>) => void} args.logger - Logger callback.
+   * @param {import("./config.js").LifecycleConfig} [args.lifecycle] - Graceful-stop lifecycle hooks (none by default).
+   * @param {import("./config.js").MemoryConfig} [args.memory] - Memory supervision config (off when omitted).
    * @param {number} args.outputLines - Recent stdout/stderr lines to retain and report.
+   * @param {import("./config.js").RestartConfig} [args.restart] - Restart policy (defaults to unlimited restarts with a constant delay).
    * @param {number} args.restartDelayMs - Restart delay.
    * @param {() => boolean} args.shouldRestart - Restart policy callback.
+   * @param {string} [args.stopSignal] - Signal sent to gracefully stop the process (default "SIGTERM").
    * @param {number} args.stopTimeoutMs - Stop timeout.
    */
-  constructor({command, cwd, env, id, logger, outputLines, restartDelayMs, shouldRestart, stopTimeoutMs}) {
+  constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, stopSignal = "SIGTERM", stopTimeoutMs}) {
     super()
 
     this.command = command
     this.cwd = cwd
     this.env = env
     this.id = id
+    this.lifecycle = lifecycle
     this.logger = logger
+    this.memory = memory
     this.outputLines = outputLines
+    this.restart = restart
     this.restartDelayMs = restartDelayMs
     this.shouldRestart = shouldRestart
+    this.stopSignal = stopSignal
     this.stopTimeoutMs = stopTimeoutMs
     this.state = /** @type {ManagedProcessState} */ ("stopped")
+    this.lastStartReason = /** @type {ManagedProcessStartReason | undefined} */ (undefined)
     this.logs = /** @type {ManagedProcessLog[]} */ ([])
     this.restarts = 0
+    this.recentRestarts = /** @type {number[]} */ ([])
+    this.rssBytes = /** @type {number | undefined} */ (undefined)
+    this.children = /** @type {import("./process-memory.js").ProcessGroupMember[]} */ ([])
+    this.memoryRestarts = 0
+    this.lastMemoryRestartAtMs = /** @type {number | undefined} */ (undefined)
+    this.memoryTimer = /** @type {ReturnType<typeof setInterval> | undefined} */ (undefined)
+    this.memoryRestarting = false
+    this.memoryWarned = false
     this.startedAtMs = /** @type {number | undefined} */ (undefined)
     this.intentionalStop = false
     this.restartTimer = undefined
@@ -50,8 +69,11 @@ export default class ManagedProcess extends EventEmitter {
     this.exitSignal = undefined
   }
 
-  /** @returns {Promise<void>} Resolves after spawn. */
-  async start() {
+  /**
+   * @param {ManagedProcessStartReason} [reason] - Why the process is being started (deploy by default; "crash" on auto-restart, "manual" via the restart command).
+   * @returns {Promise<void>} Resolves after spawn.
+   */
+  async start(reason = "deploy") {
     if (this.child) return
 
     this.intentionalStop = false
@@ -80,7 +102,9 @@ export default class ManagedProcess extends EventEmitter {
       child.once("spawn", () => {
         this.state = "running"
         this.startedAtMs = Date.now()
-        this.logger("process started", {command: this.command, id: this.id, pid: child.pid || null})
+        this.lastStartReason = reason
+        this.logger("process started", {command: this.command, id: this.id, pid: child.pid || null, reason})
+        this.startMemoryMonitor()
         this.emit("started")
         resolve(undefined)
       })
@@ -104,10 +128,14 @@ export default class ManagedProcess extends EventEmitter {
     this.command = definition.command
     this.cwd = definition.cwd
     this.env = definition.env
+    this.lifecycle = definition.lifecycle
     this.logger = definition.logger
+    this.memory = definition.memory
     this.outputLines = definition.outputLines
+    this.restart = definition.restart
     this.restartDelayMs = definition.restartDelayMs
     this.shouldRestart = definition.shouldRestart
+    this.stopSignal = definition.stopSignal
     this.stopTimeoutMs = definition.stopTimeoutMs
   }
 
@@ -141,18 +169,166 @@ export default class ManagedProcess extends EventEmitter {
     this.child = undefined
     this.pid = undefined
     this.exitPromise = undefined
+    this.rssBytes = undefined
+    this.children = []
+    this.clearMemoryMonitor()
     this.state = wasIntentional ? "stopped" : "failed"
     this.logger("process exited", {code, id: this.id, signal})
     this.emit("exit", {code, signal})
 
     if (!wasIntentional && this.shouldRestart()) {
-      this.restartTimer = setTimeout(() => {
-        this.restartTimer = undefined
-        this.restarts += 1
-        this.start().catch((error) => {
-          this.logger("process restart failed", {error: error instanceof Error ? error.message : String(error), id: this.id})
-        })
-      }, this.restartDelayMs)
+      this.scheduleRestart()
+    }
+  }
+
+  /**
+   * Schedules an automatic restart per the restart policy, or gives up once the policy's limit is reached.
+   * @returns {void}
+   */
+  scheduleRestart() {
+    const {backoffFactor, maxRestarts, windowMs} = this.restart
+
+    // Fast path: unlimited restarts with a constant delay needs no per-restart bookkeeping.
+    // The delay is constant across attempts here (backoffFactor is 1), so restartDelayFor(0)
+    // gives the right value while still applying any maxDelayMs cap.
+    if (maxRestarts === undefined && backoffFactor === 1) {
+      this.queueRestart(this.restartDelayFor(0))
+
+      return
+    }
+
+    const now = Date.now()
+
+    if (windowMs > 0) {
+      this.recentRestarts = this.recentRestarts.filter((time) => time > now - windowMs)
+    }
+
+    if (maxRestarts !== undefined && this.recentRestarts.length >= maxRestarts) {
+      this.logger("restart limit reached", {id: this.id, maxRestarts, windowMs})
+
+      return
+    }
+
+    const delay = this.restartDelayFor(this.recentRestarts.length)
+
+    this.recentRestarts.push(now)
+    this.queueRestart(delay)
+  }
+
+  /**
+   * @param {number} attempt - Number of restarts already counted in the current window.
+   * @returns {number} Backed-off restart delay in milliseconds, capped by maxDelayMs when set.
+   */
+  restartDelayFor(attempt) {
+    const backedOff = this.restartDelayMs * this.restart.backoffFactor ** attempt
+
+    return this.restart.maxDelayMs > 0 ? Math.min(backedOff, this.restart.maxDelayMs) : backedOff
+  }
+
+  /**
+   * @param {number} delayMs - Delay before the restart attempt.
+   * @returns {void}
+   */
+  queueRestart(delayMs) {
+    this.restartTimer = setTimeout(() => {
+      this.restartTimer = undefined
+      this.restarts += 1
+      this.start("crash").catch((error) => {
+        this.logger("process restart failed", {error: error instanceof Error ? error.message : String(error), id: this.id})
+      })
+    }, delayMs)
+
+    // The daemon's listening servers govern its lifetime; a pending restart must never be the sole
+    // handle keeping the process alive (like the memory and persist timers above). Otherwise a
+    // crashed process with an unlimited restart policy would respawn forever and block exit.
+    this.restartTimer.unref?.()
+  }
+
+  /**
+   * Starts the periodic RSS check for this process when memory supervision is configured.
+   * @returns {void}
+   */
+  startMemoryMonitor() {
+    this.clearMemoryMonitor()
+
+    if (!this.memory) return
+
+    this.memoryTimer = setInterval(() => this.checkMemory(), this.memory.checkIntervalMs)
+    this.memoryTimer.unref?.()
+  }
+
+  /** @returns {void} Stops the periodic RSS check. */
+  clearMemoryMonitor() {
+    if (this.memoryTimer) {
+      clearInterval(this.memoryTimer)
+      this.memoryTimer = undefined
+    }
+  }
+
+  /**
+   * Measures the process group's RSS and warns or restarts when it crosses the configured thresholds.
+   * @returns {void}
+   */
+  checkMemory() {
+    if (!this.memory || !this.pid || this.memoryRestarting) return
+
+    const members = processGroupMembers(this.pid)
+
+    if (members.length === 0) return
+
+    this.children = members
+
+    const measured = members.filter((member) => member.rssBytes !== undefined)
+
+    if (measured.length === 0) return
+
+    const rssBytes = measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
+
+    this.rssBytes = rssBytes
+
+    if (rssBytes > this.memory.limitBytes) {
+      this.logger("memory limit exceeded", {id: this.id, limitBytes: this.memory.limitBytes, rssBytes})
+      void this.restartForMemory()
+
+      return
+    }
+
+    if (this.memory.warnBytes > 0 && rssBytes > this.memory.warnBytes) {
+      if (!this.memoryWarned) {
+        this.logger("memory warning", {id: this.id, rssBytes, warnBytes: this.memory.warnBytes})
+        this.memoryWarned = true
+      }
+    } else {
+      this.memoryWarned = false
+    }
+  }
+
+  /**
+   * Gracefully restarts the process after it exceeded its memory limit (SIGTERM, then
+   * SIGKILL after the stop timeout), recording the restart so status can report it.
+   * @returns {Promise<void>} Resolves once the process has been restarted.
+   */
+  async restartForMemory() {
+    if (this.memoryRestarting) return
+
+    this.memoryRestarting = true
+
+    try {
+      await this.stop()
+
+      // Don't respawn if the supervising context no longer wants this process running
+      // (daemon shutting down, or the release draining/retired) — otherwise a restart racing
+      // a shutdown could leave a child running after shutdown collected its stop promises.
+      if (!this.shouldRestart()) return
+
+      this.memoryRestarts += 1
+      this.lastMemoryRestartAtMs = Date.now()
+      this.memoryWarned = false
+      await this.start("memory")
+    } catch (error) {
+      this.logger("memory restart failed", {error: error instanceof Error ? error.message : String(error), id: this.id})
+    } finally {
+      this.memoryRestarting = false
     }
   }
 
@@ -162,6 +338,7 @@ export default class ManagedProcess extends EventEmitter {
    */
   async stop(options = {}) {
     this.intentionalStop = true
+    this.clearMemoryMonitor()
 
     if (this.restartTimer) {
       clearTimeout(this.restartTimer)
@@ -176,21 +353,104 @@ export default class ManagedProcess extends EventEmitter {
     }
 
     this.state = "stopping"
-    this.killProcessGroup("SIGTERM")
-    const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
-    const stopped = await this.waitForExit(timeoutMs)
-
-    if (!stopped) {
-      this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: child.pid})
-      this.killProcessGroup("SIGKILL")
-      await this.waitForExit(5000)
+
+    const {drainCommand, drainTimeoutMs, quietCommand, stopCommand} = this.lifecycle
+
+    // 1. Quiesce: tell the process to stop accepting new work.
+    if (quietCommand) await this.runHook(quietCommand, this.stopTimeoutMs, "quiet command")
+
+    // 2. Drain: let in-flight work finish, bounded by drainTimeoutMs (0 skips the step). A
+    //    drainCommand blocks until drained; otherwise wait for the process to exit on its own.
+    if (this.child && drainTimeoutMs > 0) {
+      if (drainCommand) await this.runHook(drainCommand, drainTimeoutMs, "drain command")
+      else await this.waitForExit(drainTimeoutMs)
+    }
+
+    // 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
+    if (this.child) {
+      if (stopCommand) await this.runHook(stopCommand, this.stopTimeoutMs, "stop command")
+      else this.killProcessGroup(this.stopSignal)
+
+      const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
+
+      if (this.child && !(await this.waitForExit(timeoutMs))) {
+        this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: this.pid})
+        this.killProcessGroup("SIGKILL")
+        await this.waitForExit(5000)
+      }
     }
 
     this.state = "stopped"
   }
 
   /**
-   * @param {"SIGTERM" | "SIGKILL"} signal - Signal to send.
+   * Runs a lifecycle hook command, bounded by a timeout so a hung hook can never block stop().
+   * Failures are logged and swallowed — the graceful-stop sequence proceeds (and SIGKILL is the
+   * ultimate fallback) regardless of the hook's outcome.
+   * @param {string} command - Shell command to run.
+   * @param {number} timeoutMs - Maximum time to wait for the hook before killing it.
+   * @param {string} label - Hook name, for log messages.
+   * @returns {Promise<void>} Resolves when the hook exits, errors, or times out.
+   */
+  async runHook(command, timeoutMs, label) {
+    await new Promise((resolve) => {
+      let settled = false
+      const finish = () => { if (!settled) { settled = true; resolve(undefined) } }
+
+      /** @type {import("node:child_process").ChildProcess} */
+      let hook
+
+      try {
+        hook = spawn(command, {
+          cwd: this.cwd,
+          detached: true,
+          env: {...process.env, ...this.env, ROLLBRIDGE_PID: this.pid ? String(this.pid) : ""},
+          shell: true,
+          stdio: "ignore"
+        })
+      } catch (error) {
+        this.logger(`${label} failed`, {error: error instanceof Error ? error.message : String(error), id: this.id})
+        finish()
+
+        return
+      }
+
+      const timer = setTimeout(() => {
+        this.logger(`${label} timed out`, {id: this.id, timeoutMs})
+
+        if (hook.pid) {
+          try {
+            process.kill(-hook.pid, "SIGKILL")
+          } catch {
+            // The hook already exited.
+          }
+        }
+
+        finish()
+      }, timeoutMs)
+
+      hook.once("exit", (code, signal) => {
+        clearTimeout(timer)
+
+        // A non-zero/signalled exit is surfaced (but still non-fatal); skip when the timeout
+        // already killed the hook, which logs separately.
+        if (!settled) {
+          if (typeof code === "number" && code !== 0) this.logger(`${label} exited non-zero`, {code, id: this.id})
+          else if (signal) this.logger(`${label} exited on signal`, {id: this.id, signal})
+        }
+
+        finish()
+      })
+      hook.once("error", (error) => {
+        clearTimeout(timer)
+        this.logger(`${label} failed`, {error: error instanceof Error ? error.message : String(error), id: this.id})
+        finish()
+      })
+    })
+  }
+
+  /**
+   * @param {string} signal - Signal name to send (the configured stop signal, or "SIGKILL").
    * @returns {void}
    */
   killProcessGroup(signal) {
@@ -226,14 +486,19 @@ export default class ManagedProcess extends EventEmitter {
   /** @returns {ManagedProcessStatus} Status payload. */
   status() {
     return {
+      children: this.children,
       command: this.command,
       cwd: this.cwd,
       exitCode: this.exitCode,
       exitSignal: this.exitSignal,
       id: this.id,
+      lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
+      lastStartReason: this.lastStartReason,
       logs: this.logs.slice(-this.outputLines),
+      memoryRestarts: this.memoryRestarts,
       pid: this.pid,
       restarts: this.restarts,
+      rssBytes: this.rssBytes,
       startedAt: this.startedAtMs === undefined ? undefined : new Date(this.startedAtMs).toISOString(),
       state: this.state,
       uptimeMs: this.state === "running" && this.startedAtMs !== undefined ? Date.now() - this.startedAtMs : undefined

package/src/process-memory.js

@@ -0,0 +1,110 @@
+// @ts-check
+
+import fs from "node:fs"
+
+/**
+ * @typedef {{command: string, pid: number, rssBytes: number | undefined}} ProcessGroupMember
+ */
+
+/**
+ * Lists the members of a managed process group with each member's resident memory.
+ * Rollbridge spawns each process detached, so the spawned pid is the process-group
+ * leader and every process in the tree (the shell wrapper, the app, any children)
+ * shares that group id.
+ *
+ * Reads `/proc` (Linux); returns an empty array when unavailable (no `/proc`, e.g.
+ * non-Linux) or the group has no members.
+ * @param {number} pgid - Process-group id (the detached spawn's pid).
+ * @returns {ProcessGroupMember[]} Group members, ordered by pid.
+ */
+export function processGroupMembers(pgid) {
+  /** @type {string[]} */
+  let entries
+
+  try {
+    entries = fs.readdirSync("/proc")
+  } catch {
+    return []
+  }
+
+  /** @type {ProcessGroupMember[]} */
+  const members = []
+
+  for (const entry of entries) {
+    if (!/^\d+$/.test(entry)) continue
+    if (processGroupId(entry) !== pgid) continue
+
+    members.push({command: commandName(entry), pid: Number(entry), rssBytes: residentBytes(entry)})
+  }
+
+  members.sort((first, second) => first.pid - second.pid)
+
+  return members
+}
+
+/**
+ * Measures the total resident memory (RSS) of a managed process group.
+ * @param {number} pgid - Process-group id (the detached spawn's pid).
+ * @returns {number | undefined} Total resident memory in bytes, or undefined when unmeasurable.
+ */
+export function measureProcessGroupRssBytes(pgid) {
+  const measured = processGroupMembers(pgid).filter((member) => member.rssBytes !== undefined)
+
+  if (measured.length === 0) return undefined
+
+  return measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
+}
+
+/**
+ * @param {string} pid - Process id.
+ * @returns {string} The process command name (`/proc/<pid>/comm`), or "" when unavailable.
+ */
+function commandName(pid) {
+  try {
+    return fs.readFileSync(`/proc/${pid}/comm`, "utf8").trim()
+  } catch {
+    return ""
+  }
+}
+
+/**
+ * @param {string} pid - Process id.
+ * @returns {number | undefined} The process-group id, or undefined when the process is gone.
+ */
+function processGroupId(pid) {
+  let stat
+
+  try {
+    stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8")
+  } catch {
+    return undefined
+  }
+
+  // The comm field is wrapped in parens and may itself contain spaces or parens, so the
+  // numeric fields are parsed from after the final ")". They are: state, ppid, pgrp, ...
+  const commEnd = stat.lastIndexOf(")")
+
+  if (commEnd < 0) return undefined
+
+  const pgrp = Number(stat.slice(commEnd + 2).split(" ")[2])
+
+  return Number.isInteger(pgrp) ? pgrp : undefined
+}
+
+/**
+ * @param {string} pid - Process id.
+ * @returns {number | undefined} Resident memory in bytes, or undefined when unavailable.
+ */
+function residentBytes(pid) {
+  let status
+
+  try {
+    status = fs.readFileSync(`/proc/${pid}/status`, "utf8")
+  } catch {
+    return undefined
+  }
+
+  const match = status.match(/^VmRSS:\s+(\d+)\s+kB/m)
+
+  return match ? Number(match[1]) * 1024 : undefined
+}

package/src/recover.js

@@ -0,0 +1,134 @@
+// @ts-check
+
+import {inspectControlSocket} from "./daemon.js"
+import {clearState, isProcessAlive, liveProcesses, readState} from "./state-store.js"
+
+// How long to confirm a SIGKILL'd group has actually exited before reporting it un-stoppable.
+const KILL_CONFIRM_TIMEOUT_MS = 500
+
+/**
+ * @typedef {{id: string, pid: number, releaseId: string | null}} OrphanProcess
+ * @typedef {{error: string}} RecoverError
+ * @typedef {{cleared: boolean, forced: boolean, orphans: OrphanProcess[], remaining: OrphanProcess[]}} RecoverReport
+ * @typedef {RecoverError | RecoverReport} RecoverResult
+ */
+
+/**
+ * Cleans up orphaned managed processes left by a crashed daemon. Reads the persisted state
+ * (config.statePath) and finds managed processes whose pids are still alive. By default it
+ * only reports them; with `force` it stops each one's process group (SIGTERM, then SIGKILL
+ * after the configured timeout) and clears the stale state file.
+ *
+ * When `force` leaves any orphan still running (for example a process owned by another user
+ * that can't be signaled), the state file is **kept** so the operator can investigate and
+ * re-run recovery — the survivors are returned in `remaining` and `cleared` stays false.
+ *
+ * Refuses to run while a daemon (or another process) holds the control socket — those pids
+ * belong to a live daemon, not a crash. A recycled pid can be a false positive, so review the
+ * dry-run list before using `force`.
+ * @param {object} args - Options.
+ * @param {import("./config.js").RollbridgeConfig} args.config - Normalized config.
+ * @param {boolean} args.force - Whether to actually stop the orphans (otherwise list them).
+ * @param {(pid: number, timeoutMs: number) => Promise<boolean>} [args.stopGroup] - Stops a process group and resolves to whether it is gone afterward (defaults to the real implementation; injectable for tests).
+ * @returns {Promise<RecoverResult>} The orphans found and whether they were stopped, or an error.
+ */
+export async function recoverOrphans({config, force, stopGroup = stopProcessGroup}) {
+  if (config.statePath === undefined) {
+    return {error: "No statePath is configured; set statePath in the config to enable recovery."}
+  }
+
+  if (await daemonIsRunning(config.control.path)) {
+    return {error: `A daemon (or another process) is using ${config.control.path}; stop it before recovering — recover is for cleaning up after a crash.`}
+  }
+
+  const orphans = liveProcesses(await readState(config.statePath))
+
+  if (!force) return {cleared: false, forced: false, orphans, remaining: []}
+
+  /** @type {OrphanProcess[]} */
+  const remaining = []
+
+  for (const orphan of orphans) {
+    const stopped = await stopGroup(orphan.pid, config.proxy.forceStopTimeoutMs)
+
+    if (!stopped) remaining.push(orphan)
+  }
+
+  // Only clear the state file once every orphan is confirmed gone; otherwise keep it so the
+  // operator can still find and retry the survivors on the next run.
+  if (remaining.length === 0) await clearState(config.statePath)
+
+  return {cleared: remaining.length === 0, forced: true, orphans, remaining}
+}
+
+/**
+ * @param {string} socketPath - Control socket path.
+ * @returns {Promise<boolean>} True when a process is live on the socket (or it can't be probed).
+ */
+async function daemonIsRunning(socketPath) {
+  try {
+    return (await inspectControlSocket(socketPath)).alive
+  } catch {
+    // Can't tell — be conservative and refuse to stop processes.
+    return true
+  }
+}
+
+/**
+ * Stops a detached process group: SIGTERM, then SIGKILL if it outlives the timeout.
+ * @param {number} pid - Process-group leader pid (the detached spawn's pid).
+ * @param {number} timeoutMs - Grace period before SIGKILL.
+ * @returns {Promise<boolean>} True once the process is gone; false if it is still alive afterward (for example owned by another user, so it can't be signaled).
+ */
+async function stopProcessGroup(pid, timeoutMs) {
+  const term = sendSignal(pid, "SIGTERM")
+
+  if (term === "gone") return true
+  if (term === "denied") return false
+
+  if (await waitForExit(pid, timeoutMs)) return true
+
+  const kill = sendSignal(pid, "SIGKILL")
+
+  if (kill === "gone") return true
+  if (kill === "denied") return false
+
+  return waitForExit(pid, KILL_CONFIRM_TIMEOUT_MS)
+}
+
+/**
+ * Polls until the pid is no longer alive or the timeout elapses.
+ * @param {number} pid - Process pid to watch.
+ * @param {number} timeoutMs - How long to wait for it to exit.
+ * @returns {Promise<boolean>} True once the process is gone, false if it is still alive at the deadline.
+ */
+async function waitForExit(pid, timeoutMs) {
+  const deadline = Date.now() + timeoutMs
+
+  while (Date.now() < deadline) {
+    if (!isProcessAlive(pid)) return true
+
+    await new Promise((resolve) => setTimeout(resolve, 50))
+  }
+
+  return !isProcessAlive(pid)
+}
+
+/**
+ * Sends a signal to a detached process group, classifying the outcome.
+ * @param {number} pid - Process-group leader pid.
+ * @param {"SIGTERM" | "SIGKILL"} signal - Signal to send to the group.
+ * @returns {"denied" | "gone" | "sent"} `gone` when the group no longer exists (ESRCH), `denied` when it can't be signaled (for example EPERM), otherwise `sent`.
+ */
+function sendSignal(pid, signal) {
+  try {
+    process.kill(-pid, signal)
+
+    return "sent"
+  } catch (error) {
+    if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") return "gone"
+
+    // EPERM (owned by another user) or anything else: we could not deliver the signal.
+    return "denied"
+  }
+}