rollbridge 0.1.5 → 0.1.7

package/src/managed-process.js

@@ -2,14 +2,16 @@
 
 import {EventEmitter} from "node:events"
 import {spawn} from "node:child_process"
+import {processGroupMembers} from "./process-memory.js"
 
 /**
  * @typedef {import("./json.js").JsonValue} JsonValue
  * @typedef {"starting" | "running" | "stopping" | "stopped" | "failed"} ManagedProcessState
+ * @typedef {"deploy" | "crash" | "manual" | "memory"} ManagedProcessStartReason
  * @typedef {import("node:child_process").ChildProcess["signalCode"]} ProcessExitSignal
  * @typedef {{at: string, line: string, stream: "stdout" | "stderr"}} ManagedProcessLog
- * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopTimeoutMs: number}} ManagedProcessDefinition
- * @typedef {{command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, logs: ManagedProcessLog[], pid: number | undefined, restarts: number, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
+ * @typedef {{command: string, cwd: string | undefined, env: Record<string, string | undefined>, lifecycle: import("./config.js").LifecycleConfig, logger: (message: string, data?: Record<string, import("./json.js").JsonValue>) => void, memory: import("./config.js").MemoryConfig | undefined, outputLines: number, restart: import("./config.js").RestartConfig, restartDelayMs: number, shouldRestart: () => boolean, stopSignal: string, stopTimeoutMs: number}} ManagedProcessDefinition
+ * @typedef {{children: import("./process-memory.js").ProcessGroupMember[], command: string, cwd: string | undefined, exitCode: number | null | undefined, exitSignal: ProcessExitSignal | undefined, id: string, lastMemoryRestartAt: string | undefined, lastStartReason: ManagedProcessStartReason | undefined, logs: ManagedProcessLog[], memoryRestarts: number, pid: number | undefined, restarts: number, rssBytes: number | undefined, startedAt: string | undefined, state: ManagedProcessState, uptimeMs: number | undefined}} ManagedProcessStatus
  */
 
 export default class ManagedProcess extends EventEmitter {
@@ -20,29 +22,43 @@ export default class ManagedProcess extends EventEmitter {
    * @param {Record<string, string | undefined>} args.env - Environment.
    * @param {string} args.id - Process id.
    * @param {(message: string, data?: Record<string, JsonValue>) => void} args.logger - Logger callback.
+   * @param {import("./config.js").LifecycleConfig} [args.lifecycle] - Graceful-stop lifecycle hooks (none by default).
+   * @param {import("./config.js").MemoryConfig} [args.memory] - Memory supervision config (off when omitted).
    * @param {number} args.outputLines - Recent stdout/stderr lines to retain and report.
    * @param {import("./config.js").RestartConfig} [args.restart] - Restart policy (defaults to unlimited restarts with a constant delay).
    * @param {number} args.restartDelayMs - Restart delay.
    * @param {() => boolean} args.shouldRestart - Restart policy callback.
+   * @param {string} [args.stopSignal] - Signal sent to gracefully stop the process (default "SIGTERM").
    * @param {number} args.stopTimeoutMs - Stop timeout.
    */
-  constructor({command, cwd, env, id, logger, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, stopTimeoutMs}) {
+  constructor({command, cwd, env, id, lifecycle = {drainTimeoutMs: 0}, logger, memory, outputLines, restart = {backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0}, restartDelayMs, shouldRestart, stopSignal = "SIGTERM", stopTimeoutMs}) {
     super()
 
     this.command = command
     this.cwd = cwd
     this.env = env
     this.id = id
+    this.lifecycle = lifecycle
     this.logger = logger
+    this.memory = memory
     this.outputLines = outputLines
     this.restart = restart
     this.restartDelayMs = restartDelayMs
     this.shouldRestart = shouldRestart
+    this.stopSignal = stopSignal
     this.stopTimeoutMs = stopTimeoutMs
     this.state = /** @type {ManagedProcessState} */ ("stopped")
+    this.lastStartReason = /** @type {ManagedProcessStartReason | undefined} */ (undefined)
     this.logs = /** @type {ManagedProcessLog[]} */ ([])
     this.restarts = 0
     this.recentRestarts = /** @type {number[]} */ ([])
+    this.rssBytes = /** @type {number | undefined} */ (undefined)
+    this.children = /** @type {import("./process-memory.js").ProcessGroupMember[]} */ ([])
+    this.memoryRestarts = 0
+    this.lastMemoryRestartAtMs = /** @type {number | undefined} */ (undefined)
+    this.memoryTimer = /** @type {ReturnType<typeof setInterval> | undefined} */ (undefined)
+    this.memoryRestarting = false
+    this.memoryWarned = false
     this.startedAtMs = /** @type {number | undefined} */ (undefined)
     this.intentionalStop = false
     this.restartTimer = undefined
@@ -53,8 +69,11 @@ export default class ManagedProcess extends EventEmitter {
     this.exitSignal = undefined
   }
 
-  /** @returns {Promise<void>} Resolves after spawn. */
-  async start() {
+  /**
+   * @param {ManagedProcessStartReason} [reason] - Why the process is being started (deploy by default; "crash" on auto-restart, "manual" via the restart command).
+   * @returns {Promise<void>} Resolves after spawn.
+   */
+  async start(reason = "deploy") {
     if (this.child) return
 
     this.intentionalStop = false
@@ -83,7 +102,9 @@ export default class ManagedProcess extends EventEmitter {
       child.once("spawn", () => {
         this.state = "running"
         this.startedAtMs = Date.now()
-        this.logger("process started", {command: this.command, id: this.id, pid: child.pid || null})
+        this.lastStartReason = reason
+        this.logger("process started", {command: this.command, id: this.id, pid: child.pid || null, reason})
+        this.startMemoryMonitor()
         this.emit("started")
         resolve(undefined)
       })
@@ -107,11 +128,14 @@ export default class ManagedProcess extends EventEmitter {
     this.command = definition.command
     this.cwd = definition.cwd
     this.env = definition.env
+    this.lifecycle = definition.lifecycle
     this.logger = definition.logger
+    this.memory = definition.memory
     this.outputLines = definition.outputLines
     this.restart = definition.restart
     this.restartDelayMs = definition.restartDelayMs
     this.shouldRestart = definition.shouldRestart
+    this.stopSignal = definition.stopSignal
     this.stopTimeoutMs = definition.stopTimeoutMs
   }
 
@@ -145,6 +169,9 @@ export default class ManagedProcess extends EventEmitter {
     this.child = undefined
     this.pid = undefined
     this.exitPromise = undefined
+    this.rssBytes = undefined
+    this.children = []
+    this.clearMemoryMonitor()
     this.state = wasIntentional ? "stopped" : "failed"
     this.logger("process exited", {code, id: this.id, signal})
     this.emit("exit", {code, signal})
@@ -206,10 +233,103 @@ export default class ManagedProcess extends EventEmitter {
     this.restartTimer = setTimeout(() => {
       this.restartTimer = undefined
       this.restarts += 1
-      this.start().catch((error) => {
+      this.start("crash").catch((error) => {
         this.logger("process restart failed", {error: error instanceof Error ? error.message : String(error), id: this.id})
       })
     }, delayMs)
+
+    // The daemon's listening servers govern its lifetime; a pending restart must never be the sole
+    // handle keeping the process alive (like the memory and persist timers above). Otherwise a
+    // crashed process with an unlimited restart policy would respawn forever and block exit.
+    this.restartTimer.unref?.()
+  }
+
+  /**
+   * Starts the periodic RSS check for this process when memory supervision is configured.
+   * @returns {void}
+   */
+  startMemoryMonitor() {
+    this.clearMemoryMonitor()
+
+    if (!this.memory) return
+
+    this.memoryTimer = setInterval(() => this.checkMemory(), this.memory.checkIntervalMs)
+    this.memoryTimer.unref?.()
+  }
+
+  /** @returns {void} Stops the periodic RSS check. */
+  clearMemoryMonitor() {
+    if (this.memoryTimer) {
+      clearInterval(this.memoryTimer)
+      this.memoryTimer = undefined
+    }
+  }
+
+  /**
+   * Measures the process group's RSS and warns or restarts when it crosses the configured thresholds.
+   * @returns {void}
+   */
+  checkMemory() {
+    if (!this.memory || !this.pid || this.memoryRestarting) return
+
+    const members = processGroupMembers(this.pid)
+
+    if (members.length === 0) return
+
+    this.children = members
+
+    const measured = members.filter((member) => member.rssBytes !== undefined)
+
+    if (measured.length === 0) return
+
+    const rssBytes = measured.reduce((total, member) => total + (member.rssBytes ?? 0), 0)
+
+    this.rssBytes = rssBytes
+
+    if (rssBytes > this.memory.limitBytes) {
+      this.logger("memory limit exceeded", {id: this.id, limitBytes: this.memory.limitBytes, rssBytes})
+      void this.restartForMemory()
+
+      return
+    }
+
+    if (this.memory.warnBytes > 0 && rssBytes > this.memory.warnBytes) {
+      if (!this.memoryWarned) {
+        this.logger("memory warning", {id: this.id, rssBytes, warnBytes: this.memory.warnBytes})
+        this.memoryWarned = true
+      }
+    } else {
+      this.memoryWarned = false
+    }
+  }
+
+  /**
+   * Gracefully restarts the process after it exceeded its memory limit (SIGTERM, then
+   * SIGKILL after the stop timeout), recording the restart so status can report it.
+   * @returns {Promise<void>} Resolves once the process has been restarted.
+   */
+  async restartForMemory() {
+    if (this.memoryRestarting) return
+
+    this.memoryRestarting = true
+
+    try {
+      await this.stop()
+
+      // Don't respawn if the supervising context no longer wants this process running
+      // (daemon shutting down, or the release draining/retired) — otherwise a restart racing
+      // a shutdown could leave a child running after shutdown collected its stop promises.
+      if (!this.shouldRestart()) return
+
+      this.memoryRestarts += 1
+      this.lastMemoryRestartAtMs = Date.now()
+      this.memoryWarned = false
+      await this.start("memory")
+    } catch (error) {
+      this.logger("memory restart failed", {error: error instanceof Error ? error.message : String(error), id: this.id})
+    } finally {
+      this.memoryRestarting = false
+    }
   }
 
   /**
@@ -218,6 +338,7 @@ export default class ManagedProcess extends EventEmitter {
    */
   async stop(options = {}) {
     this.intentionalStop = true
+    this.clearMemoryMonitor()
 
     if (this.restartTimer) {
       clearTimeout(this.restartTimer)
@@ -232,21 +353,104 @@ export default class ManagedProcess extends EventEmitter {
     }
 
     this.state = "stopping"
-    this.killProcessGroup("SIGTERM")
-    const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
-    const stopped = await this.waitForExit(timeoutMs)
-
-    if (!stopped) {
-      this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: child.pid})
-      this.killProcessGroup("SIGKILL")
-      await this.waitForExit(5000)
+
+    const {drainCommand, drainTimeoutMs, quietCommand, stopCommand} = this.lifecycle
+
+    // 1. Quiesce: tell the process to stop accepting new work.
+    if (quietCommand) await this.runHook(quietCommand, this.stopTimeoutMs, "quiet command")
+
+    // 2. Drain: let in-flight work finish, bounded by drainTimeoutMs (0 skips the step). A
+    //    drainCommand blocks until drained; otherwise wait for the process to exit on its own.
+    if (this.child && drainTimeoutMs > 0) {
+      if (drainCommand) await this.runHook(drainCommand, drainTimeoutMs, "drain command")
+      else await this.waitForExit(drainTimeoutMs)
+    }
+
+    // 3. Stop whatever is still running, then SIGKILL if it outlasts the graceful window.
+    if (this.child) {
+      if (stopCommand) await this.runHook(stopCommand, this.stopTimeoutMs, "stop command")
+      else this.killProcessGroup(this.stopSignal)
+
+      const timeoutMs = options.timeoutMs ?? this.stopTimeoutMs
+
+      if (this.child && !(await this.waitForExit(timeoutMs))) {
+        this.logger("process stop timed out; sending SIGKILL", {id: this.id, pid: this.pid})
+        this.killProcessGroup("SIGKILL")
+        await this.waitForExit(5000)
+      }
     }
 
     this.state = "stopped"
   }
 
   /**
-   * @param {"SIGTERM" | "SIGKILL"} signal - Signal to send.
+   * Runs a lifecycle hook command, bounded by a timeout so a hung hook can never block stop().
+   * Failures are logged and swallowed — the graceful-stop sequence proceeds (and SIGKILL is the
+   * ultimate fallback) regardless of the hook's outcome.
+   * @param {string} command - Shell command to run.
+   * @param {number} timeoutMs - Maximum time to wait for the hook before killing it.
+   * @param {string} label - Hook name, for log messages.
+   * @returns {Promise<void>} Resolves when the hook exits, errors, or times out.
+   */
+  async runHook(command, timeoutMs, label) {
+    await new Promise((resolve) => {
+      let settled = false
+      const finish = () => { if (!settled) { settled = true; resolve(undefined) } }
+
+      /** @type {import("node:child_process").ChildProcess} */
+      let hook
+
+      try {
+        hook = spawn(command, {
+          cwd: this.cwd,
+          detached: true,
+          env: {...process.env, ...this.env, ROLLBRIDGE_PID: this.pid ? String(this.pid) : ""},
+          shell: true,
+          stdio: "ignore"
+        })
+      } catch (error) {
+        this.logger(`${label} failed`, {error: error instanceof Error ? error.message : String(error), id: this.id})
+        finish()
+
+        return
+      }
+
+      const timer = setTimeout(() => {
+        this.logger(`${label} timed out`, {id: this.id, timeoutMs})
+
+        if (hook.pid) {
+          try {
+            process.kill(-hook.pid, "SIGKILL")
+          } catch {
+            // The hook already exited.
+          }
+        }
+
+        finish()
+      }, timeoutMs)
+
+      hook.once("exit", (code, signal) => {
+        clearTimeout(timer)
+
+        // A non-zero/signalled exit is surfaced (but still non-fatal); skip when the timeout
+        // already killed the hook, which logs separately.
+        if (!settled) {
+          if (typeof code === "number" && code !== 0) this.logger(`${label} exited non-zero`, {code, id: this.id})
+          else if (signal) this.logger(`${label} exited on signal`, {id: this.id, signal})
+        }
+
+        finish()
+      })
+      hook.once("error", (error) => {
+        clearTimeout(timer)
+        this.logger(`${label} failed`, {error: error instanceof Error ? error.message : String(error), id: this.id})
+        finish()
+      })
+    })
+  }
+
+  /**
+   * @param {string} signal - Signal name to send (the configured stop signal, or "SIGKILL").
    * @returns {void}
    */
   killProcessGroup(signal) {
@@ -282,14 +486,19 @@ export default class ManagedProcess extends EventEmitter {
   /** @returns {ManagedProcessStatus} Status payload. */
   status() {
     return {
+      children: this.children,
       command: this.command,
       cwd: this.cwd,
       exitCode: this.exitCode,
       exitSignal: this.exitSignal,
       id: this.id,
+      lastMemoryRestartAt: this.lastMemoryRestartAtMs === undefined ? undefined : new Date(this.lastMemoryRestartAtMs).toISOString(),
+      lastStartReason: this.lastStartReason,
       logs: this.logs.slice(-this.outputLines),
+      memoryRestarts: this.memoryRestarts,
       pid: this.pid,
       restarts: this.restarts,
+      rssBytes: this.rssBytes,
       startedAt: this.startedAtMs === undefined ? undefined : new Date(this.startedAtMs).toISOString(),
       state: this.state,
       uptimeMs: this.state === "running" && this.startedAtMs !== undefined ? Date.now() - this.startedAtMs : undefined