rollbridge 0.1.5 → 0.1.6

package/test/managed-process.test.js

@@ -1,6 +1,8 @@
 // @ts-check
 
 import assert from "node:assert/strict"
+import fs from "node:fs"
+import os from "node:os"
 import path from "node:path"
 import test from "node:test"
 import {fileURLToPath} from "node:url"
@@ -105,6 +107,32 @@ test("counts automatic restarts and reports startedAt and uptime while running",
   }
 })
 
+test("a queued auto-restart timer is unref'd so it can't keep the process alive", async () => {
+  const managed = new ManagedProcess({
+    command: `${JSON.stringify(process.execPath)} ${JSON.stringify(crasherPath)}`,
+    cwd: undefined,
+    env: {},
+    id: "crasher",
+    logger: () => {},
+    outputLines: 50,
+    restartDelayMs: 5000,
+    shouldRestart: () => true,
+    stopTimeoutMs: 500
+  })
+
+  try {
+    await managed.start()
+
+    // After the fixture crashes a restart is queued. Under the default unlimited restart policy a
+    // ref'd timer would respawn forever and block process exit, so the queued timer must be unref'd.
+    await waitFor(() => managed.restartTimer !== undefined)
+
+    assert.equal(managed.restartTimer?.hasRef(), false)
+  } finally {
+    await managed.stop()
+  }
+})
+
 /**
  * Builds a managed crasher with a specific restart policy.
  * @param {import("../src/config.js").RestartConfig} restart - Restart policy.
@@ -125,6 +153,268 @@ function buildCrasher(restart) {
   })
 }
 
+test("records the start reason, marking crash auto-restarts", async () => {
+  const managed = buildCrasher({backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0})
+
+  try {
+    await managed.start()
+
+    assert.equal(managed.status().lastStartReason, "deploy")
+
+    // The fixture crashes ~40ms after each start, so it auto-restarts with reason "crash".
+    await waitFor(() => managed.status().restarts >= 1)
+
+    assert.equal(managed.status().lastStartReason, "crash")
+  } finally {
+    await managed.stop()
+  }
+})
+
+test("records the manual start reason", async () => {
+  // Restarts disabled, so the crash does not overwrite the manual reason.
+  const managed = buildCrasher({backoffFactor: 1, maxDelayMs: 0, maxRestarts: 0, windowMs: 0})
+
+  try {
+    await managed.start("manual")
+
+    assert.equal(managed.status().lastStartReason, "manual")
+  } finally {
+    await managed.stop()
+  }
+})
+
+test("does not record a start reason when the spawn fails", async () => {
+  const managed = new ManagedProcess({
+    command: "true",
+    cwd: "/nonexistent-rollbridge-spawn-dir",
+    env: {},
+    id: "broken",
+    logger: () => {},
+    outputLines: 50,
+    restartDelayMs: 10,
+    shouldRestart: () => false,
+    stopTimeoutMs: 500
+  })
+
+  // The cwd does not exist, so the spawn fails before the process ever runs.
+  await assert.rejects(() => managed.start("manual"))
+  assert.equal(managed.status().lastStartReason, undefined)
+})
+
+/**
+ * Builds a long-lived managed process (stays running until stopped) with a restart gate.
+ * @param {() => boolean} shouldRestart - Restart policy callback.
+ * @returns {ManagedProcess} Managed process.
+ */
+function buildLongLived(shouldRestart) {
+  return new ManagedProcess({
+    command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
+    cwd: undefined,
+    env: {},
+    id: "worker",
+    logger: () => {},
+    outputLines: 50,
+    restartDelayMs: 10,
+    shouldRestart,
+    stopTimeoutMs: 500
+  })
+}
+
+test("runs quiet and drain lifecycle hooks before stopping", async () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-hooks-"))
+  const logPath = path.join(dir, "hooks.log")
+  const append = (/** @type {string} */ word) => `${JSON.stringify("/bin/sh")} -c ${JSON.stringify(`echo ${word} >> ${logPath}`)}`
+  const managed = new ManagedProcess({
+    command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
+    cwd: undefined,
+    env: {},
+    id: "worker",
+    lifecycle: {drainCommand: append("drain"), drainTimeoutMs: 500, quietCommand: append("quiet")},
+    logger: () => {},
+    outputLines: 50,
+    restartDelayMs: 10,
+    shouldRestart: () => false,
+    stopSignal: "SIGTERM",
+    stopTimeoutMs: 2000
+  })
+
+  try {
+    await managed.start()
+    await managed.stop()
+
+    assert.equal(managed.status().state, "stopped")
+    // quietCommand ran, then drainCommand, then the worker was stopped via stopSignal.
+    assert.deepEqual(fs.readFileSync(logPath, "utf8").trim().split("\n"), ["quiet", "drain"])
+  } finally {
+    await managed.stop()
+    fs.rmSync(dir, {force: true, recursive: true})
+  }
+})
+
+test("a configured stopCommand is used instead of the stop signal", async () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-hooks-"))
+  const logPath = path.join(dir, "stop.log")
+  const managed = new ManagedProcess({
+    command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
+    cwd: undefined,
+    env: {},
+    id: "worker",
+    // The stop command logs and kills the worker's process group, so no SIGKILL fallback is needed.
+    lifecycle: {drainTimeoutMs: 0, stopCommand: `${JSON.stringify("/bin/sh")} -c ${JSON.stringify(`echo stop >> ${logPath}; kill -KILL -$ROLLBRIDGE_PID`)}`},
+    logger: () => {},
+    outputLines: 50,
+    restartDelayMs: 10,
+    shouldRestart: () => false,
+    stopSignal: "SIGTERM",
+    stopTimeoutMs: 2000
+  })
+
+  /** @type {string[]} */
+  const signals = []
+  const killProcessGroup = managed.killProcessGroup.bind(managed)
+
+  managed.killProcessGroup = (signal) => {
+    signals.push(signal)
+    killProcessGroup(signal)
+  }
+
+  try {
+    await managed.start()
+    await managed.stop()
+
+    assert.equal(managed.status().state, "stopped")
+    assert.deepEqual(fs.readFileSync(logPath, "utf8").trim().split("\n"), ["stop"])
+    // The stop signal is replaced by the stop command (only a SIGKILL fallback may be sent).
+    assert.ok(!signals.includes("SIGTERM"), `expected no stopSignal, got ${signals.join(",")}`)
+  } finally {
+    await managed.stop()
+    fs.rmSync(dir, {force: true, recursive: true})
+  }
+})
+
+test("a failing lifecycle hook is logged but does not fail the stop", async () => {
+  /** @type {string[]} */
+  const messages = []
+  const managed = new ManagedProcess({
+    command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
+    cwd: undefined,
+    env: {},
+    id: "worker",
+    lifecycle: {drainTimeoutMs: 0, quietCommand: `${JSON.stringify("/bin/sh")} -c "exit 3"`},
+    logger: (message) => { messages.push(message) },
+    outputLines: 50,
+    restartDelayMs: 10,
+    shouldRestart: () => false,
+    stopSignal: "SIGTERM",
+    stopTimeoutMs: 2000
+  })
+
+  try {
+    await managed.start()
+    await managed.stop()
+
+    assert.equal(managed.status().state, "stopped")
+    assert.ok(messages.includes("quiet command exited non-zero"), `expected a non-zero hook log, got ${messages.join(",")}`)
+  } finally {
+    await managed.stop()
+  }
+})
+
+test("a hanging lifecycle hook is bounded so stop still completes", async () => {
+  const managed = new ManagedProcess({
+    command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
+    cwd: undefined,
+    env: {},
+    id: "worker",
+    lifecycle: {drainTimeoutMs: 0, quietCommand: "sleep 30"},
+    logger: () => {},
+    outputLines: 50,
+    restartDelayMs: 10,
+    shouldRestart: () => false,
+    stopSignal: "SIGTERM",
+    stopTimeoutMs: 300
+  })
+
+  try {
+    await managed.start()
+
+    const startedAt = Date.now()
+
+    await managed.stop()
+
+    assert.equal(managed.status().state, "stopped")
+    // The hung quietCommand is killed at stopTimeoutMs rather than blocking stop indefinitely.
+    assert.ok(Date.now() - startedAt < 5000, "stop should not wait for the hung hook")
+  } finally {
+    await managed.stop()
+  }
+})
+
+test("sends the configured stopSignal as the graceful stop signal", async () => {
+  const managed = new ManagedProcess({
+    command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`,
+    cwd: undefined,
+    env: {},
+    id: "worker",
+    logger: () => {},
+    outputLines: 50,
+    restartDelayMs: 10,
+    shouldRestart: () => false,
+    stopSignal: "SIGINT",
+    stopTimeoutMs: 500
+  })
+
+  /** @type {string[]} */
+  const signals = []
+  const killProcessGroup = managed.killProcessGroup.bind(managed)
+
+  managed.killProcessGroup = (signal) => {
+    signals.push(signal)
+    killProcessGroup(signal)
+  }
+
+  await managed.start()
+  await managed.stop()
+
+  // The graceful stop uses the configured signal (a SIGKILL fallback, if any, comes after).
+  assert.equal(signals[0], "SIGINT")
+  assert.equal(managed.status().state, "stopped")
+})
+
+test("a memory restart respawns and is counted when the supervisor still wants the process", async () => {
+  const managed = buildLongLived(() => true)
+
+  try {
+    await managed.start()
+    await managed.restartForMemory()
+
+    assert.equal(managed.status().state, "running")
+    assert.equal(managed.memoryRestarts, 1)
+    assert.equal(managed.status().lastStartReason, "memory")
+  } finally {
+    await managed.stop()
+  }
+})
+
+test("a memory restart does not respawn when shouldRestart is false", async () => {
+  let allowRestart = true
+  const managed = buildLongLived(() => allowRestart)
+
+  try {
+    await managed.start()
+    assert.equal(managed.status().state, "running")
+
+    // The supervisor (e.g. daemon shutdown or a draining release) no longer wants it running.
+    allowRestart = false
+    await managed.restartForMemory()
+
+    assert.equal(managed.status().state, "stopped")
+    assert.equal(managed.memoryRestarts, 0)
+  } finally {
+    await managed.stop()
+  }
+})
+
 test("does not auto-restart when the restart policy is disabled (maxRestarts: 0)", async () => {
   const managed = buildCrasher({backoffFactor: 1, maxDelayMs: 0, maxRestarts: 0, windowMs: 0})
 

package/test/process-memory.test.js

@@ -0,0 +1,40 @@
+// @ts-check
+
+import assert from "node:assert/strict"
+import fs from "node:fs"
+import test from "node:test"
+import {measureProcessGroupRssBytes, processGroupMembers} from "../src/process-memory.js"
+
+const linuxOnly = process.platform !== "linux" && "requires /proc (Linux)"
+
+/**
+ * @returns {number} The current process's group id, read from /proc.
+ */
+function currentProcessGroupId() {
+  const stat = fs.readFileSync("/proc/self/stat", "utf8")
+
+  return Number(stat.slice(stat.lastIndexOf(")") + 2).split(" ")[2])
+}
+
+test("measures the resident memory of a live process group", {skip: linuxOnly}, () => {
+  const rssBytes = measureProcessGroupRssBytes(currentProcessGroupId())
+
+  assert.ok(typeof rssBytes === "number" && rssBytes > 0, `expected a positive RSS, got ${rssBytes}`)
+})
+
+test("returns undefined for a process group with no members", {skip: linuxOnly}, () => {
+  assert.equal(measureProcessGroupRssBytes(2147483646), undefined)
+})
+
+test("lists process-group members with their command and resident memory", {skip: linuxOnly}, () => {
+  const members = processGroupMembers(currentProcessGroupId())
+  const self = members.find((member) => member.pid === process.pid)
+
+  assert.ok(self, "the current process should be a group member")
+  assert.ok(typeof self.rssBytes === "number" && self.rssBytes > 0)
+  assert.equal(typeof self.command, "string")
+})
+
+test("returns an empty list for a process group with no members", {skip: linuxOnly}, () => {
+  assert.deepEqual(processGroupMembers(2147483646), [])
+})

package/test/recover.test.js

@@ -0,0 +1,162 @@
+// @ts-check
+
+import assert from "node:assert/strict"
+import {spawn} from "node:child_process"
+import {once} from "node:events"
+import fs from "node:fs/promises"
+import os from "node:os"
+import path from "node:path"
+import test from "node:test"
+import RollbridgeDaemon from "../src/daemon.js"
+import {normalizeConfig} from "../src/config.js"
+import {recoverOrphans} from "../src/recover.js"
+import {isProcessAlive, readState, writeState} from "../src/state-store.js"
+
+/**
+ * @param {string} dir - Working directory.
+ * @param {{statePath?: string}} [options] - Config options.
+ * @returns {import("../src/config.js").RollbridgeConfig} Normalized config.
+ */
+function buildConfig(dir, {statePath} = {}) {
+  return normalizeConfig({
+    application: "recover-test",
+    control: {path: path.join(dir, "rollbridge.sock")},
+    processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
+    proxy: {forceStopTimeoutMs: 1000, host: "127.0.0.1", port: 0},
+    ...(statePath ? {statePath} : {})
+  })
+}
+
+/**
+ * @returns {Promise<import("node:child_process").ChildProcess>} A detached, long-lived process (its own group leader).
+ */
+async function spawnOrphan() {
+  const orphan = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {detached: true, stdio: "ignore"})
+
+  await once(orphan, "spawn")
+
+  return orphan
+}
+
+/**
+ * @param {string} statePath - State file path.
+ * @param {number | undefined} pid - Orphan pid to record.
+ * @returns {Promise<void>} Resolves once written.
+ */
+async function writeOrphanState(statePath, pid) {
+  await writeState(statePath, {activeReleaseId: "v1", releases: [{processes: [{id: "worker", pid}], releaseId: "v1"}], services: [], singletons: []})
+}
+
+/**
+ * @param {() => boolean} probe - Condition to await.
+ * @returns {Promise<void>} Resolves once the probe returns true.
+ */
+async function waitFor(probe) {
+  const deadline = Date.now() + 3000
+
+  while (Date.now() < deadline) {
+    if (probe()) return
+    await new Promise((resolve) => setTimeout(resolve, 25))
+  }
+
+  throw new Error("Timed out waiting for condition")
+}
+
+test("recover requires a configured statePath", async () => {
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-recover-"))
+
+  try {
+    const result = await recoverOrphans({config: buildConfig(dir), force: true})
+
+    assert.ok("error" in result && /statePath/.test(result.error))
+  } finally {
+    await fs.rm(dir, {force: true, recursive: true})
+  }
+})
+
+test("recover lists orphans without stopping them unless forced", async () => {
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-recover-"))
+  const statePath = path.join(dir, "state.json")
+  const orphan = await spawnOrphan()
+
+  try {
+    await writeOrphanState(statePath, orphan.pid)
+
+    const result = await recoverOrphans({config: buildConfig(dir, {statePath}), force: false})
+
+    assert.ok(!("error" in result))
+    assert.equal(result.forced, false)
+    assert.equal(result.cleared, false)
+    assert.deepEqual(result.remaining, [])
+    assert.equal(result.orphans.length, 1)
+    assert.equal(result.orphans[0].pid, orphan.pid)
+    assert.ok(orphan.pid !== undefined && isProcessAlive(orphan.pid), "the orphan must not be stopped by a dry run")
+    assert.ok(await readState(statePath), "a dry run must not clear the state file")
+  } finally {
+    orphan.kill("SIGKILL")
+    await fs.rm(dir, {force: true, recursive: true})
+  }
+})
+
+test("recover --force stops orphan process groups and clears the state file", async () => {
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-recover-"))
+  const statePath = path.join(dir, "state.json")
+  const orphan = await spawnOrphan()
+
+  try {
+    await writeOrphanState(statePath, orphan.pid)
+
+    const result = await recoverOrphans({config: buildConfig(dir, {statePath}), force: true})
+
+    assert.ok(!("error" in result))
+    assert.equal(result.forced, true)
+    assert.equal(result.cleared, true)
+    assert.deepEqual(result.remaining, [])
+    await waitFor(() => orphan.pid === undefined || !isProcessAlive(orphan.pid))
+    assert.equal(await readState(statePath), undefined, "the state file is cleared after a forced recovery")
+  } finally {
+    orphan.kill("SIGKILL")
+    await fs.rm(dir, {force: true, recursive: true})
+  }
+})
+
+test("recover --force keeps the state file when an orphan cannot be stopped", async () => {
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-recover-"))
+  const statePath = path.join(dir, "state.json")
+  const orphan = await spawnOrphan()
+
+  try {
+    await writeOrphanState(statePath, orphan.pid)
+
+    // Simulate an orphan that can't be signaled (for example owned by another user): stopGroup
+    // reports it is still alive.
+    const result = await recoverOrphans({config: buildConfig(dir, {statePath}), force: true, stopGroup: async () => false})
+
+    assert.ok(!("error" in result))
+    assert.equal(result.forced, true)
+    assert.equal(result.cleared, false, "the state file is kept when an orphan survives")
+    assert.equal(result.remaining.length, 1)
+    assert.equal(result.remaining[0].pid, orphan.pid)
+    assert.ok(await readState(statePath), "the state file must remain so the operator can retry")
+  } finally {
+    orphan.kill("SIGKILL")
+    await fs.rm(dir, {force: true, recursive: true})
+  }
+})
+
+test("recover refuses while a daemon is running", async () => {
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-recover-"))
+  const config = buildConfig(dir, {statePath: path.join(dir, "state.json")})
+  const daemon = new RollbridgeDaemon({config, logger: () => {}})
+
+  await daemon.start()
+
+  try {
+    const result = await recoverOrphans({config, force: true})
+
+    assert.ok("error" in result && /is using/.test(result.error))
+  } finally {
+    await daemon.shutdown()
+    await fs.rm(dir, {force: true, recursive: true})
+  }
+})

package/test/release-group.test.js

@@ -41,6 +41,28 @@ test("templates interpolate values from the daemon environment", () => {
   }
 })
 
+test("replica processes get a replica index, count, and template context", () => {
+  const config = normalizeConfig({
+    application: "demo",
+    control: {path: "/tmp/rollbridge-release-group.sock"},
+    processes: [
+      {command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}},
+      {command: "worker {{replicaIndex}}/{{replicaCount}}", env: {SLOT: "{{replicaIndex}}"}, id: "worker", policy: "companion", replicas: 3}
+    ],
+    proxy: {host: "127.0.0.1", port: 0}
+  })
+  const release = new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1"})
+  const workerConfig = release.config.processes[1]
+  const replica = release.buildProcess(workerConfig, {count: 3, index: 1, instanceId: "worker#1"})
+
+  assert.equal(replica.id, "worker#1")
+  assert.equal(replica.command, "worker 1/3")
+  assert.equal(replica.env.ROLLBRIDGE_REPLICA_INDEX, "1")
+  assert.equal(replica.env.ROLLBRIDGE_REPLICA_COUNT, "3")
+  assert.equal(replica.env.ROLLBRIDGE_PROCESS_ID, "worker")
+  assert.equal(replica.env.SLOT, "1")
+})
+
 test("a referenced daemon environment variable that is unset fails fast", () => {
   const release = buildRelease({
     command: "run {{env.ROLLBRIDGE_ENV_MISSING}}",