rollbridge 0.1.5 → 0.1.7

package/docs/workers.md

@@ -0,0 +1,115 @@
+# Background-job worker deployment
+
+This guide covers deploying background-job workers (or any non-HTTP worker pool)
+with Rollbridge so that in-flight jobs finish across a deploy. It uses features
+that exist today; the command-based lifecycle hooks mentioned at the end are
+still on the roadmap.
+
+## Run workers as a `companion`
+
+Give each worker the `companion` policy. Companions are **release-scoped**: every
+release starts its own workers running that release's code, and a release's
+workers are stopped only when that release is retired (drained) after a newer
+release takes over. They start **before** the `proxied` web process, so they're
+ready before traffic switches.
+
+```js
+{
+  id: "worker",
+  policy: "companion",
+  cwd: "{{releasePath}}",
+  command: "npx velocious background-jobs-worker"
+}
+```
+
+## Scale the pool with `replicas`
+
+Set `replicas` to run several identical workers (a port-less companion only).
+Each instance runs as `worker#0`, `worker#1`, … and gets
+`ROLLBRIDGE_REPLICA_INDEX` / `ROLLBRIDGE_REPLICA_COUNT` (and `{{replicaIndex}}` /
+`{{replicaCount}}`), so an instance can claim a distinct shard, queue, or lock:
+
+```js
+{id: "worker", policy: "companion", command: "npx velocious background-jobs-worker", replicas: 4}
+```
+
+Restart the pool with `rollbridge restart --process worker` (all replicas) or a
+single instance with `rollbridge restart --process worker#0`.
+
+## Finish in-flight jobs on stop (`stopSignal` + `gracefulStopMs`)
+
+When Rollbridge stops a worker — during a deploy's drain, a `rollbridge restart`,
+or shutdown — it sends the worker's **`stopSignal`** (default `SIGTERM`), waits up
+to **`gracefulStopMs`**, then `SIGKILL`s it if it hasn't exited. That window is
+the worker's chance to finish its current job and exit cleanly.
+
+- Set `stopSignal` to the signal your worker quiets/drains on. Many job runners
+  finish the current job and exit on `SIGTERM` (the default); some use `SIGINT`
+  or `SIGQUIT`. Use the one your worker treats as "drain and exit".
+- Set `gracefulStopMs` to at least your longest job's duration, so a job in
+  progress is not cut off by the `SIGKILL` fallback.
+
+```js
+{
+  id: "worker",
+  policy: "companion",
+  command: "npx velocious background-jobs-worker",
+  replicas: 4,
+  stopSignal: "SIGTERM",
+  gracefulStopMs: 60000
+}
+```
+
+## What happens across a deploy
+
+1. The new release's workers start (running the **new** code) before traffic
+   switches to the new web process.
+2. Both old and new workers run while the previous release drains, so **both
+   code versions consume the shared queue at once.** Keep job code
+   backwards-compatible across a deploy — the same rule as database migrations.
+3. When the previous release is retired (its HTTP/WebSocket connections close or
+   `proxy.drainTimeoutMs` elapses), its workers are stopped: `stopSignal`, then
+   `SIGKILL` after `gracefulStopMs`.
+
+Because old workers are retired on the release's **connection** drain (not on
+their own job queue draining), a job still running when the release is retired
+gets only the `gracefulStopMs` window to finish. Keep jobs **idempotent and
+safe to retry** so a job interrupted at the `SIGKILL` fallback can run again.
+
+## Command-based lifecycle hooks
+
+For workers that quiesce or drain via a command rather than a single signal, set
+a `lifecycle` block. When Rollbridge gracefully stops the worker it runs
+`quietCommand` (stop accepting new work), then drains (`drainCommand`, or waits up
+to `drainTimeoutMs` for the worker to exit), then `stopCommand` or `stopSignal`,
+then `SIGKILL` after `gracefulStopMs`. Each hook gets `ROLLBRIDGE_PID` and is
+bounded by a timeout, so a slow hook can't wedge a deploy.
+
+```js
+{
+  id: "worker",
+  policy: "companion",
+  command: "npx velocious background-jobs-worker",
+  replicas: 4,
+  lifecycle: {quietCommand: "kill -TSTP -$ROLLBRIDGE_PID", drainTimeoutMs: 60000}
+}
+```
+
+See [`docs/config.md`](config.md#processeslifecycle) for the hook reference.
+
+## Non-blocking drain
+
+By default a retired release's workers are stopped only after the proxied
+process's connections have drained. Set `nonBlockingDrain: true` on a worker
+companion whose work is independent of the web process (a job worker on a shared
+queue) to start its graceful stop **immediately** when the release is retired —
+in parallel with the connection drain. The new release's workers handle new work
+while the old workers finish their in-flight jobs:
+
+```js
+{id: "worker", policy: "companion", command: "…", nonBlockingDrain: true, gracefulStopMs: 60000}
+```
+
+See [`docs/config.md`](config.md) for `stopSignal`, `replicas`, and
+`gracefulStopMs`, and [`docs/velocious.md`](velocious.md) for a full Velocious
+deployment (Beacon, jobs-main, workers, web) example.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rollbridge",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "Zero-downtime process supervisor and local traffic switcher for deploy-managed apps.",
   "keywords": [
     "deploy",

package/src/cli.js

@@ -7,7 +7,9 @@ import {spawn} from "node:child_process"
 import {Command} from "commander"
 import RollbridgeDaemon from "./daemon.js"
 import {loadConfig, parseConfigFile, resolveConfigPath, validateConfig} from "./config.js"
-import {runEnvironmentChecks} from "./doctor.js"
+import {runEnvironmentChecks, runReleaseChecks} from "./doctor.js"
+import {predeployCleanup} from "./predeploy-cleanup.js"
+import {recoverOrphans} from "./recover.js"
 import {sendControlCommand} from "./control-client.js"
 
 const DEFAULT_DAEMON_START_TIMEOUT_MS = 10000
@@ -82,6 +84,25 @@ export async function runCli(argv) {
       console.log(JSON.stringify(response, null, 2))
     })
 
+  program
+    .command("rollback")
+    .description("Roll back to a previous release: re-start it, health-check it, and switch traffic.")
+    .option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
+    .option("--release-id <id>", "Release id to roll back to (defaults to the most recently retired release)")
+    .action(async (options) => {
+      const configPath = await resolveConfigPath(options.config)
+      const config = await loadConfig(configPath)
+      const response = await sendControlCommand({
+        command: {
+          command: "rollback",
+          releaseId: options.releaseId
+        },
+        path: config.control.path
+      })
+
+      console.log(JSON.stringify(response, null, 2))
+    })
+
   program
     .command("ensure-daemon")
     .description("Start the daemon if the control socket is not already accepting commands.")
@@ -228,6 +249,9 @@ export async function runCli(argv) {
     .command("doctor")
     .description("Check the environment before starting the daemon: config, control socket, and proxy port.")
     .option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
+    .option("--release-path <path>", "Also pre-flight a prepared release: render its templates and check the release and working directories")
+    .option("--release-id <id>", "Release id used when rendering templates (defaults to --revision or the release path basename)")
+    .option("--revision <sha>", "Revision used when rendering templates (defaults to --release-id)")
     .option("--json", "Output machine-readable JSON")
     .action(async (options) => {
       let configPath
@@ -252,6 +276,10 @@ export async function runCli(argv) {
       } else {
         checks.push({detail: `valid: ${config.processes.length} ${config.processes.length === 1 ? "process" : "processes"}, proxy on ${config.proxy.host}:${config.proxy.port}`, name: "config", ok: true})
         checks.push(...await runEnvironmentChecks(config))
+
+        if (options.releasePath) {
+          checks.push(...await runReleaseChecks(config, {releaseId: options.releaseId, releasePath: options.releasePath, revision: options.revision}))
+        }
       }
 
       const failed = checks.filter((check) => !check.ok).length
@@ -295,9 +323,307 @@ export async function runCli(argv) {
       console.log(formatLogSources(sources, options.process))
     })
 
+  program
+    .command("events")
+    .description("Print recent structured daemon events (deploys, switches, stops, crashes, restarts, failures).")
+    .option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
+    .option("--limit <count>", "Show only the most recent <count> events")
+    .option("--json", "Output machine-readable JSON")
+    .action(async (options) => {
+      let limit
+
+      if (options.limit !== undefined) {
+        limit = Number(options.limit)
+
+        if (!Number.isInteger(limit) || limit < 1) {
+          console.error("--limit must be a positive integer.")
+          process.exitCode = 1
+          return
+        }
+      }
+
+      const configPath = await resolveConfigPath(options.config)
+      const config = await loadConfig(configPath)
+      const response = await sendControlCommand({
+        command: {command: "events", limit},
+        path: config.control.path
+      })
+      const events = /** @type {import("./event-log.js").DaemonEvent[]} */ (response.events ?? [])
+
+      if (options.json) {
+        console.log(JSON.stringify(events, null, 2))
+        return
+      }
+
+      console.log(formatEvents(events))
+    })
+
+  program
+    .command("predeploy-cleanup")
+    .description("Prepare a host for deploy: recover Rollbridge orphans and stop configured legacy processes when no release is active.")
+    .option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
+    .option("--release-path <path>", "Pending release path; restarts the daemon if this release changes Rollbridge itself")
+    .action(async (options) => {
+      const configPath = await resolveConfigPath(options.config)
+      const config = await loadConfig(configPath)
+      const result = await predeployCleanup({config, releasePath: options.releasePath})
+
+      console.log(formatPredeployCleanupResult(result))
+    })
+
+  program
+    .command("recover")
+    .description("Stop orphaned processes left by a crashed daemon (reads statePath; lists them unless --force).")
+    .option("-c, --config <path>", "Config file path (defaults to rollbridge.js)")
+    .option("--force", "Stop the orphaned processes; without it, recover only lists them")
+    .action(async (options) => {
+      const configPath = await resolveConfigPath(options.config)
+      const config = await loadConfig(configPath)
+      const result = await recoverOrphans({config, force: Boolean(options.force)})
+
+      if ("error" in result) {
+        console.error(result.error)
+        process.exitCode = 1
+        return
+      }
+
+      if (result.remaining.length > 0) {
+        console.error(formatRecoverResult(result))
+        process.exitCode = 1
+        return
+      }
+
+      console.log(formatRecoverResult(result))
+    })
+
+  program
+    .command("completion")
+    .description("Print a shell completion script. Enable with: source <(rollbridge completion <shell>)")
+    .argument("<shell>", "Shell to generate completion for (bash or zsh)")
+    .action((shell) => {
+      if (shell !== "bash" && shell !== "zsh") {
+        console.error(`Unsupported shell "${shell}". Supported shells: bash, zsh.`)
+        process.exitCode = 1
+        return
+      }
+
+      console.log(generateCompletionScript(program, shell))
+    })
+
   await program.parseAsync(argv)
 }
 
+/**
+ * @param {import("./predeploy-cleanup.js").PredeployCleanupResult} result - Cleanup result.
+ * @returns {string} Human-readable summary.
+ */
+function formatPredeployCleanupResult(result) {
+  if (result.action === "daemon-active") {
+    return "Rollbridge daemon already has an active release; no legacy cleanup needed."
+  }
+
+  const lines = []
+
+  if (result.action === "daemon-stopped") {
+    lines.push("Stopped existing Rollbridge daemon before deploy.")
+  } else {
+    lines.push("No active Rollbridge daemon found.")
+  }
+
+  lines.push(`Recovered ${result.recoveredOrphans} Rollbridge orphaned process${result.recoveredOrphans === 1 ? "" : "es"}.`)
+  lines.push(`Stopped ${result.legacyProcesses.length} legacy process${result.legacyProcesses.length === 1 ? "" : "es"}.`)
+
+  return lines.join("\n")
+}
+
+/**
+ * @typedef {import("./recover.js").OrphanProcess} OrphanProcess
+ */
+
+/**
+ * Formats the result of a recover run (the report case, not the error case).
+ * @param {{cleared: boolean, forced: boolean, orphans: OrphanProcess[], remaining: OrphanProcess[]}} result - Recover report.
+ * @returns {string} Human-readable summary.
+ */
+export function formatRecoverResult(result) {
+  if (result.orphans.length === 0) {
+    return result.forced ? "No orphaned processes found; cleared the state file." : "No orphaned processes found."
+  }
+
+  if (!result.forced) {
+    return [
+      `Found ${orphanCountLabel(result.orphans.length)} (run with --force to stop):`,
+      ...listOrphans(result.orphans),
+      "Review the list first — a recycled pid could be an unrelated process."
+    ].join("\n")
+  }
+
+  const remainingPids = new Set(result.remaining.map((orphan) => orphan.pid))
+  const stopped = result.orphans.filter((orphan) => !remainingPids.has(orphan.pid))
+  const lines = []
+
+  if (stopped.length > 0) lines.push(`Stopped ${orphanCountLabel(stopped.length)}:`, ...listOrphans(stopped))
+
+  if (result.remaining.length > 0) {
+    lines.push(
+      `Could not stop ${orphanCountLabel(result.remaining.length)} — still running (check permissions/ownership):`,
+      ...listOrphans(result.remaining),
+      "Left the state file in place so you can investigate and re-run recover."
+    )
+  }
+
+  return lines.join("\n")
+}
+
+/**
+ * @param {number} count - Number of orphaned processes.
+ * @returns {string} A pluralized label such as "1 orphaned process" or "3 orphaned processes".
+ */
+function orphanCountLabel(count) {
+  return `${count} orphaned process${count === 1 ? "" : "es"}`
+}
+
+/**
+ * @param {OrphanProcess[]} orphans - Orphans to render.
+ * @returns {string[]} One indented line per orphan.
+ */
+function listOrphans(orphans) {
+  return orphans.map((orphan) => `  ${orphan.id} (pid ${orphan.pid}${orphan.releaseId ? `, release ${orphan.releaseId}` : ""})`)
+}
+
+/**
+ * @typedef {{name: string, options: string[], valueOptions: string[]}} CompletionCommand
+ */
+
+/**
+ * Builds a shell completion script by introspecting the CLI's commands and options,
+ * so completions never drift from the actual command surface.
+ * @param {import("commander").Command} program - Configured CLI program.
+ * @param {"bash" | "zsh"} shell - Target shell.
+ * @returns {string} A sourceable completion script.
+ */
+export function generateCompletionScript(program, shell) {
+  const commands = describeCommands(program)
+
+  return shell === "zsh" ? zshCompletionScript(commands) : bashCompletionScript(commands)
+}
+
+/**
+ * @param {import("commander").Command} program - Configured CLI program.
+ * @returns {CompletionCommand[]} Each command's name, long option flags, and value-taking option flags.
+ */
+function describeCommands(program) {
+  return program.commands.map((command) => {
+    /** @type {string[]} */
+    const options = []
+    /** @type {string[]} */
+    const valueOptions = []
+
+    for (const option of command.options) {
+      if (!option.long) continue
+
+      options.push(option.long)
+      if (option.required || option.optional) valueOptions.push(option.long)
+    }
+
+    return {name: command.name(), options, valueOptions}
+  })
+}
+
+/**
+ * @param {CompletionCommand[]} commands - Command descriptors.
+ * @returns {string} A bash completion script.
+ */
+function bashCompletionScript(commands) {
+  const names = commands.map((command) => command.name).join(" ")
+  const branches = commands
+    .map((command) => `    ${command.name})\n      opts="${command.options.join(" ")}"\n      values="${command.valueOptions.join(" ")}"\n      ;;`)
+    .join("\n")
+
+  return `# rollbridge bash completion
+# Enable with: source <(rollbridge completion bash)
+_rollbridge() {
+  local cur prev cmd opts values i
+  cur="\${COMP_WORDS[COMP_CWORD]}"
+  prev="\${COMP_WORDS[COMP_CWORD-1]}"
+
+  cmd=""
+  for ((i = 1; i < COMP_CWORD; i++)); do
+    case "\${COMP_WORDS[i]}" in
+      -*) ;;
+      *) cmd="\${COMP_WORDS[i]}"; break ;;
+    esac
+  done
+
+  if [[ -z "$cmd" ]]; then
+    COMPREPLY=( $(compgen -W "${names}" -- "$cur") )
+    return
+  fi
+
+  opts=""
+  values=""
+  case "$cmd" in
+${branches}
+  esac
+
+  if [[ -n "$values" && " $values " == *" $prev "* ]]; then
+    COMPREPLY=( $(compgen -f -- "$cur") )
+    return
+  fi
+
+  COMPREPLY=( $(compgen -W "$opts" -- "$cur") )
+}
+complete -F _rollbridge rollbridge
+`
+}
+
+/**
+ * @param {CompletionCommand[]} commands - Command descriptors.
+ * @returns {string} A zsh completion script.
+ */
+function zshCompletionScript(commands) {
+  const names = commands.map((command) => command.name).join(" ")
+  const branches = commands
+    .map((command) => `    ${command.name}) compadd -- ${command.options.join(" ")} ;;`)
+    .join("\n")
+
+  return `#compdef rollbridge
+# rollbridge zsh completion
+# Enable with: source <(rollbridge completion zsh)
+_rollbridge() {
+  local -a commands
+  commands=(${names})
+
+  if (( CURRENT == 2 )); then
+    compadd -- $commands
+    return
+  fi
+
+  case "\${words[2]}" in
+${branches}
+  esac
+}
+compdef _rollbridge rollbridge
+`
+}
+
+/**
+ * Formats structured daemon events as human-readable lines.
+ * @param {import("./event-log.js").DaemonEvent[]} events - Recent events, oldest first.
+ * @returns {string} One line per event, or a placeholder when empty.
+ */
+export function formatEvents(events) {
+  if (events.length === 0) return "No events recorded yet."
+
+  return events
+    .map((event) => {
+      const data = Object.keys(event.data).length > 0 ? ` ${JSON.stringify(event.data)}` : ""
+
+      return `${event.at}  ${event.message}${data}`
+    })
+    .join("\n")
+}
+
 /**
  * @typedef {{id: string, logs: import("./managed-process.js").ManagedProcessLog[], source: string}} LogSource
  */