velocious 1.0.524 → 1.0.526

package/package.json

@@ -3,7 +3,7 @@
     "velocious": "build/bin/velocious.js"
   },
   "name": "velocious",
-  "version": "1.0.524",
+  "version": "1.0.526",
   "main": "build/index.js",
   "types": "build/index.d.ts",
   "files": [

package/src/background-jobs/main.js

@@ -784,6 +784,37 @@ export default class BackgroundJobsMain {
     errorEvents.emit("all-error", {...payload, errorType: "background-job-failed"})
   }
 
+  /**
+   * Emits `background-job-orphaned` (mirrored to `all-error`) for a job the time-based orphan sweep
+   * reclaimed after its worker died mid-run. Unlike `background-job-failed`, which fires on a
+   * worker's failure report, this fires from the main process's sweep, so applications can react to
+   * a dead worker's specific job — recover the work it left behind — without polling. `willRetry`
+   * reflects whether the reclaim returned the job to the queue for another attempt.
+   * @param {{job: import("./types.js").BackgroundJobRow}} args - The orphaned job.
+   * @returns {void}
+   */
+  _emitBackgroundJobOrphaned({job}) {
+    const normalizedError = this._normalizeFailureError(job.lastError ?? "Job orphaned after timeout")
+    const payload = {
+      context: {
+        attempts: job.attempts,
+        jobArgs: job.args,
+        jobId: job.id,
+        jobName: job.jobName,
+        maxRetries: job.maxRetries,
+        stage: "background-job-orphaned",
+        status: job.status,
+        terminal: job.status === "failed" || job.status === "orphaned",
+        willRetry: job.status === "queued"
+      },
+      error: normalizedError
+    }
+    const errorEvents = this.configuration.getErrorEvents()
+
+    errorEvents.emit("background-job-orphaned", payload)
+    errorEvents.emit("all-error", {...payload, errorType: "background-job-orphaned"})
+  }
+
   /**
    * Runs normalize failure error.
    * @param {?} error - Reported failure value.
@@ -1170,14 +1201,26 @@ export default class BackgroundJobsMain {
 
   async _sweepOrphans() {
     try {
-      const count = await this.store.markOrphanedJobs()
+      const orphanedJobs = await this.store.markOrphanedJobs()
 
-      if (count > 0) {
-        this.logger.warn(() => ["Marked orphaned background jobs", count])
-        // Reclaimed orphans become `queued` again — wake the dispatcher
-        // so they aren't stranded until the next external signal.
+      if (orphanedJobs.length > 0) {
+        this.logger.warn(() => ["Marked orphaned background jobs", orphanedJobs.length])
+        // Reclaimed orphans become `queued` again — wake the dispatcher first so
+        // an application event handler that throws below cannot strand them
+        // queued until the next external enqueue/reconnect.
         this._notifyEnqueued()
         await this._drain()
+        // Emit an event per orphaned job so applications can react to a dead
+        // worker's specific job (e.g. targeted recovery) instead of only polling
+        // for its aftermath. Isolate each so one throwing handler can't suppress
+        // the events for the rest.
+        for (const job of orphanedJobs) {
+          try {
+            this._emitBackgroundJobOrphaned({job})
+          } catch (error) {
+            this.logger.error(() => ["A background-job-orphaned event handler threw:", error])
+          }
+        }
       }
     } catch (error) {
       this.logger.error(() => ["Failed to mark orphaned jobs:", error])

package/src/background-jobs/store.js

@@ -547,7 +547,7 @@ export default class BackgroundJobsStore {
    * Runs mark orphaned jobs.
    * @param {object} [args] - Options.
    * @param {number} [args.orphanedAfterMs] - Mark jobs orphaned after this duration.
-   * @returns {Promise<number>} - Count of orphaned jobs.
+   * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - The jobs this sweep marked orphaned.
    */
   async markOrphanedJobs({orphanedAfterMs = ORPHANED_AFTER_MS} = {}) {
     await this.ensureReady()
@@ -562,7 +562,8 @@ export default class BackgroundJobsStore {
 
       const rows = await query.results()
 
-      let orphanedCount = 0
+      /** @type {import("./types.js").BackgroundJobRow[]} */
+      const orphanedJobs = []
 
       for (const row of rows) {
         const job = this._normalizeJobRow(row)
@@ -588,10 +589,10 @@ export default class BackgroundJobsStore {
           conditions: {id: job.id, status: "handed_off", handed_off_at_ms: job.handedOffAtMs}
         })
 
-        if (orphanedJob) orphanedCount += 1
+        if (orphanedJob) orphanedJobs.push(orphanedJob)
       }
 
-      return orphanedCount
+      return orphanedJobs
     })
   }
 
@@ -1075,10 +1076,30 @@ export default class BackgroundJobsStore {
     if (affectedRows !== 1) return null
     await this._releaseConcurrency(db, job.concurrencyKey)
 
-    const updatedJob = await this._getJobRowById(db, job.id)
+    // Return a snapshot of the transition this update just applied rather than re-reading the row.
+    // We won the conditional update (affectedRows === 1), so this state is authoritative; re-reading
+    // could instead observe a newer state if another dispatcher reclaims a requeued job between the
+    // update and the read (overlapping mains / polling dispatch), which would misreport the
+    // status/terminal/willRetry of this transition to failure/orphan event listeners.
+    const status = shouldRetry ? "queued" : (markOrphaned ? "orphaned" : "failed")
+    /** @type {import("./types.js").BackgroundJobRow} */
+    const transitionedJob = {
+      ...job,
+      attempts: nextAttempt,
+      handedOffAtMs: null,
+      lastError: failureMessage,
+      status,
+      workerId: null
+    }
+
+    if (markOrphaned) transitionedJob.orphanedAtMs = now
+    if (shouldRetry) {
+      transitionedJob.scheduledAtMs = scheduledAt
+    } else if (!markOrphaned) {
+      transitionedJob.failedAtMs = now
+    }
 
-    if (!updatedJob) return null
-    return updatedJob
+    return transitionedJob
   }
 
   /**

package/src/background-jobs/worker.js

@@ -11,6 +11,13 @@ import {fileURLToPath} from "node:url"
 
 /** Grace period after SIGTERM before a lingering process runner is SIGKILLed. */
 const FORKED_CHILD_SIGKILL_GRACE_MS = 5000
+/**
+ * Largest delay Node's `setTimeout` accepts without overflowing to a 1ms delay
+ * (a 32-bit signed int of ms, ~24.8 days). A `jobTimeoutMs` above this — or a
+ * non-finite one like `Infinity` — is clamped/disabled rather than coerced to
+ * ~1ms, which would otherwise terminate every forked job almost immediately.
+ */
+const MAX_FORKED_JOB_TIMEOUT_MS = 2_147_483_647
 const FORKED_RUNNER_ENTRY_PATH = fileURLToPath(new URL("./forked-runner-child.js", import.meta.url))
 /** How often the worker sends a liveness heartbeat to the main. */
 const HEARTBEAT_INTERVAL_MS = 15000
@@ -21,6 +28,15 @@ const SOCKET_KEEPALIVE_MS = 10000
  * @type {import("./types.js").BackgroundJobExecutionMode[]} */
 const EXECUTION_MODES = ["inline", "forked", "spawned"]
 
+/**
+ * Per-forked-child timeout bookkeeping.
+ * @typedef {object} ForkedJobTimeoutState
+ * @property {boolean} timedOut - Whether the timeout fired and the child was terminated.
+ * @property {number | null} timeoutMs - The armed timeout in ms, or null when disabled.
+ * @property {ReturnType<typeof setTimeout> | null} timer - The pending timeout timer, cleared on exit.
+ * @property {ReturnType<typeof setTimeout> | null} sigkillTimer - The pending SIGKILL grace timer, cleared on exit.
+ */
+
 export default class BackgroundJobsWorker {
   /**
    * Runs constructor.
@@ -32,8 +48,9 @@ export default class BackgroundJobsWorker {
    * @param {number} [args.maxConcurrentInlineJobs] - Override the inline-job concurrency cap from `configuration.getBackgroundJobsConfig()`.
    * @param {number} [args.forkedChildSigkillGraceMs] - Override the grace period between SIGTERM and SIGKILL when reaping lingering process runners on stop.
    * @param {number} [args.heartbeatIntervalMs] - Override the liveness heartbeat interval (default 15000ms).
+   * @param {number} [args.jobTimeoutMs] - Override the forked-job wall-clock timeout from `configuration.getBackgroundJobsConfig()`. `0` disables it.
    */
-  constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs} = {}) {
+  constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs, jobTimeoutMs} = {}) {
     /**
      * Narrows the runtime value to the documented type.
      * @type {Promise<import("../configuration.js").default>} */
@@ -77,6 +94,13 @@ export default class BackgroundJobsWorker {
     this.forkedChildSigkillGraceMs = typeof forkedChildSigkillGraceMs === "number" && forkedChildSigkillGraceMs >= 0
       ? forkedChildSigkillGraceMs
       : FORKED_CHILD_SIGKILL_GRACE_MS
+    /**
+     * Constructor override for the forked-job wall-clock timeout. When unset the
+     * timeout is read from `configuration.getBackgroundJobsConfig().jobTimeoutMs`
+     * at fork time (default: disabled).
+     * @type {number | undefined}
+     */
+    this.jobTimeoutMsOverride = typeof jobTimeoutMs === "number" ? jobTimeoutMs : undefined
     this.shouldStop = false
     this.workerId = randomUUID()
     this.heartbeatIntervalMs = typeof heartbeatIntervalMs === "number" && heartbeatIntervalMs >= 1
@@ -590,16 +614,109 @@ export default class BackgroundJobsWorker {
    * @returns {Promise<void>} - Resolves when the child exits.
    */
   _waitForForkedChild({child, payload}) {
+    const timeoutState = this._armForkedJobTimeout({child})
+
     return new Promise((resolve) => {
       child.once("exit", (code, signal) => {
-        this._handleForkedChildExit({child, code, signal, payload, resolve})
+        this._clearForkedJobTimeout(timeoutState)
+        this._handleForkedChildExit({child, code, signal, payload, resolve, timeoutState})
       })
       child.once("error", (error) => {
+        this._clearForkedJobTimeout(timeoutState)
         this._handleForkedChildError({child, error, payload, resolve})
       })
     })
   }
 
+  /**
+   * Arms a wall-clock backstop for a forked job runner. A forked job still
+   * running after `jobTimeoutMs` is terminated (SIGTERM, then SIGKILL after the
+   * grace) so a single genuinely-hung runner can't pin a draining worker — and
+   * its full-app boot and database connections — indefinitely. Returns a state
+   * object the exit/error handlers use to cancel the timer and to report a
+   * timeout-specific failure. When no timeout is configured the timer is null
+   * and behavior is unchanged.
+   * @param {object} args - Options.
+   * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
+   * @returns {ForkedJobTimeoutState} - Timeout state.
+   */
+  _armForkedJobTimeout({child}) {
+    const timeoutMs = this._resolveForkedJobTimeoutMs()
+    /** @type {ForkedJobTimeoutState} */
+    const state = {timedOut: false, timeoutMs, timer: null, sigkillTimer: null}
+
+    if (!(typeof timeoutMs === "number" && timeoutMs > 0)) return state
+
+    state.timer = setTimeout(() => this._onForkedJobTimeout({child, state}), timeoutMs)
+
+    return state
+  }
+
+  /**
+   * Resolves the effective forked-job timeout in ms, or null when disabled. The
+   * constructor override wins; otherwise the value comes from the background-jobs
+   * configuration. A non-positive value disables the backstop.
+   * @returns {number | null} - Timeout in ms, or null when disabled.
+   */
+  _resolveForkedJobTimeoutMs() {
+    const raw = typeof this.jobTimeoutMsOverride === "number"
+      ? this.jobTimeoutMsOverride
+      : (this.configuration ? this.configuration.getBackgroundJobsConfig().jobTimeoutMs : null)
+
+    // A non-finite (e.g. Infinity) or non-positive value disables the backstop;
+    // a finite value beyond Node's timer range is clamped to the max rather than
+    // silently coerced to ~1ms (which would kill every forked job immediately).
+    if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) return null
+
+    return Math.min(raw, MAX_FORKED_JOB_TIMEOUT_MS)
+  }
+
+  /**
+   * Fired when a forked runner overruns its timeout. Sends SIGTERM for a clean
+   * shutdown, then SIGKILL after the grace for a runner that ignores it. The
+   * resulting non-clean exit flows through `_handleForkedChildExit`, which frees
+   * the slot and reports the job failed.
+   * @param {object} args - Options.
+   * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
+   * @param {ForkedJobTimeoutState} args.state - Timeout state.
+   * @returns {void}
+   */
+  _onForkedJobTimeout({child, state}) {
+    state.timedOut = true
+
+    try {
+      child.kill("SIGTERM")
+    } catch {
+      // Child already exited; nothing to do.
+    }
+
+    state.sigkillTimer = setTimeout(() => {
+      try {
+        child.kill("SIGKILL")
+      } catch {
+        // Child already exited; nothing to do.
+      }
+    }, this.forkedChildSigkillGraceMs)
+  }
+
+  /**
+   * Cancels any pending timeout/SIGKILL timers for a forked runner that has
+   * exited (or errored) so they never fire against a gone or reused child.
+   * @param {ForkedJobTimeoutState} state - Timeout state.
+   * @returns {void}
+   */
+  _clearForkedJobTimeout(state) {
+    if (state.timer) {
+      clearTimeout(state.timer)
+      state.timer = null
+    }
+
+    if (state.sigkillTimer) {
+      clearTimeout(state.sigkillTimer)
+      state.sigkillTimer = null
+    }
+  }
+
   /**
    * Runs handle forked child exit.
    * @param {object} args - Options.
@@ -608,9 +725,10 @@ export default class BackgroundJobsWorker {
    * @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
    * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
    * @param {(value: void) => void} args.resolve - Promise resolver.
+   * @param {ForkedJobTimeoutState} [args.timeoutState] - Timeout state, when the runner had a wall-clock backstop.
    * @returns {void}
    */
-  _handleForkedChildExit({child, code, signal, payload, resolve}) {
+  _handleForkedChildExit({child, code, signal, payload, resolve, timeoutState}) {
     this.inflightProcessChildren.delete(child)
 
     // Free the worker slot as soon as the child is gone — never gate it on the
@@ -620,10 +738,11 @@ export default class BackgroundJobsWorker {
 
     if (this._forkedChildExitedCleanly({code, signal})) return
 
-    this._reportForkedChildFailure({
-      payload,
-      error: new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
-    })
+    const error = timeoutState?.timedOut
+      ? new Error(`Forked background job runner timed out after ${timeoutState.timeoutMs}ms and was terminated: code=${code} signal=${signal || "none"}`)
+      : new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
+
+    this._reportForkedChildFailure({payload, error})
   }
 
   /**

package/src/configuration-types.js

@@ -192,6 +192,15 @@
  *   to restore the legacy fixed-interval poll.
  * @property {number} [pollIntervalMs] - Poll interval in milliseconds. Only used
  *   when `dispatchStrategy === "polling"`. Default: `1000`.
+ * @property {number | null} [jobTimeoutMs] - Wall-clock backstop, in ms, for a
+ *   `"forked"` job runner. A forked job still running after this is terminated
+ *   (SIGTERM, then SIGKILL after the reaping grace) and reported failed, so a
+ *   single genuinely-hung runner can't pin a draining worker — and its full-app
+ *   boot and DB connections — indefinitely (e.g. across a deploy where a retired
+ *   worker drains in-flight jobs). This is a coarse safety net, not per-job
+ *   tuning: set it well above the longest legitimate forked job (build runners,
+ *   long imports) so only genuinely-stuck runners are killed. Omit, `null`, or
+ *   `<= 0` to disable (default), which preserves the prior unbounded behavior.
  * @property {BackgroundJobsRetentionConfiguration} [retention] - Retention/pruning
  *   of terminal job rows. Without pruning the jobs table grows unbounded
  *   (completed rows accumulate forever), which bloats storage and indexes and

package/src/configuration.js

@@ -1270,10 +1270,12 @@ export default class VelociousConfiguration {
     const envMaxConcurrentRaw = process.env.VELOCIOUS_BACKGROUND_JOBS_MAX_CONCURRENT_INLINE_JOBS
     const envDispatchStrategy = process.env.VELOCIOUS_BACKGROUND_JOBS_DISPATCH_STRATEGY
     const envPollIntervalRaw = process.env.VELOCIOUS_BACKGROUND_JOBS_POLL_INTERVAL_MS
+    const envJobTimeoutRaw = process.env.VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS
     const envPort = envPortRaw ? Number(envPortRaw) : undefined
     const envMaxConcurrentForked = envMaxConcurrentForkedRaw ? Number(envMaxConcurrentForkedRaw) : undefined
     const envMaxConcurrent = envMaxConcurrentRaw ? Number(envMaxConcurrentRaw) : undefined
     const envPollInterval = envPollIntervalRaw ? Number(envPollIntervalRaw) : undefined
+    const envJobTimeout = envJobTimeoutRaw ? Number(envJobTimeoutRaw) : undefined
     const configured = this._backgroundJobs || {}
     const host = configured.host || envHost || "127.0.0.1"
     const port = typeof configured.port === "number"
@@ -1292,6 +1294,12 @@ export default class VelociousConfiguration {
       ? configured.pollIntervalMs
       : (typeof envPollInterval === "number" && Number.isFinite(envPollInterval) && envPollInterval >= 1 ? envPollInterval : 1000)
     const queues = configured.queues && typeof configured.queues === "object" ? configured.queues : {}
+    // An explicit config value wins over the env var — including `null`/`0`,
+    // which disable the backstop even when the environment sets a default.
+    // Only fall through to the env var when config omits `jobTimeoutMs` entirely.
+    const jobTimeoutMs = "jobTimeoutMs" in configured
+      ? (typeof configured.jobTimeoutMs === "number" && configured.jobTimeoutMs > 0 ? configured.jobTimeoutMs : null)
+      : (typeof envJobTimeout === "number" && Number.isFinite(envJobTimeout) && envJobTimeout > 0 ? envJobTimeout : null)
     const configuredRetention = configured.retention && typeof configured.retention === "object" ? configured.retention : {}
     const retention = {
       completedTtlMs: typeof configuredRetention.completedTtlMs === "number" || configuredRetention.completedTtlMs === null
@@ -1308,7 +1316,7 @@ export default class VelociousConfiguration {
         : 60 * 60 * 1000
     }
 
-    return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, dispatchStrategy, pollIntervalMs, queues, retention}
+    return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, dispatchStrategy, pollIntervalMs, queues, jobTimeoutMs, retention}
   }
 
   /**