velocious 1.0.519 → 1.0.520

package/src/background-jobs/main.js

@@ -5,6 +5,7 @@ import JsonSocket from "./json-socket.js"
 import BackgroundJobsScheduler from "./scheduler.js"
 import BackgroundJobsStore from "./store.js"
 import Logger from "../logger.js"
+import PruneTerminalBackgroundJobsJob from "../jobs/prune-terminal-background-jobs.js"
 
 /**
  * Channel used by `background-jobs-main` to coordinate dispatch wake-ups
@@ -21,6 +22,10 @@ const DISPATCH_CHANNEL = "velocious-background-jobs-dispatch"
  * scheduled-job timer here and re-arm when it expires.
  */
 const MAX_TIMER_MS = 2_147_483_647 // ~24.8 days
+/** A worker silent (no heartbeat/ready/report) longer than this is dropped. */
+const WORKER_STALE_TIMEOUT_MS = 60000
+/** How often the main scans workers for staleness. */
+const WORKER_LIVENESS_SWEEP_MS = 15000
 /**
  * WorkerExecutionModeCapability type.
  * @typedef {object} WorkerExecutionModeCapability
@@ -46,8 +51,10 @@ export default class BackgroundJobsMain {
    * @param {import("../configuration.js").default} args.configuration - Configuration.
    * @param {string} [args.host] - Hostname.
    * @param {number} [args.port] - Port.
+   * @param {number} [args.workerStaleTimeoutMs] - Override how long a silent worker may go before being dropped (default 60000ms).
+   * @param {number} [args.workerLivenessSweepMs] - Override how often stale workers are swept for (default 15000ms).
    */
-  constructor({configuration, host, port}) {
+  constructor({configuration, host, port, workerStaleTimeoutMs, workerLivenessSweepMs}) {
     this.configuration = configuration
     const config = configuration.getBackgroundJobsConfig()
     this.host = host || config.host
@@ -55,6 +62,10 @@ export default class BackgroundJobsMain {
     this.dispatchStrategy = config.dispatchStrategy
     this.pollIntervalMs = config.pollIntervalMs
     this.retention = config.retention
+    // A worker that stops sending anything (heartbeat/ready/report) for this
+    // long is treated as wedged/dead: its leases are released and it is dropped.
+    this.workerStaleTimeoutMs = typeof workerStaleTimeoutMs === "number" && workerStaleTimeoutMs >= 1 ? workerStaleTimeoutMs : WORKER_STALE_TIMEOUT_MS
+    this.workerLivenessSweepMs = typeof workerLivenessSweepMs === "number" && workerLivenessSweepMs >= 1 ? workerLivenessSweepMs : WORKER_LIVENESS_SWEEP_MS
     this.store = new BackgroundJobsStore({configuration, databaseIdentifier: config.databaseIdentifier})
     this.logger = new Logger(this)
     /**
@@ -91,8 +102,8 @@ export default class BackgroundJobsMain {
     this._orphanTimer = undefined
     /**
      * Narrows the runtime value to the documented type.
-     * @type {ReturnType<typeof setTimeout> | undefined} */
-    this._retentionTimer = undefined
+     * @type {ReturnType<typeof setInterval> | undefined} */
+    this._workerStaleTimer = undefined
     /**
      * Narrows the runtime value to the documented type.
      * @type {BackgroundJobsScheduler | undefined} */
@@ -144,9 +155,9 @@ export default class BackgroundJobsMain {
         void this._sweepOrphans()
       }, 60000)
 
-      this._retentionTimer = setInterval(() => {
-        void this._sweepRetention()
-      }, this.retention.sweepIntervalMs)
+      this._workerStaleTimer = setInterval(() => {
+        void this._sweepStaleWorkers()
+      }, this.workerLivenessSweepMs)
 
       this.scheduler = new BackgroundJobsScheduler({
         configuration: this.configuration,
@@ -154,7 +165,10 @@ export default class BackgroundJobsMain {
           await this.store.enqueue({
             jobName: jobClass.jobName(),
             args,
-            options
+            // Fold in the job class's static `queue` (as performLater* do) so a
+            // scheduled job with `static queue = "..."` lands on its queue and
+            // honors the configured cap without every schedule repeating it.
+            options: jobClass._withQueue(options)
           })
           this._notifyEnqueued()
           await this._drain()
@@ -162,6 +176,16 @@ export default class BackgroundJobsMain {
       })
       await this.scheduler.start()
 
+      // Retention pruning runs as an ordinary scheduled job on the normal
+      // scheduler (so it is visible in the job tables and dispatched to a
+      // worker), rather than a hidden in-process timer. Skipped when retention
+      // is disabled. The scheduler owns the timer, so scheduler.stop() clears it.
+      const retentionSchedule = PruneTerminalBackgroundJobsJob.scheduleConfiguration(this.retention)
+
+      if (retentionSchedule) {
+        this.scheduler.scheduleJob({jobConfiguration: retentionSchedule, jobKey: "velociousPruneTerminalBackgroundJobs"})
+      }
+
       // Startup catch-up: drain anything that was waiting before this
       // process came up. In beacon mode this is also the safety net for
       // races between attaching the connect listener and the initial
@@ -206,12 +230,12 @@ export default class BackgroundJobsMain {
     if (this._scheduledTimer) clearTimeout(this._scheduledTimer)
     if (this._errorRetryTimer) clearTimeout(this._errorRetryTimer)
     if (this._orphanTimer) clearInterval(this._orphanTimer)
-    if (this._retentionTimer) clearInterval(this._retentionTimer)
+    if (this._workerStaleTimer) clearInterval(this._workerStaleTimer)
     this._pollTimer = undefined
     this._scheduledTimer = undefined
     this._errorRetryTimer = undefined
     this._orphanTimer = undefined
-    this._retentionTimer = undefined
+    this._workerStaleTimer = undefined
   }
 
   /**
@@ -394,6 +418,8 @@ export default class BackgroundJobsMain {
     if (message.role === "worker") {
       jsonSocket.workerId = message.workerId
       jsonSocket.supportsHandoffIdReporting = message.supportsHandoffIdReporting === true
+      jsonSocket.supportsHeartbeat = message.supportsHeartbeat === true
+      jsonSocket.lastSeenAt = Date.now()
       this.workers.add(jsonSocket)
       this.workerHandoffs.set(jsonSocket, new Map())
     }
@@ -422,6 +448,14 @@ export default class BackgroundJobsMain {
    * @returns {void}
    */
   _handleWorkerSocketMessage({jsonSocket, message}) {
+    // Any message from the worker proves it is alive; the liveness sweep uses
+    // this to detect a wedged/silent worker.
+    jsonSocket.lastSeenAt = Date.now()
+
+    if (message?.type === "heartbeat") {
+      return
+    }
+
     if (message?.type === "ready") {
       this._handleWorkerReady({jsonSocket, message})
       return
@@ -1098,22 +1132,43 @@ export default class BackgroundJobsMain {
   }
 
   /**
-   * Deletes terminal job rows past their retention window so the jobs table
-   * does not grow unbounded. Runs on its own interval; failures are logged and
-   * retried on the next tick.
-   * @returns {Promise<void>} - Resolves after one retention sweep.
+   * Drops workers that have gone silent past `workerStaleTimeoutMs` (no
+   * heartbeat, ready, or report). A wedged worker keeps its socket open, so the
+   * `close`-based cleanup never fires and its in-flight leases — and the whole
+   * queue — stay stuck until a human notices. Releasing the lost worker's
+   * leases lets its jobs run elsewhere and stops dispatch to it; the worker's
+   * own process lifecycle is the supervisor's concern.
+   * @returns {Promise<void>} - Resolves after the sweep.
    */
-  async _sweepRetention() {
-    try {
-      const deleted = await this.store.pruneTerminalJobs({
-        completedTtlMs: this.retention.completedTtlMs,
-        failedTtlMs: this.retention.failedTtlMs,
-        batchSize: this.retention.batchSize
-      })
+  async _sweepStaleWorkers() {
+    if (this._stopped) return
 
-      if (deleted > 0) this.logger.warn(() => ["Pruned terminal background jobs", deleted])
-    } catch (error) {
-      this.logger.error(() => ["Failed to prune terminal jobs:", error])
+    const cutoff = Date.now() - this.workerStaleTimeoutMs
+    /** @type {JsonSocket[]} */
+    const stale = []
+
+    for (const worker of this.workers) {
+      // Only evict heartbeat-capable workers. A legacy worker (e.g. one from the
+      // previous release during a rolling deploy) never heartbeats, so evicting
+      // it on silence would wrongly release the leases of a job it is still
+      // running. Its disconnect is still handled by the socket `close` path.
+      if (!worker.supportsHeartbeat) continue
+
+      const lastSeenAt = typeof worker.lastSeenAt === "number" ? worker.lastSeenAt : 0
+
+      if (lastSeenAt <= cutoff) stale.push(worker)
+    }
+
+    for (const worker of stale) {
+      this.logger.warn(() => ["Dropping stale background jobs worker", {workerId: worker.workerId, lastSeenAt: worker.lastSeenAt}])
+
+      try {
+        worker.close()
+      } catch {
+        // Already closing; the lease release below is what matters.
+      }
+
+      await this._handleWorkerSocketClosed(worker)
     }
   }
 }

package/src/background-jobs/store.js

@@ -51,6 +51,7 @@ export default class BackgroundJobsStore {
     this.databaseIdentifier = databaseIdentifier
     this.logger = new Logger(this)
     this._readyPromise = null
+    this._queueConcurrencyReconciled = false
   }
 
   /**
@@ -101,8 +102,26 @@ export default class BackgroundJobsStore {
     const argsJson = JSON.stringify(args || [])
     const queue = this._normalizeQueue(options)
     const concurrency = this._resolveConcurrency(options, queue)
+    /** @type {string} */
+    let resultJobId = jobId
 
     await this._withDb(async (db) => {
+      if (options?.deduplicateWhileQueued && concurrency?.concurrencyKey) {
+        const existing = await db
+          .newQuery()
+          .from(JOBS_TABLE)
+          .select("id")
+          .where({status: "queued", concurrency_key: concurrency.concurrencyKey})
+          .limit(1)
+          .results()
+
+        if (existing[0]) {
+          resultJobId = String(/** @type {Record<string, ?>} */ (existing[0]).id)
+
+          return
+        }
+      }
+
       if (concurrency) {
         if (concurrency.queueDerived) {
           await this._ensureQueueConcurrencyKey(db, concurrency)
@@ -130,7 +149,7 @@ export default class BackgroundJobsStore {
       })
     })
 
-    return jobId
+    return resultJobId
   }
 
   /**
@@ -648,6 +667,7 @@ export default class BackgroundJobsStore {
       if (alreadyApplied && await db.tableExists(JOBS_TABLE)) {
         await this._ensureJobsTableColumns(db)
         await this._ensureConcurrencyTable(db)
+        await this._reconcileQueueConcurrency(db)
         await this._reconcileConcurrency(db)
         return
       }
@@ -655,6 +675,7 @@ export default class BackgroundJobsStore {
       await this._applyMigrations(db)
       await this._ensureJobsTableColumns(db)
       await this._ensureConcurrencyTable(db)
+      await this._reconcileQueueConcurrency(db)
       await this._reconcileConcurrency(db)
 
       if (alreadyApplied) return
@@ -834,16 +855,15 @@ export default class BackgroundJobsStore {
    * @returns {Promise<void>} - Resolves when ensured.
    */
   async _ensureQueueColumn(db) {
-    const table = await db.getTableByNameOrFail(JOBS_TABLE)
-
-    if (await table.getColumnByName("queue")) return
-
     const lockName = `${MIGRATION_SCOPE}:queue_column`
     const acquired = await db.acquireAdvisoryLock(lockName)
 
     if (!acquired) throw new Error("Failed to acquire background jobs queue schema lock")
 
     try {
+      // SQL Server schema reads can deadlock with a concurrent ALTER TABLE, so
+      // acquire the lock before inspecting the column rather than only
+      // protecting the mutation (mirrors the concurrency-column migration).
       db.clearSchemaCache()
       const lockedTable = await db.getTableByNameOrFail(JOBS_TABLE)
 
@@ -1101,6 +1121,9 @@ export default class BackgroundJobsStore {
     if (typeof key !== "string" || key.length === 0 || !Number.isInteger(cap) || Number(cap) <= 0) {
       throw new Error("background job concurrencyKey and maxConcurrency must be paired; concurrencyKey must be non-empty and maxConcurrency must be a positive integer")
     }
+    if (key.startsWith(QUEUE_CONCURRENCY_KEY_PREFIX)) {
+      throw new Error(`background job concurrencyKey must not start with the reserved "${QUEUE_CONCURRENCY_KEY_PREFIX}" prefix, which is reserved for queue-derived concurrency caps`)
+    }
     return {concurrencyKey: key, maxConcurrency: Number(cap)}
   }
 
@@ -1277,6 +1300,65 @@ export default class BackgroundJobsStore {
     )
   }
 
+  /**
+   * Reconciles queue-derived concurrency with the current configuration, once
+   * per process. Enqueue only consults config for new jobs, so a cap added,
+   * removed, or changed while a backlog exists otherwise leaves persisted rows
+   * stale: pre-cap jobs keep a null key and bypass the cap, post-removal jobs
+   * stay capped under a now-unconfigured key, and a changed numeric cap stays
+   * stale until the next enqueue. Bring the durable state in line with config:
+   * sync each configured queue's stored cap, adopt not-yet-keyed non-terminal
+   * jobs onto their queue key, and release non-terminal jobs from queue keys
+   * whose queue is no longer capped. Runs before {@link _reconcileConcurrency}
+   * so the rebuilt active counts reflect the adopted/released keys.
+   * @param {import("../database/drivers/base.js").default} db - Database connection.
+   * @returns {Promise<void>} - Resolves when reconciled.
+   */
+  async _reconcileQueueConcurrency(db) {
+    if (this._queueConcurrencyReconciled) return
+    if (!(await db.tableExists(CONCURRENCY_TABLE))) return
+
+    const queuesConfig = this.configuration.getBackgroundJobsConfig().queues || {}
+    const jobsTable = db.quoteTable(JOBS_TABLE)
+    const keyColumn = db.quoteColumn("concurrency_key")
+    const capColumn = db.quoteColumn("max_concurrency")
+    const queueColumn = db.quoteColumn("queue")
+    const nonTerminal = `${db.quoteColumn("status")} IN (${db.quote("queued")}, ${db.quote("handed_off")})`
+    /** @type {Set<string>} */
+    const cappedQueues = new Set()
+
+    for (const queue of Object.keys(queuesConfig)) {
+      const cap = this._queueMaxConcurrency(queue)
+
+      if (cap === null) continue
+
+      cappedQueues.add(queue)
+      const concurrencyKey = `${QUEUE_CONCURRENCY_KEY_PREFIX}${queue}`
+
+      await this._ensureQueueConcurrencyKey(db, {concurrencyKey, maxConcurrency: cap})
+      await db.query(
+        `UPDATE ${jobsTable} SET ${keyColumn} = ${db.quote(concurrencyKey)}, ${capColumn} = ${Number(cap)} ` +
+        `WHERE ${queueColumn} = ${db.quote(queue)} AND ${keyColumn} IS NULL AND ${nonTerminal}`
+      )
+    }
+
+    const concurrencyRows = await db.newQuery().from(CONCURRENCY_TABLE).select("concurrency_key").results()
+
+    for (const row of concurrencyRows) {
+      const concurrencyKey = String(/** @type {Record<string, ?>} */ (row).concurrency_key)
+
+      if (!concurrencyKey.startsWith(QUEUE_CONCURRENCY_KEY_PREFIX)) continue
+      if (cappedQueues.has(concurrencyKey.slice(QUEUE_CONCURRENCY_KEY_PREFIX.length))) continue
+
+      await db.query(
+        `UPDATE ${jobsTable} SET ${keyColumn} = NULL, ${capColumn} = NULL ` +
+        `WHERE ${keyColumn} = ${db.quote(concurrencyKey)} AND ${nonTerminal}`
+      )
+    }
+
+    this._queueConcurrencyReconciled = true
+  }
+
   /**
    * Runs normalize number.
    * @param {?} value - Input value.

package/src/background-jobs/types.js

@@ -16,6 +16,7 @@
  * @property {string} [queue] - Queue name. Defaults to `"default"`. When the queue has a configured cap in `backgroundJobs.queues`, that cap is enforced cluster-wide.
  * @property {string} [concurrencyKey] - Opaque non-empty key used to share a concurrency cap. Overrides any queue-derived cap.
  * @property {number} [maxConcurrency] - Positive integer cap; must be paired with `concurrencyKey`.
+ * @property {boolean} [deduplicateWhileQueued] - When true (requires `concurrencyKey`), skip the enqueue if a still-queued job with the same `concurrencyKey` already exists, returning that job's id. Keeps an interval-scheduled recurring job (e.g. retention pruning) from piling up redundant queued rows when it runs slower than its interval or no worker is free.
  */
 /**
  * @typedef {object} BackgroundJobPayload
@@ -65,9 +66,10 @@
  * @typedef {"worker" | "client" | "reporter"} BackgroundJobSocketRole
  */
 /**
- * @typedef {{type: "hello", role: BackgroundJobSocketRole, supportsHandoffIdReporting?: boolean, workerId?: string}} BackgroundJobHelloMessage
+ * @typedef {{type: "hello", role: BackgroundJobSocketRole, supportsHandoffIdReporting?: boolean, supportsHeartbeat?: boolean, workerId?: string}} BackgroundJobHelloMessage
  * @typedef {{type: "ready", acceptsForked?: boolean, acceptsInline?: boolean, acceptsSpawned?: boolean}} BackgroundJobReadyMessage
  * @typedef {{type: "draining"}} BackgroundJobDrainingMessage
+ * @typedef {{type: "heartbeat", workerId?: string}} BackgroundJobHeartbeatMessage
  * @typedef {{type: "enqueue", jobName: string, args?: Array<?>, options?: BackgroundJobOptions}} BackgroundJobEnqueueMessage
  * @typedef {{type: "enqueued", jobId: string}} BackgroundJobEnqueuedMessage
  * @typedef {{type: "enqueue-error", error?: string}} BackgroundJobEnqueueErrorMessage
@@ -78,7 +80,7 @@
  * @typedef {{type: "job-update-error", jobId: string, error?: string}} BackgroundJobUpdateErrorMessage
  */
 /**
- * @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
+ * @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobHeartbeatMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
  */
 
 export const nothing = {}

package/src/background-jobs/worker.js

@@ -12,6 +12,10 @@ import {fileURLToPath} from "node:url"
 /** Grace period after SIGTERM before a lingering process runner is SIGKILLed. */
 const FORKED_CHILD_SIGKILL_GRACE_MS = 5000
 const FORKED_RUNNER_ENTRY_PATH = fileURLToPath(new URL("./forked-runner-child.js", import.meta.url))
+/** How often the worker sends a liveness heartbeat to the main. */
+const HEARTBEAT_INTERVAL_MS = 15000
+/** TCP keepalive so a half-open connection to the main surfaces as a close. */
+const SOCKET_KEEPALIVE_MS = 10000
 /**
  * Execution modes.
  * @type {import("./types.js").BackgroundJobExecutionMode[]} */
@@ -27,8 +31,9 @@ export default class BackgroundJobsWorker {
    * @param {number} [args.maxConcurrentForkedJobs] - Override the process runner concurrency cap from `configuration.getBackgroundJobsConfig()`.
    * @param {number} [args.maxConcurrentInlineJobs] - Override the inline-job concurrency cap from `configuration.getBackgroundJobsConfig()`.
    * @param {number} [args.forkedChildSigkillGraceMs] - Override the grace period between SIGTERM and SIGKILL when reaping lingering process runners on stop.
+   * @param {number} [args.heartbeatIntervalMs] - Override the liveness heartbeat interval (default 15000ms).
    */
-  constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs} = {}) {
+  constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs} = {}) {
     /**
      * Narrows the runtime value to the documented type.
      * @type {Promise<import("../configuration.js").default>} */
@@ -74,6 +79,21 @@ export default class BackgroundJobsWorker {
       : FORKED_CHILD_SIGKILL_GRACE_MS
     this.shouldStop = false
     this.workerId = randomUUID()
+    this.heartbeatIntervalMs = typeof heartbeatIntervalMs === "number" && heartbeatIntervalMs >= 1
+      ? heartbeatIntervalMs
+      : HEARTBEAT_INTERVAL_MS
+    /**
+     * Narrows the runtime value to the documented type.
+     * @type {ReturnType<typeof setInterval> | undefined} */
+    this._heartbeatTimer = undefined
+    /**
+     * In-flight job-result reports to the main. Reporting is decoupled from the
+     * job/child slot (freeing the slot never waits on a report) and retried
+     * durably, so a transient main/DB outage cannot leak slots or lose a
+     * terminal report. Tracked so a graceful `stop()` can drain them.
+     * @type {Set<Promise<void>>}
+     */
+    this.inflightReports = new Set()
     /**
      * Narrows the runtime value to the documented type.
      * @type {JsonSocket | undefined} */
@@ -154,6 +174,7 @@ export default class BackgroundJobsWorker {
   async stop({timeoutMs} = {}) {
     if (this.shouldStop) return
     this.shouldStop = true
+    this._stopHeartbeat()
 
     // Announce drain so main stops dispatching but keeps the connection
     // open until we close it ourselves below.
@@ -168,6 +189,9 @@ export default class BackgroundJobsWorker {
     await this._drainInflight(this.inflightInlineJobs, timeoutMs)
     await this._drainInflight(this.inflightProcessJobs, timeoutMs)
     await this._terminateProcessChildren()
+    // Give in-flight result reports (now decoupled from job slots) a bounded
+    // chance to land before the socket closes.
+    await this._drainInflight(this.inflightReports, timeoutMs)
 
     if (this.jsonSocket) this.jsonSocket.close()
     if (this.configuration) {
@@ -238,6 +262,7 @@ export default class BackgroundJobsWorker {
     const host = this.host || config.host
     const port = typeof this.port === "number" ? this.port : config.port
     const socket = net.createConnection({host, port})
+    socket.setKeepAlive(true, SOCKET_KEEPALIVE_MS)
     const jsonSocket = new JsonSocket(socket)
     this.jsonSocket = jsonSocket
 
@@ -256,16 +281,51 @@ export default class BackgroundJobsWorker {
     })
 
     jsonSocket.on("close", () => {
+      this._stopHeartbeat()
       if (this.shouldStop) return
       setTimeout(() => { void this._connect() }, 1000)
     })
 
     socket.on("connect", () => {
-      jsonSocket.send({type: "hello", role: "worker", supportsHandoffIdReporting: true, workerId: this.workerId})
+      jsonSocket.send({type: "hello", role: "worker", supportsHandoffIdReporting: true, supportsHeartbeat: true, workerId: this.workerId})
       this._sendReadyIfRunning()
+      this._startHeartbeat()
     })
   }
 
+  /**
+   * Sends periodic liveness heartbeats to the main so a wedged or silent worker
+   * can be detected and dropped there (its leases released) instead of freezing
+   * the queue until a human notices.
+   * @returns {void}
+   */
+  _startHeartbeat() {
+    this._stopHeartbeat()
+
+    this._heartbeatTimer = setInterval(() => {
+      if (this.shouldStop || !this.jsonSocket) return
+
+      try {
+        this.jsonSocket.send({type: "heartbeat", workerId: this.workerId})
+      } catch {
+        // Socket is closing/closed; the close handler drives reconnect.
+      }
+    }, this.heartbeatIntervalMs)
+
+    if (typeof this._heartbeatTimer.unref === "function") this._heartbeatTimer.unref()
+  }
+
+  /**
+   * Stops the liveness heartbeat timer.
+   * @returns {void}
+   */
+  _stopHeartbeat() {
+    if (this._heartbeatTimer) {
+      clearInterval(this._heartbeatTimer)
+      this._heartbeatTimer = undefined
+    }
+  }
+
   /**
    * Runs handle job.
    * @param {import("./types.js").BackgroundJobPayload} payload - Payload.
@@ -394,9 +454,12 @@ export default class BackgroundJobsWorker {
    * @returns {Promise<void>} - Resolves when complete (success or failure reported).
    */
   async _runInlineJobAndReport(payload) {
+    // Report in the background so freeing this inline slot never waits on the
+    // report. Reporting is durable (retried until it lands), so a transient
+    // main/DB outage neither wedges the slot nor loses the terminal result.
     try {
       await this._runJobInline(payload)
-      await this._reportJobResult({
+      this._reportJobResultInBackground({
         jobId: payload.id,
         status: "completed",
         handoffId: payload.handoffId,
@@ -404,7 +467,7 @@ export default class BackgroundJobsWorker {
         workerId: payload.workerId || this.workerId
       })
     } catch (error) {
-      await this._reportJobResult({
+      this._reportJobResultInBackground({
         jobId: payload.id,
         status: "failed",
         error,
@@ -522,10 +585,10 @@ export default class BackgroundJobsWorker {
   _waitForForkedChild({child, payload}) {
     return new Promise((resolve) => {
       child.once("exit", (code, signal) => {
-        void this._handleForkedChildExit({child, code, signal, payload, resolve})
+        this._handleForkedChildExit({child, code, signal, payload, resolve})
       })
       child.once("error", (error) => {
-        void this._handleForkedChildError({child, error, payload, resolve})
+        this._handleForkedChildError({child, error, payload, resolve})
       })
     })
   }
@@ -538,22 +601,22 @@ export default class BackgroundJobsWorker {
    * @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
    * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
    * @param {(value: void) => void} args.resolve - Promise resolver.
-   * @returns {Promise<void>} - Resolves after failure is reported.
+   * @returns {void}
    */
-  async _handleForkedChildExit({child, code, signal, payload, resolve}) {
+  _handleForkedChildExit({child, code, signal, payload, resolve}) {
     this.inflightProcessChildren.delete(child)
 
-    if (this._forkedChildExitedCleanly({code, signal})) {
-      resolve(undefined)
-      return
-    }
+    // Free the worker slot as soon as the child is gone — never gate it on the
+    // failure report. A hung/slow report must not leak the slot; enough leaked
+    // slots drive `acceptsForked` to false and silently wedge the worker.
+    resolve(undefined)
+
+    if (this._forkedChildExitedCleanly({code, signal})) return
 
-    await this._reportForkedChildFailure({
+    this._reportForkedChildFailure({
       payload,
       error: new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
     })
-
-    resolve(undefined)
   }
 
   /**
@@ -574,13 +637,14 @@ export default class BackgroundJobsWorker {
    * @param {Error} args.error - Child process error.
    * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
    * @param {(value: void) => void} args.resolve - Promise resolver.
-   * @returns {Promise<void>} - Resolves after failure is reported.
+   * @returns {void}
    */
-  async _handleForkedChildError({child, error, payload, resolve}) {
+  _handleForkedChildError({child, error, payload, resolve}) {
     this.inflightProcessChildren.delete(child)
-    console.error("Background jobs forked runner error:", error)
-    await this._reportForkedChildFailure({payload, error})
+    // Free the slot first (see _handleForkedChildExit) — reporting is best-effort.
     resolve(undefined)
+    console.error("Background jobs forked runner error:", error)
+    this._reportForkedChildFailure({payload, error})
   }
 
   /**
@@ -595,7 +659,7 @@ export default class BackgroundJobsWorker {
       child.send({type: "job", payload})
     } catch (error) {
       child.kill("SIGTERM")
-      void this._reportForkedChildFailure({payload, error})
+      this._reportForkedChildFailure({payload, error})
     }
   }
 
@@ -604,10 +668,10 @@ export default class BackgroundJobsWorker {
    * @param {object} args - Options.
    * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
    * @param {?} args.error - Error.
-   * @returns {Promise<void>} - Resolves after failure is reported.
+   * @returns {void}
    */
-  async _reportForkedChildFailure({payload, error}) {
-    await this._reportJobResult({
+  _reportForkedChildFailure({payload, error}) {
+    this._reportJobResultInBackground({
       jobId: payload.id,
       status: "failed",
       error,
@@ -682,4 +746,30 @@ export default class BackgroundJobsWorker {
       console.error("Background job status reporting failed:", reportError)
     }
   }
+
+  /**
+   * Fires a durable job-result report without blocking the caller (so freeing a
+   * job/child slot never waits on the report). The report is tracked so a
+   * graceful `stop()` can drain in-flight reports before closing the socket.
+   * @param {object} args - Options.
+   * @param {string} args.jobId - Job id.
+   * @param {"completed" | "failed"} args.status - Status.
+   * @param {?} [args.error] - Error.
+   * @param {string} [args.handoffId] - Handoff lease id.
+   * @param {number} [args.handedOffAtMs] - Handed off timestamp.
+   * @param {string} [args.workerId] - Worker id.
+   * @returns {void}
+   */
+  _reportJobResultInBackground({jobId, status, error, handoffId, handedOffAtMs, workerId}) {
+    /**
+     * Defines report.
+     * @type {Promise<void>} */
+    let report
+
+    report = this._reportJobResult({jobId, status, error, handoffId, handedOffAtMs, workerId}).finally(() => {
+      this.inflightReports.delete(report)
+    })
+
+    this.inflightReports.add(report)
+  }
 }

package/src/configuration-types.js

@@ -194,8 +194,17 @@
  *   jobs older than this many ms. `null` or `<= 0` disables (keeps them for
  *   debugging). Default: `2592000000` (30 days).
  * @property {number} [batchSize] - Rows deleted per batch. Default: `1000`.
- * @property {number} [sweepIntervalMs] - How often the main process runs the
- *   retention sweep. Default: `3600000` (1 hour).
+ * @property {number} [sweepIntervalMs] - How often the retention sweep runs.
+ *   Default: `3600000` (1 hour).
+ */
+/**
+ * Fully-resolved retention config as returned by `getBackgroundJobsConfig()`
+ * (every field defaulted), as opposed to the partial user-provided input.
+ * @typedef {object} ResolvedBackgroundJobsRetentionConfiguration
+ * @property {number | null} completedTtlMs - Resolved completed-job TTL in ms (`null` disables).
+ * @property {number | null} failedTtlMs - Resolved failed/orphaned TTL in ms (`null` disables).
+ * @property {number} batchSize - Resolved delete batch size.
+ * @property {number} sweepIntervalMs - Resolved sweep interval in ms.
  */
 
 /**

package/src/configuration.js

@@ -1260,7 +1260,7 @@ export default class VelociousConfiguration {
 
   /**
    * Runs get background jobs config.
-   * @returns {Required<import("./configuration-types.js").BackgroundJobsConfiguration>} - Background jobs configuration.
+   * @returns {Required<import("./configuration-types.js").BackgroundJobsConfiguration> & {retention: import("./configuration-types.js").ResolvedBackgroundJobsRetentionConfiguration}} - Background jobs configuration.
    */
   getBackgroundJobsConfig() {
     const envHost = process.env.VELOCIOUS_BACKGROUND_JOBS_HOST

package/src/environment-handlers/node/cli/commands/generate/base-models.js

@@ -24,7 +24,7 @@ const jsDocTypeByColumnType = {
   float: "number",
   int: "number",
   integer: "number",
-  json: "Record<string, ?>",
+  json: "Record<string, unknown>",
   longtext: "string",
   mediumtext: "string",
   numeric: "number",