velocious 1.0.519 → 1.0.521

package/src/background-jobs/worker.js

@@ -12,6 +12,10 @@ import {fileURLToPath} from "node:url"
 /** Grace period after SIGTERM before a lingering process runner is SIGKILLed. */
 const FORKED_CHILD_SIGKILL_GRACE_MS = 5000
 const FORKED_RUNNER_ENTRY_PATH = fileURLToPath(new URL("./forked-runner-child.js", import.meta.url))
+/** How often the worker sends a liveness heartbeat to the main. */
+const HEARTBEAT_INTERVAL_MS = 15000
+/** TCP keepalive so a half-open connection to the main surfaces as a close. */
+const SOCKET_KEEPALIVE_MS = 10000
 /**
  * Execution modes.
  * @type {import("./types.js").BackgroundJobExecutionMode[]} */
@@ -27,8 +31,9 @@ export default class BackgroundJobsWorker {
    * @param {number} [args.maxConcurrentForkedJobs] - Override the process runner concurrency cap from `configuration.getBackgroundJobsConfig()`.
    * @param {number} [args.maxConcurrentInlineJobs] - Override the inline-job concurrency cap from `configuration.getBackgroundJobsConfig()`.
    * @param {number} [args.forkedChildSigkillGraceMs] - Override the grace period between SIGTERM and SIGKILL when reaping lingering process runners on stop.
+   * @param {number} [args.heartbeatIntervalMs] - Override the liveness heartbeat interval (default 15000ms).
    */
-  constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs} = {}) {
+  constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs} = {}) {
     /**
      * Narrows the runtime value to the documented type.
      * @type {Promise<import("../configuration.js").default>} */
@@ -74,6 +79,21 @@ export default class BackgroundJobsWorker {
       : FORKED_CHILD_SIGKILL_GRACE_MS
     this.shouldStop = false
     this.workerId = randomUUID()
+    this.heartbeatIntervalMs = typeof heartbeatIntervalMs === "number" && heartbeatIntervalMs >= 1
+      ? heartbeatIntervalMs
+      : HEARTBEAT_INTERVAL_MS
+    /**
+     * Narrows the runtime value to the documented type.
+     * @type {ReturnType<typeof setInterval> | undefined} */
+    this._heartbeatTimer = undefined
+    /**
+     * In-flight job-result reports to the main. Reporting is decoupled from the
+     * job/child slot (freeing the slot never waits on a report) and retried
+     * durably, so a transient main/DB outage cannot leak slots or lose a
+     * terminal report. Tracked so a graceful `stop()` can drain them.
+     * @type {Set<Promise<void>>}
+     */
+    this.inflightReports = new Set()
     /**
      * Narrows the runtime value to the documented type.
      * @type {JsonSocket | undefined} */
@@ -154,6 +174,7 @@ export default class BackgroundJobsWorker {
   async stop({timeoutMs} = {}) {
     if (this.shouldStop) return
     this.shouldStop = true
+    this._stopHeartbeat()
 
     // Announce drain so main stops dispatching but keeps the connection
     // open until we close it ourselves below.
@@ -168,6 +189,9 @@ export default class BackgroundJobsWorker {
     await this._drainInflight(this.inflightInlineJobs, timeoutMs)
     await this._drainInflight(this.inflightProcessJobs, timeoutMs)
     await this._terminateProcessChildren()
+    // Give in-flight result reports (now decoupled from job slots) a bounded
+    // chance to land before the socket closes.
+    await this._drainInflight(this.inflightReports, timeoutMs)
 
     if (this.jsonSocket) this.jsonSocket.close()
     if (this.configuration) {
@@ -238,6 +262,7 @@ export default class BackgroundJobsWorker {
     const host = this.host || config.host
     const port = typeof this.port === "number" ? this.port : config.port
     const socket = net.createConnection({host, port})
+    socket.setKeepAlive(true, SOCKET_KEEPALIVE_MS)
     const jsonSocket = new JsonSocket(socket)
     this.jsonSocket = jsonSocket
 
@@ -256,16 +281,51 @@ export default class BackgroundJobsWorker {
     })
 
     jsonSocket.on("close", () => {
+      this._stopHeartbeat()
       if (this.shouldStop) return
       setTimeout(() => { void this._connect() }, 1000)
     })
 
     socket.on("connect", () => {
-      jsonSocket.send({type: "hello", role: "worker", supportsHandoffIdReporting: true, workerId: this.workerId})
+      jsonSocket.send({type: "hello", role: "worker", supportsHandoffIdReporting: true, supportsHeartbeat: true, workerId: this.workerId})
       this._sendReadyIfRunning()
+      this._startHeartbeat()
     })
   }
 
+  /**
+   * Sends periodic liveness heartbeats to the main so a wedged or silent worker
+   * can be detected and dropped there (its leases released) instead of freezing
+   * the queue until a human notices.
+   * @returns {void}
+   */
+  _startHeartbeat() {
+    this._stopHeartbeat()
+
+    this._heartbeatTimer = setInterval(() => {
+      if (this.shouldStop || !this.jsonSocket) return
+
+      try {
+        this.jsonSocket.send({type: "heartbeat", workerId: this.workerId})
+      } catch {
+        // Socket is closing/closed; the close handler drives reconnect.
+      }
+    }, this.heartbeatIntervalMs)
+
+    if (typeof this._heartbeatTimer.unref === "function") this._heartbeatTimer.unref()
+  }
+
+  /**
+   * Stops the liveness heartbeat timer.
+   * @returns {void}
+   */
+  _stopHeartbeat() {
+    if (this._heartbeatTimer) {
+      clearInterval(this._heartbeatTimer)
+      this._heartbeatTimer = undefined
+    }
+  }
+
   /**
    * Runs handle job.
    * @param {import("./types.js").BackgroundJobPayload} payload - Payload.
@@ -394,9 +454,12 @@ export default class BackgroundJobsWorker {
    * @returns {Promise<void>} - Resolves when complete (success or failure reported).
    */
   async _runInlineJobAndReport(payload) {
+    // Report in the background so freeing this inline slot never waits on the
+    // report. Reporting is durable (retried until it lands), so a transient
+    // main/DB outage neither wedges the slot nor loses the terminal result.
     try {
       await this._runJobInline(payload)
-      await this._reportJobResult({
+      this._reportJobResultInBackground({
         jobId: payload.id,
         status: "completed",
         handoffId: payload.handoffId,
@@ -404,7 +467,7 @@ export default class BackgroundJobsWorker {
         workerId: payload.workerId || this.workerId
       })
     } catch (error) {
-      await this._reportJobResult({
+      this._reportJobResultInBackground({
         jobId: payload.id,
         status: "failed",
         error,
@@ -522,10 +585,10 @@ export default class BackgroundJobsWorker {
   _waitForForkedChild({child, payload}) {
     return new Promise((resolve) => {
       child.once("exit", (code, signal) => {
-        void this._handleForkedChildExit({child, code, signal, payload, resolve})
+        this._handleForkedChildExit({child, code, signal, payload, resolve})
       })
       child.once("error", (error) => {
-        void this._handleForkedChildError({child, error, payload, resolve})
+        this._handleForkedChildError({child, error, payload, resolve})
       })
     })
   }
@@ -538,22 +601,22 @@ export default class BackgroundJobsWorker {
    * @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
    * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
    * @param {(value: void) => void} args.resolve - Promise resolver.
-   * @returns {Promise<void>} - Resolves after failure is reported.
+   * @returns {void}
    */
-  async _handleForkedChildExit({child, code, signal, payload, resolve}) {
+  _handleForkedChildExit({child, code, signal, payload, resolve}) {
     this.inflightProcessChildren.delete(child)
 
-    if (this._forkedChildExitedCleanly({code, signal})) {
-      resolve(undefined)
-      return
-    }
+    // Free the worker slot as soon as the child is gone — never gate it on the
+    // failure report. A hung/slow report must not leak the slot; enough leaked
+    // slots drive `acceptsForked` to false and silently wedge the worker.
+    resolve(undefined)
+
+    if (this._forkedChildExitedCleanly({code, signal})) return
 
-    await this._reportForkedChildFailure({
+    this._reportForkedChildFailure({
       payload,
       error: new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
     })
-
-    resolve(undefined)
   }
 
   /**
@@ -574,13 +637,14 @@ export default class BackgroundJobsWorker {
    * @param {Error} args.error - Child process error.
    * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
    * @param {(value: void) => void} args.resolve - Promise resolver.
-   * @returns {Promise<void>} - Resolves after failure is reported.
+   * @returns {void}
    */
-  async _handleForkedChildError({child, error, payload, resolve}) {
+  _handleForkedChildError({child, error, payload, resolve}) {
     this.inflightProcessChildren.delete(child)
-    console.error("Background jobs forked runner error:", error)
-    await this._reportForkedChildFailure({payload, error})
+    // Free the slot first (see _handleForkedChildExit) — reporting is best-effort.
     resolve(undefined)
+    console.error("Background jobs forked runner error:", error)
+    this._reportForkedChildFailure({payload, error})
   }
 
   /**
@@ -595,7 +659,7 @@ export default class BackgroundJobsWorker {
       child.send({type: "job", payload})
     } catch (error) {
       child.kill("SIGTERM")
-      void this._reportForkedChildFailure({payload, error})
+      this._reportForkedChildFailure({payload, error})
     }
   }
 
@@ -604,10 +668,10 @@ export default class BackgroundJobsWorker {
    * @param {object} args - Options.
    * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
    * @param {?} args.error - Error.
-   * @returns {Promise<void>} - Resolves after failure is reported.
+   * @returns {void}
    */
-  async _reportForkedChildFailure({payload, error}) {
-    await this._reportJobResult({
+  _reportForkedChildFailure({payload, error}) {
+    this._reportJobResultInBackground({
       jobId: payload.id,
       status: "failed",
       error,
@@ -682,4 +746,30 @@ export default class BackgroundJobsWorker {
       console.error("Background job status reporting failed:", reportError)
     }
   }
+
+  /**
+   * Fires a durable job-result report without blocking the caller (so freeing a
+   * job/child slot never waits on the report). The report is tracked so a
+   * graceful `stop()` can drain in-flight reports before closing the socket.
+   * @param {object} args - Options.
+   * @param {string} args.jobId - Job id.
+   * @param {"completed" | "failed"} args.status - Status.
+   * @param {?} [args.error] - Error.
+   * @param {string} [args.handoffId] - Handoff lease id.
+   * @param {number} [args.handedOffAtMs] - Handed off timestamp.
+   * @param {string} [args.workerId] - Worker id.
+   * @returns {void}
+   */
+  _reportJobResultInBackground({jobId, status, error, handoffId, handedOffAtMs, workerId}) {
+    /**
+     * Defines report.
+     * @type {Promise<void>} */
+    let report
+
+    report = this._reportJobResult({jobId, status, error, handoffId, handedOffAtMs, workerId}).finally(() => {
+      this.inflightReports.delete(report)
+    })
+
+    this.inflightReports.add(report)
+  }
 }

package/src/configuration-types.js

@@ -164,15 +164,29 @@
  *   allowed to keep in flight. Default: `4`. This is a per-worker safety cap;
  *   for workload-shaped limits use per-queue caps (`queues`) instead, which are
  *   enforced cluster-wide and are immune to duplicate worker processes.
- * @property {Record<string, {maxConcurrent?: number}>} [queues] - Per-queue
- *   concurrency caps, Sidekiq-style. A job declares its queue (static `queue` on
- *   the job class, or the `queue` enqueue option; defaults to `"default"`), and
+ * @property {Record<string, {maxConcurrent?: number, priority?: number}>} [queues] - Per-queue
+ *   concurrency caps and dispatch priorities, Sidekiq-style. A job declares its queue (static
+ *   `queue` on the job class, or the `queue` enqueue option; defaults to `"default"`), and
  *   `queues[name].maxConcurrent` bounds how many jobs from that queue may be
  *   in flight across the whole cluster (enforced via durable per-key
  *   concurrency, so it holds regardless of how many worker processes run).
  *   Size each queue to its workload: I/O-bound queues (e.g. build runners
  *   waiting on remote Docker servers) can run far above the core count, while
- *   CPU-bound queues should stay near it. Default: `{}` (no queue caps).
+ *   CPU-bound queues should stay near it. `queues[name].priority` (default `0`)
+ *   makes the main process dispatch higher-priority queues before lower-priority
+ *   ones regardless of enqueue order, so a small time-critical queue can never be
+ *   starved by a flood of low-priority work sharing a worker pool. Unlike
+ *   Sidekiq's strict queue ordering, priority composes with the per-queue caps:
+ *   a higher-priority queue that is already at its `maxConcurrent` is skipped and
+ *   dispatch falls through to the next eligible lower-priority job, so a busy
+ *   high-priority queue does not block everything else. This fallthrough is a
+ *   property of the queue-derived cap; a job that supplies its own explicit
+ *   `concurrencyKey`/`maxConcurrency` bypasses the queue cap entirely (an
+ *   explicit key always wins — see above) and is bounded only by that key, so it
+ *   is not held back by the queue's cap and priority simply orders it normally.
+ *   Priorities may be any number, including negative to sink a queue below the
+ *   default. Jobs within the same priority keep FIFO (`scheduled_at`, then
+ *   `created_at`) order. Default: `{}` (no queue caps, all queues priority `0`).
  * @property {BackgroundJobsDispatchStrategy} [dispatchStrategy] - How the main process
  *   detects new work. Defaults to `"beacon"` (event-driven). Set to `"polling"`
  *   to restore the legacy fixed-interval poll.
@@ -194,8 +208,17 @@
  *   jobs older than this many ms. `null` or `<= 0` disables (keeps them for
  *   debugging). Default: `2592000000` (30 days).
  * @property {number} [batchSize] - Rows deleted per batch. Default: `1000`.
- * @property {number} [sweepIntervalMs] - How often the main process runs the
- *   retention sweep. Default: `3600000` (1 hour).
+ * @property {number} [sweepIntervalMs] - How often the retention sweep runs.
+ *   Default: `3600000` (1 hour).
+ */
+/**
+ * Fully-resolved retention config as returned by `getBackgroundJobsConfig()`
+ * (every field defaulted), as opposed to the partial user-provided input.
+ * @typedef {object} ResolvedBackgroundJobsRetentionConfiguration
+ * @property {number | null} completedTtlMs - Resolved completed-job TTL in ms (`null` disables).
+ * @property {number | null} failedTtlMs - Resolved failed/orphaned TTL in ms (`null` disables).
+ * @property {number} batchSize - Resolved delete batch size.
+ * @property {number} sweepIntervalMs - Resolved sweep interval in ms.
  */
 
 /**

package/src/configuration.js

@@ -1260,7 +1260,7 @@ export default class VelociousConfiguration {
 
   /**
    * Runs get background jobs config.
-   * @returns {Required<import("./configuration-types.js").BackgroundJobsConfiguration>} - Background jobs configuration.
+   * @returns {Required<import("./configuration-types.js").BackgroundJobsConfiguration> & {retention: import("./configuration-types.js").ResolvedBackgroundJobsRetentionConfiguration}} - Background jobs configuration.
    */
   getBackgroundJobsConfig() {
     const envHost = process.env.VELOCIOUS_BACKGROUND_JOBS_HOST

package/src/environment-handlers/node/cli/commands/generate/base-models.js

@@ -24,7 +24,7 @@ const jsDocTypeByColumnType = {
   float: "number",
   int: "number",
   integer: "number",
-  json: "Record<string, ?>",
+  json: "Record<string, unknown>",
   longtext: "string",
   mediumtext: "string",
   numeric: "number",

package/src/jobs/prune-terminal-background-jobs.js

@@ -0,0 +1,67 @@
+// @ts-check
+
+import BackgroundJobsStore from "../background-jobs/store.js"
+import Configuration from "../configuration.js"
+import VelociousJob from "../background-jobs/job.js"
+
+/**
+ * Built-in job that prunes terminal `background_jobs` rows past their retention
+ * window so the table does not grow unbounded. The main process registers this
+ * on the normal background-jobs scheduler when `backgroundJobs.retention` is
+ * enabled, so it runs as an ordinary scheduled/queued job — visible in the job
+ * tables and dispatched to a worker — rather than a hidden in-process timer.
+ * @augments {VelociousJob<[]>}
+ */
+export default class PruneTerminalBackgroundJobsJob extends VelociousJob {
+  /**
+   * Reserved job name that an application job cannot shadow. The registry loads
+   * app `src/jobs` first and skips duplicate built-in names, so if this used the
+   * default class-name identity an app class named `PruneTerminalBackgroundJobsJob`
+   * would be dispatched instead. A `:`-namespaced name can never collide with a
+   * default (class-name) identity, since class names cannot contain `:`.
+   * @returns {string} - Reserved job name.
+   */
+  static jobName() {
+    return "velocious:prune-terminal-background-jobs"
+  }
+
+  /**
+   * Builds the scheduler configuration for this job from a resolved retention
+   * config, or returns `null` when retention is fully disabled (nothing to
+   * prune, so nothing to schedule). `maxConcurrency: 1` keeps runs from
+   * overlapping, and `deduplicateWhileQueued` stops the interval scheduler from
+   * piling up redundant queued rows when a prune is slow or no worker is free.
+   * @param {import("../configuration-types.js").ResolvedBackgroundJobsRetentionConfiguration} retention - Resolved retention config.
+   * @returns {import("../configuration-types.js").ScheduledBackgroundJobConfiguration | null} - Scheduler config for the prune job, or null when retention is disabled.
+   */
+  static scheduleConfiguration(retention) {
+    const prunesCompleted = typeof retention.completedTtlMs === "number" && retention.completedTtlMs > 0
+    const prunesFailed = typeof retention.failedTtlMs === "number" && retention.failedTtlMs > 0
+
+    if (!prunesCompleted && !prunesFailed) {
+      return null
+    }
+
+    return {
+      class: this,
+      every: retention.sweepIntervalMs,
+      options: {concurrencyKey: "velocious-prune-terminal-background-jobs", maxConcurrency: 1, deduplicateWhileQueued: true}
+    }
+  }
+
+  /**
+   * Prunes terminal job rows past their retention window.
+   * @returns {Promise<void>}
+   */
+  async perform() {
+    const configuration = Configuration.current()
+    const config = configuration.getBackgroundJobsConfig()
+    const store = new BackgroundJobsStore({configuration, databaseIdentifier: config.databaseIdentifier})
+
+    await store.pruneTerminalJobs({
+      completedTtlMs: config.retention.completedTtlMs,
+      failedTtlMs: config.retention.failedTtlMs,
+      batchSize: config.retention.batchSize
+    })
+  }
+}