velocious 1.0.519 → 1.0.520

package/README.md

@@ -2131,6 +2131,40 @@ await MyJob.performLaterWithOptions({
 
 If a handed-off job does not report back within 2 hours, it is marked orphaned and re-queued if retries remain.
 
+Workers also send periodic heartbeats (and use TCP keepalive) so the main can drop a wedged or half-open worker that never fires a socket `close` and release its leases, and job slots are freed independently of durable, background result reporting so a transient outage can't wedge a worker or lose a completion. See [docs/background-jobs.md](docs/background-jobs.md#worker-liveness).
+
+## Queues
+
+Give a job a queue with `static queue = "..."` (or a `{queue}` job option; the option wins) and cap how many of that queue's jobs run in flight across the whole cluster under `backgroundJobs.queues`:
+
+```js
+backgroundJobs: {
+  queues: {
+    builds: {maxConcurrent: 100}, // I/O-bound: can run well above the core count
+    default: {maxConcurrent: 8}   // CPU-bound: keep near the core count
+  }
+}
+```
+
+A job with no queue runs on `"default"`; a queue with no cap is unlimited. Caps are enforced through the durable per-key concurrency mechanism (the reserved `queue:<name>` key), hold regardless of how many worker processes run, and are reconciled against the existing backlog on startup when you change them. Scheduled jobs honor a job's `static queue` too. See [docs/background-jobs.md](docs/background-jobs.md#queues-per-queue-concurrency-caps).
+
+## Retention
+
+Terminal `background_jobs` rows are not deleted automatically unless you configure retention, so a busy app otherwise grows the table indefinitely. Set `backgroundJobs.retention` to prune old terminal rows:
+
+```js
+backgroundJobs: {
+  retention: {
+    completedTtlMs: 7 * 24 * 60 * 60 * 1000, // default: 7 days (null/0 disables)
+    failedTtlMs: 30 * 24 * 60 * 60 * 1000,   // default: 30 days for failed/orphaned (null/0 disables)
+    batchSize: 1000,                          // default: 1000 rows per delete batch
+    sweepIntervalMs: 60 * 60 * 1000           // default: 1 hour
+  }
+}
+```
+
+`background-jobs-main` registers a built-in `velocious:prune-terminal-background-jobs` job on the scheduler when retention is enabled, so pruning runs as an ordinary scheduled/queued job (it needs a worker, appears in the job tables, and is bounded to one non-overlapping run). See [docs/background-jobs.md](docs/background-jobs.md#retention-pruning-old-job-rows).
+
 ## Dashboard
 
 Velocious ships a mountable read-only HTTP API for inspecting jobs (queued, running, completed, failed, orphaned and scheduled), similar in spirit to `sidekiq-web`. Mount it in your routes file the way `Sidekiq::Web` is mounted in Rails:

package/build/background-jobs/json-socket.js

@@ -30,6 +30,18 @@ export default class JsonSocket extends EventEmitter {
      * Narrows the runtime value to the documented type.
      * @type {boolean} */
     this.acceptsInlineJobs = true
+    /**
+     * Whether this worker advertised heartbeat support in its hello. Only
+     * heartbeat-capable workers are subject to the main's stale-liveness
+     * eviction; a legacy worker (e.g. mid rolling deploy) is exempt so its
+     * active leases are not released while it is still running them.
+     * @type {boolean} */
+    this.supportsHeartbeat = false
+    /**
+     * Last time (ms) the main saw any message from this worker socket; used by
+     * the main's liveness sweep to drop a wedged/silent worker.
+     * @type {number | undefined} */
+    this.lastSeenAt = undefined
     this.buffer = ""
     this.socket.setEncoding("utf8")
     this.socket.on("data", (chunk) => this._onData(String(chunk)))

package/build/background-jobs/main.js

@@ -5,6 +5,7 @@ import JsonSocket from "./json-socket.js"
 import BackgroundJobsScheduler from "./scheduler.js"
 import BackgroundJobsStore from "./store.js"
 import Logger from "../logger.js"
+import PruneTerminalBackgroundJobsJob from "../jobs/prune-terminal-background-jobs.js"
 
 /**
  * Channel used by `background-jobs-main` to coordinate dispatch wake-ups
@@ -21,6 +22,10 @@ const DISPATCH_CHANNEL = "velocious-background-jobs-dispatch"
  * scheduled-job timer here and re-arm when it expires.
  */
 const MAX_TIMER_MS = 2_147_483_647 // ~24.8 days
+/** A worker silent (no heartbeat/ready/report) longer than this is dropped. */
+const WORKER_STALE_TIMEOUT_MS = 60000
+/** How often the main scans workers for staleness. */
+const WORKER_LIVENESS_SWEEP_MS = 15000
 /**
  * WorkerExecutionModeCapability type.
  * @typedef {object} WorkerExecutionModeCapability
@@ -46,8 +51,10 @@ export default class BackgroundJobsMain {
    * @param {import("../configuration.js").default} args.configuration - Configuration.
    * @param {string} [args.host] - Hostname.
    * @param {number} [args.port] - Port.
+   * @param {number} [args.workerStaleTimeoutMs] - Override how long a silent worker may go before being dropped (default 60000ms).
+   * @param {number} [args.workerLivenessSweepMs] - Override how often stale workers are swept for (default 15000ms).
    */
-  constructor({configuration, host, port}) {
+  constructor({configuration, host, port, workerStaleTimeoutMs, workerLivenessSweepMs}) {
     this.configuration = configuration
     const config = configuration.getBackgroundJobsConfig()
     this.host = host || config.host
@@ -55,6 +62,10 @@ export default class BackgroundJobsMain {
     this.dispatchStrategy = config.dispatchStrategy
     this.pollIntervalMs = config.pollIntervalMs
     this.retention = config.retention
+    // A worker that stops sending anything (heartbeat/ready/report) for this
+    // long is treated as wedged/dead: its leases are released and it is dropped.
+    this.workerStaleTimeoutMs = typeof workerStaleTimeoutMs === "number" && workerStaleTimeoutMs >= 1 ? workerStaleTimeoutMs : WORKER_STALE_TIMEOUT_MS
+    this.workerLivenessSweepMs = typeof workerLivenessSweepMs === "number" && workerLivenessSweepMs >= 1 ? workerLivenessSweepMs : WORKER_LIVENESS_SWEEP_MS
     this.store = new BackgroundJobsStore({configuration, databaseIdentifier: config.databaseIdentifier})
     this.logger = new Logger(this)
     /**
@@ -91,8 +102,8 @@ export default class BackgroundJobsMain {
     this._orphanTimer = undefined
     /**
      * Narrows the runtime value to the documented type.
-     * @type {ReturnType<typeof setTimeout> | undefined} */
-    this._retentionTimer = undefined
+     * @type {ReturnType<typeof setInterval> | undefined} */
+    this._workerStaleTimer = undefined
     /**
      * Narrows the runtime value to the documented type.
      * @type {BackgroundJobsScheduler | undefined} */
@@ -144,9 +155,9 @@ export default class BackgroundJobsMain {
         void this._sweepOrphans()
       }, 60000)
 
-      this._retentionTimer = setInterval(() => {
-        void this._sweepRetention()
-      }, this.retention.sweepIntervalMs)
+      this._workerStaleTimer = setInterval(() => {
+        void this._sweepStaleWorkers()
+      }, this.workerLivenessSweepMs)
 
       this.scheduler = new BackgroundJobsScheduler({
         configuration: this.configuration,
@@ -154,7 +165,10 @@ export default class BackgroundJobsMain {
           await this.store.enqueue({
             jobName: jobClass.jobName(),
             args,
-            options
+            // Fold in the job class's static `queue` (as performLater* do) so a
+            // scheduled job with `static queue = "..."` lands on its queue and
+            // honors the configured cap without every schedule repeating it.
+            options: jobClass._withQueue(options)
           })
           this._notifyEnqueued()
           await this._drain()
@@ -162,6 +176,16 @@ export default class BackgroundJobsMain {
       })
       await this.scheduler.start()
 
+      // Retention pruning runs as an ordinary scheduled job on the normal
+      // scheduler (so it is visible in the job tables and dispatched to a
+      // worker), rather than a hidden in-process timer. Skipped when retention
+      // is disabled. The scheduler owns the timer, so scheduler.stop() clears it.
+      const retentionSchedule = PruneTerminalBackgroundJobsJob.scheduleConfiguration(this.retention)
+
+      if (retentionSchedule) {
+        this.scheduler.scheduleJob({jobConfiguration: retentionSchedule, jobKey: "velociousPruneTerminalBackgroundJobs"})
+      }
+
       // Startup catch-up: drain anything that was waiting before this
       // process came up. In beacon mode this is also the safety net for
       // races between attaching the connect listener and the initial
@@ -206,12 +230,12 @@ export default class BackgroundJobsMain {
     if (this._scheduledTimer) clearTimeout(this._scheduledTimer)
     if (this._errorRetryTimer) clearTimeout(this._errorRetryTimer)
     if (this._orphanTimer) clearInterval(this._orphanTimer)
-    if (this._retentionTimer) clearInterval(this._retentionTimer)
+    if (this._workerStaleTimer) clearInterval(this._workerStaleTimer)
     this._pollTimer = undefined
     this._scheduledTimer = undefined
     this._errorRetryTimer = undefined
     this._orphanTimer = undefined
-    this._retentionTimer = undefined
+    this._workerStaleTimer = undefined
   }
 
   /**
@@ -394,6 +418,8 @@ export default class BackgroundJobsMain {
     if (message.role === "worker") {
       jsonSocket.workerId = message.workerId
       jsonSocket.supportsHandoffIdReporting = message.supportsHandoffIdReporting === true
+      jsonSocket.supportsHeartbeat = message.supportsHeartbeat === true
+      jsonSocket.lastSeenAt = Date.now()
       this.workers.add(jsonSocket)
       this.workerHandoffs.set(jsonSocket, new Map())
     }
@@ -422,6 +448,14 @@ export default class BackgroundJobsMain {
    * @returns {void}
    */
   _handleWorkerSocketMessage({jsonSocket, message}) {
+    // Any message from the worker proves it is alive; the liveness sweep uses
+    // this to detect a wedged/silent worker.
+    jsonSocket.lastSeenAt = Date.now()
+
+    if (message?.type === "heartbeat") {
+      return
+    }
+
     if (message?.type === "ready") {
       this._handleWorkerReady({jsonSocket, message})
       return
@@ -1098,22 +1132,43 @@ export default class BackgroundJobsMain {
   }
 
   /**
-   * Deletes terminal job rows past their retention window so the jobs table
-   * does not grow unbounded. Runs on its own interval; failures are logged and
-   * retried on the next tick.
-   * @returns {Promise<void>} - Resolves after one retention sweep.
+   * Drops workers that have gone silent past `workerStaleTimeoutMs` (no
+   * heartbeat, ready, or report). A wedged worker keeps its socket open, so the
+   * `close`-based cleanup never fires and its in-flight leases — and the whole
+   * queue — stay stuck until a human notices. Releasing the lost worker's
+   * leases lets its jobs run elsewhere and stops dispatch to it; the worker's
+   * own process lifecycle is the supervisor's concern.
+   * @returns {Promise<void>} - Resolves after the sweep.
    */
-  async _sweepRetention() {
-    try {
-      const deleted = await this.store.pruneTerminalJobs({
-        completedTtlMs: this.retention.completedTtlMs,
-        failedTtlMs: this.retention.failedTtlMs,
-        batchSize: this.retention.batchSize
-      })
+  async _sweepStaleWorkers() {
+    if (this._stopped) return
 
-      if (deleted > 0) this.logger.warn(() => ["Pruned terminal background jobs", deleted])
-    } catch (error) {
-      this.logger.error(() => ["Failed to prune terminal jobs:", error])
+    const cutoff = Date.now() - this.workerStaleTimeoutMs
+    /** @type {JsonSocket[]} */
+    const stale = []
+
+    for (const worker of this.workers) {
+      // Only evict heartbeat-capable workers. A legacy worker (e.g. one from the
+      // previous release during a rolling deploy) never heartbeats, so evicting
+      // it on silence would wrongly release the leases of a job it is still
+      // running. Its disconnect is still handled by the socket `close` path.
+      if (!worker.supportsHeartbeat) continue
+
+      const lastSeenAt = typeof worker.lastSeenAt === "number" ? worker.lastSeenAt : 0
+
+      if (lastSeenAt <= cutoff) stale.push(worker)
+    }
+
+    for (const worker of stale) {
+      this.logger.warn(() => ["Dropping stale background jobs worker", {workerId: worker.workerId, lastSeenAt: worker.lastSeenAt}])
+
+      try {
+        worker.close()
+      } catch {
+        // Already closing; the lease release below is what matters.
+      }
+
+      await this._handleWorkerSocketClosed(worker)
     }
   }
 }

package/build/background-jobs/store.js

@@ -51,6 +51,7 @@ export default class BackgroundJobsStore {
     this.databaseIdentifier = databaseIdentifier
     this.logger = new Logger(this)
     this._readyPromise = null
+    this._queueConcurrencyReconciled = false
   }
 
   /**
@@ -101,8 +102,26 @@ export default class BackgroundJobsStore {
     const argsJson = JSON.stringify(args || [])
     const queue = this._normalizeQueue(options)
     const concurrency = this._resolveConcurrency(options, queue)
+    /** @type {string} */
+    let resultJobId = jobId
 
     await this._withDb(async (db) => {
+      if (options?.deduplicateWhileQueued && concurrency?.concurrencyKey) {
+        const existing = await db
+          .newQuery()
+          .from(JOBS_TABLE)
+          .select("id")
+          .where({status: "queued", concurrency_key: concurrency.concurrencyKey})
+          .limit(1)
+          .results()
+
+        if (existing[0]) {
+          resultJobId = String(/** @type {Record<string, ?>} */ (existing[0]).id)
+
+          return
+        }
+      }
+
       if (concurrency) {
         if (concurrency.queueDerived) {
           await this._ensureQueueConcurrencyKey(db, concurrency)
@@ -130,7 +149,7 @@ export default class BackgroundJobsStore {
       })
     })
 
-    return jobId
+    return resultJobId
   }
 
   /**
@@ -648,6 +667,7 @@ export default class BackgroundJobsStore {
       if (alreadyApplied && await db.tableExists(JOBS_TABLE)) {
         await this._ensureJobsTableColumns(db)
         await this._ensureConcurrencyTable(db)
+        await this._reconcileQueueConcurrency(db)
         await this._reconcileConcurrency(db)
         return
       }
@@ -655,6 +675,7 @@ export default class BackgroundJobsStore {
       await this._applyMigrations(db)
       await this._ensureJobsTableColumns(db)
       await this._ensureConcurrencyTable(db)
+      await this._reconcileQueueConcurrency(db)
       await this._reconcileConcurrency(db)
 
       if (alreadyApplied) return
@@ -834,16 +855,15 @@ export default class BackgroundJobsStore {
    * @returns {Promise<void>} - Resolves when ensured.
    */
   async _ensureQueueColumn(db) {
-    const table = await db.getTableByNameOrFail(JOBS_TABLE)
-
-    if (await table.getColumnByName("queue")) return
-
     const lockName = `${MIGRATION_SCOPE}:queue_column`
     const acquired = await db.acquireAdvisoryLock(lockName)
 
     if (!acquired) throw new Error("Failed to acquire background jobs queue schema lock")
 
     try {
+      // SQL Server schema reads can deadlock with a concurrent ALTER TABLE, so
+      // acquire the lock before inspecting the column rather than only
+      // protecting the mutation (mirrors the concurrency-column migration).
       db.clearSchemaCache()
       const lockedTable = await db.getTableByNameOrFail(JOBS_TABLE)
 
@@ -1101,6 +1121,9 @@ export default class BackgroundJobsStore {
     if (typeof key !== "string" || key.length === 0 || !Number.isInteger(cap) || Number(cap) <= 0) {
       throw new Error("background job concurrencyKey and maxConcurrency must be paired; concurrencyKey must be non-empty and maxConcurrency must be a positive integer")
     }
+    if (key.startsWith(QUEUE_CONCURRENCY_KEY_PREFIX)) {
+      throw new Error(`background job concurrencyKey must not start with the reserved "${QUEUE_CONCURRENCY_KEY_PREFIX}" prefix, which is reserved for queue-derived concurrency caps`)
+    }
     return {concurrencyKey: key, maxConcurrency: Number(cap)}
   }
 
@@ -1277,6 +1300,65 @@ export default class BackgroundJobsStore {
     )
   }
 
+  /**
+   * Reconciles queue-derived concurrency with the current configuration, once
+   * per process. Enqueue only consults config for new jobs, so a cap added,
+   * removed, or changed while a backlog exists otherwise leaves persisted rows
+   * stale: pre-cap jobs keep a null key and bypass the cap, post-removal jobs
+   * stay capped under a now-unconfigured key, and a changed numeric cap stays
+   * stale until the next enqueue. Bring the durable state in line with config:
+   * sync each configured queue's stored cap, adopt not-yet-keyed non-terminal
+   * jobs onto their queue key, and release non-terminal jobs from queue keys
+   * whose queue is no longer capped. Runs before {@link _reconcileConcurrency}
+   * so the rebuilt active counts reflect the adopted/released keys.
+   * @param {import("../database/drivers/base.js").default} db - Database connection.
+   * @returns {Promise<void>} - Resolves when reconciled.
+   */
+  async _reconcileQueueConcurrency(db) {
+    if (this._queueConcurrencyReconciled) return
+    if (!(await db.tableExists(CONCURRENCY_TABLE))) return
+
+    const queuesConfig = this.configuration.getBackgroundJobsConfig().queues || {}
+    const jobsTable = db.quoteTable(JOBS_TABLE)
+    const keyColumn = db.quoteColumn("concurrency_key")
+    const capColumn = db.quoteColumn("max_concurrency")
+    const queueColumn = db.quoteColumn("queue")
+    const nonTerminal = `${db.quoteColumn("status")} IN (${db.quote("queued")}, ${db.quote("handed_off")})`
+    /** @type {Set<string>} */
+    const cappedQueues = new Set()
+
+    for (const queue of Object.keys(queuesConfig)) {
+      const cap = this._queueMaxConcurrency(queue)
+
+      if (cap === null) continue
+
+      cappedQueues.add(queue)
+      const concurrencyKey = `${QUEUE_CONCURRENCY_KEY_PREFIX}${queue}`
+
+      await this._ensureQueueConcurrencyKey(db, {concurrencyKey, maxConcurrency: cap})
+      await db.query(
+        `UPDATE ${jobsTable} SET ${keyColumn} = ${db.quote(concurrencyKey)}, ${capColumn} = ${Number(cap)} ` +
+        `WHERE ${queueColumn} = ${db.quote(queue)} AND ${keyColumn} IS NULL AND ${nonTerminal}`
+      )
+    }
+
+    const concurrencyRows = await db.newQuery().from(CONCURRENCY_TABLE).select("concurrency_key").results()
+
+    for (const row of concurrencyRows) {
+      const concurrencyKey = String(/** @type {Record<string, ?>} */ (row).concurrency_key)
+
+      if (!concurrencyKey.startsWith(QUEUE_CONCURRENCY_KEY_PREFIX)) continue
+      if (cappedQueues.has(concurrencyKey.slice(QUEUE_CONCURRENCY_KEY_PREFIX.length))) continue
+
+      await db.query(
+        `UPDATE ${jobsTable} SET ${keyColumn} = NULL, ${capColumn} = NULL ` +
+        `WHERE ${keyColumn} = ${db.quote(concurrencyKey)} AND ${nonTerminal}`
+      )
+    }
+
+    this._queueConcurrencyReconciled = true
+  }
+
   /**
    * Runs normalize number.
    * @param {?} value - Input value.

package/build/background-jobs/types.js

@@ -16,6 +16,7 @@
  * @property {string} [queue] - Queue name. Defaults to `"default"`. When the queue has a configured cap in `backgroundJobs.queues`, that cap is enforced cluster-wide.
  * @property {string} [concurrencyKey] - Opaque non-empty key used to share a concurrency cap. Overrides any queue-derived cap.
  * @property {number} [maxConcurrency] - Positive integer cap; must be paired with `concurrencyKey`.
+ * @property {boolean} [deduplicateWhileQueued] - When true (requires `concurrencyKey`), skip the enqueue if a still-queued job with the same `concurrencyKey` already exists, returning that job's id. Keeps an interval-scheduled recurring job (e.g. retention pruning) from piling up redundant queued rows when it runs slower than its interval or no worker is free.
  */
 /**
  * @typedef {object} BackgroundJobPayload
@@ -65,9 +66,10 @@
  * @typedef {"worker" | "client" | "reporter"} BackgroundJobSocketRole
  */
 /**
- * @typedef {{type: "hello", role: BackgroundJobSocketRole, supportsHandoffIdReporting?: boolean, workerId?: string}} BackgroundJobHelloMessage
+ * @typedef {{type: "hello", role: BackgroundJobSocketRole, supportsHandoffIdReporting?: boolean, supportsHeartbeat?: boolean, workerId?: string}} BackgroundJobHelloMessage
  * @typedef {{type: "ready", acceptsForked?: boolean, acceptsInline?: boolean, acceptsSpawned?: boolean}} BackgroundJobReadyMessage
  * @typedef {{type: "draining"}} BackgroundJobDrainingMessage
+ * @typedef {{type: "heartbeat", workerId?: string}} BackgroundJobHeartbeatMessage
  * @typedef {{type: "enqueue", jobName: string, args?: Array<?>, options?: BackgroundJobOptions}} BackgroundJobEnqueueMessage
  * @typedef {{type: "enqueued", jobId: string}} BackgroundJobEnqueuedMessage
  * @typedef {{type: "enqueue-error", error?: string}} BackgroundJobEnqueueErrorMessage
@@ -78,7 +80,7 @@
  * @typedef {{type: "job-update-error", jobId: string, error?: string}} BackgroundJobUpdateErrorMessage
  */
 /**
- * @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
+ * @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobHeartbeatMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
  */
 
 export const nothing = {}