velocious 1.0.518 → 1.0.520

package/README.md

@@ -440,7 +440,7 @@ This creates `src/frontend-models/user.js` (and one file per configured resource
 - Attribute methods like `user.name()` and `user.setName(...)`
 - Relationship helpers (when `relationships` are configured), for example `task.project()`, `await task.projectOrLoad()`, `await project.tasks().toArray()`, `await project.tasks().load()`, and `project.tasks().build({...})`
 - Preload relationships onto records you already have with `await record.preload(Model.preload({...}).select({...}))` (or `Preloader.preload(records, ...)` for arrays), including `selectsExtra(...)` and a `{force: true}` reload option — see [docs/frontend-models.md](docs/frontend-models.md#preloading-onto-loaded-records)
-- Attachment helpers (when `attachments` are configured), for example `await task.descriptionFile().attach(file)`, `await task.descriptionFile().download()`, and `await task.update({descriptionFile: file})`
+- Attachment helpers (when `attachments` are configured), for example `await task.descriptionFile().attach(file)`, `await task.descriptionFile().download()`, `await task.files().purgeAll()`, and `await task.update({descriptionFile: file})`
 
 React components can subscribe to lifecycle broadcasts without manual cleanup code:
 
@@ -501,6 +501,14 @@ await task.update({
 })
 ```
 
+Purge a record's attachments — both the stored files and their rows — for example before destroying the owner record:
+
+```js
+const purgedCount = await task.files().purgeAll()
+```
+
+`purgeAll()` deletes each attachment's backing storage and then its row, and removes only the attachments that existed when the purge started (a concurrent `attach()` for the same record/name is left intact). It throws without deleting anything if a storage driver has no `delete` operation, so a driver configured without deletion can never silently leak storage. It is a no-op for unpersisted records and returns the number of attachments purged.
+
 Configure attachment storage drivers in `Configuration`:
 
 ```js
@@ -2123,6 +2131,40 @@ await MyJob.performLaterWithOptions({
 
 If a handed-off job does not report back within 2 hours, it is marked orphaned and re-queued if retries remain.
 
+Workers also send periodic heartbeats (and use TCP keepalive) so the main can drop a wedged or half-open worker that never fires a socket `close` and release its leases, and job slots are freed independently of durable, background result reporting so a transient outage can't wedge a worker or lose a completion. See [docs/background-jobs.md](docs/background-jobs.md#worker-liveness).
+
+## Queues
+
+Give a job a queue with `static queue = "..."` (or a `{queue}` job option; the option wins) and cap how many of that queue's jobs run in flight across the whole cluster under `backgroundJobs.queues`:
+
+```js
+backgroundJobs: {
+  queues: {
+    builds: {maxConcurrent: 100}, // I/O-bound: can run well above the core count
+    default: {maxConcurrent: 8}   // CPU-bound: keep near the core count
+  }
+}
+```
+
+A job with no queue runs on `"default"`; a queue with no cap is unlimited. Caps are enforced through the durable per-key concurrency mechanism (the reserved `queue:<name>` key), hold regardless of how many worker processes run, and are reconciled against the existing backlog on startup when you change them. Scheduled jobs honor a job's `static queue` too. See [docs/background-jobs.md](docs/background-jobs.md#queues-per-queue-concurrency-caps).
+
+## Retention
+
+Terminal `background_jobs` rows are not deleted automatically unless you configure retention, so a busy app otherwise grows the table indefinitely. Set `backgroundJobs.retention` to prune old terminal rows:
+
+```js
+backgroundJobs: {
+  retention: {
+    completedTtlMs: 7 * 24 * 60 * 60 * 1000, // default: 7 days (null/0 disables)
+    failedTtlMs: 30 * 24 * 60 * 60 * 1000,   // default: 30 days for failed/orphaned (null/0 disables)
+    batchSize: 1000,                          // default: 1000 rows per delete batch
+    sweepIntervalMs: 60 * 60 * 1000           // default: 1 hour
+  }
+}
+```
+
+`background-jobs-main` registers a built-in `velocious:prune-terminal-background-jobs` job on the scheduler when retention is enabled, so pruning runs as an ordinary scheduled/queued job (it needs a worker, appears in the job tables, and is bounded to one non-overlapping run). See [docs/background-jobs.md](docs/background-jobs.md#retention-pruning-old-job-rows).
+
 ## Dashboard
 
 Velocious ships a mountable read-only HTTP API for inspecting jobs (queued, running, completed, failed, orphaned and scheduled), similar in spirit to `sidekiq-web`. Mount it in your routes file the way `Sidekiq::Web` is mounted in Rails:

package/build/background-jobs/json-socket.js

@@ -30,6 +30,18 @@ export default class JsonSocket extends EventEmitter {
      * Narrows the runtime value to the documented type.
      * @type {boolean} */
     this.acceptsInlineJobs = true
+    /**
+     * Whether this worker advertised heartbeat support in its hello. Only
+     * heartbeat-capable workers are subject to the main's stale-liveness
+     * eviction; a legacy worker (e.g. mid rolling deploy) is exempt so its
+     * active leases are not released while it is still running them.
+     * @type {boolean} */
+    this.supportsHeartbeat = false
+    /**
+     * Last time (ms) the main saw any message from this worker socket; used by
+     * the main's liveness sweep to drop a wedged/silent worker.
+     * @type {number | undefined} */
+    this.lastSeenAt = undefined
     this.buffer = ""
     this.socket.setEncoding("utf8")
     this.socket.on("data", (chunk) => this._onData(String(chunk)))

package/build/background-jobs/main.js

@@ -5,6 +5,7 @@ import JsonSocket from "./json-socket.js"
 import BackgroundJobsScheduler from "./scheduler.js"
 import BackgroundJobsStore from "./store.js"
 import Logger from "../logger.js"
+import PruneTerminalBackgroundJobsJob from "../jobs/prune-terminal-background-jobs.js"
 
 /**
  * Channel used by `background-jobs-main` to coordinate dispatch wake-ups
@@ -21,6 +22,10 @@ const DISPATCH_CHANNEL = "velocious-background-jobs-dispatch"
  * scheduled-job timer here and re-arm when it expires.
  */
 const MAX_TIMER_MS = 2_147_483_647 // ~24.8 days
+/** A worker silent (no heartbeat/ready/report) longer than this is dropped. */
+const WORKER_STALE_TIMEOUT_MS = 60000
+/** How often the main scans workers for staleness. */
+const WORKER_LIVENESS_SWEEP_MS = 15000
 /**
  * WorkerExecutionModeCapability type.
  * @typedef {object} WorkerExecutionModeCapability
@@ -46,14 +51,21 @@ export default class BackgroundJobsMain {
    * @param {import("../configuration.js").default} args.configuration - Configuration.
    * @param {string} [args.host] - Hostname.
    * @param {number} [args.port] - Port.
+   * @param {number} [args.workerStaleTimeoutMs] - Override how long a silent worker may go before being dropped (default 60000ms).
+   * @param {number} [args.workerLivenessSweepMs] - Override how often stale workers are swept for (default 15000ms).
    */
-  constructor({configuration, host, port}) {
+  constructor({configuration, host, port, workerStaleTimeoutMs, workerLivenessSweepMs}) {
     this.configuration = configuration
     const config = configuration.getBackgroundJobsConfig()
     this.host = host || config.host
     this.port = typeof port === "number" ? port : config.port
     this.dispatchStrategy = config.dispatchStrategy
     this.pollIntervalMs = config.pollIntervalMs
+    this.retention = config.retention
+    // A worker that stops sending anything (heartbeat/ready/report) for this
+    // long is treated as wedged/dead: its leases are released and it is dropped.
+    this.workerStaleTimeoutMs = typeof workerStaleTimeoutMs === "number" && workerStaleTimeoutMs >= 1 ? workerStaleTimeoutMs : WORKER_STALE_TIMEOUT_MS
+    this.workerLivenessSweepMs = typeof workerLivenessSweepMs === "number" && workerLivenessSweepMs >= 1 ? workerLivenessSweepMs : WORKER_LIVENESS_SWEEP_MS
     this.store = new BackgroundJobsStore({configuration, databaseIdentifier: config.databaseIdentifier})
     this.logger = new Logger(this)
     /**
@@ -88,6 +100,10 @@ export default class BackgroundJobsMain {
      * Narrows the runtime value to the documented type.
      * @type {ReturnType<typeof setTimeout> | undefined} */
     this._orphanTimer = undefined
+    /**
+     * Narrows the runtime value to the documented type.
+     * @type {ReturnType<typeof setInterval> | undefined} */
+    this._workerStaleTimer = undefined
     /**
      * Narrows the runtime value to the documented type.
      * @type {BackgroundJobsScheduler | undefined} */
@@ -139,13 +155,20 @@ export default class BackgroundJobsMain {
         void this._sweepOrphans()
       }, 60000)
 
+      this._workerStaleTimer = setInterval(() => {
+        void this._sweepStaleWorkers()
+      }, this.workerLivenessSweepMs)
+
       this.scheduler = new BackgroundJobsScheduler({
         configuration: this.configuration,
         enqueueJob: async ({args, jobClass, options}) => {
           await this.store.enqueue({
             jobName: jobClass.jobName(),
             args,
-            options
+            // Fold in the job class's static `queue` (as performLater* do) so a
+            // scheduled job with `static queue = "..."` lands on its queue and
+            // honors the configured cap without every schedule repeating it.
+            options: jobClass._withQueue(options)
           })
           this._notifyEnqueued()
           await this._drain()
@@ -153,6 +176,16 @@ export default class BackgroundJobsMain {
       })
       await this.scheduler.start()
 
+      // Retention pruning runs as an ordinary scheduled job on the normal
+      // scheduler (so it is visible in the job tables and dispatched to a
+      // worker), rather than a hidden in-process timer. Skipped when retention
+      // is disabled. The scheduler owns the timer, so scheduler.stop() clears it.
+      const retentionSchedule = PruneTerminalBackgroundJobsJob.scheduleConfiguration(this.retention)
+
+      if (retentionSchedule) {
+        this.scheduler.scheduleJob({jobConfiguration: retentionSchedule, jobKey: "velociousPruneTerminalBackgroundJobs"})
+      }
+
       // Startup catch-up: drain anything that was waiting before this
       // process came up. In beacon mode this is also the safety net for
       // races between attaching the connect listener and the initial
@@ -197,10 +230,12 @@ export default class BackgroundJobsMain {
     if (this._scheduledTimer) clearTimeout(this._scheduledTimer)
     if (this._errorRetryTimer) clearTimeout(this._errorRetryTimer)
     if (this._orphanTimer) clearInterval(this._orphanTimer)
+    if (this._workerStaleTimer) clearInterval(this._workerStaleTimer)
     this._pollTimer = undefined
     this._scheduledTimer = undefined
     this._errorRetryTimer = undefined
     this._orphanTimer = undefined
+    this._workerStaleTimer = undefined
   }
 
   /**
@@ -383,6 +418,8 @@ export default class BackgroundJobsMain {
     if (message.role === "worker") {
       jsonSocket.workerId = message.workerId
       jsonSocket.supportsHandoffIdReporting = message.supportsHandoffIdReporting === true
+      jsonSocket.supportsHeartbeat = message.supportsHeartbeat === true
+      jsonSocket.lastSeenAt = Date.now()
       this.workers.add(jsonSocket)
       this.workerHandoffs.set(jsonSocket, new Map())
     }
@@ -411,6 +448,14 @@ export default class BackgroundJobsMain {
    * @returns {void}
    */
   _handleWorkerSocketMessage({jsonSocket, message}) {
+    // Any message from the worker proves it is alive; the liveness sweep uses
+    // this to detect a wedged/silent worker.
+    jsonSocket.lastSeenAt = Date.now()
+
+    if (message?.type === "heartbeat") {
+      return
+    }
+
     if (message?.type === "ready") {
       this._handleWorkerReady({jsonSocket, message})
       return
@@ -1085,4 +1130,45 @@ export default class BackgroundJobsMain {
       this.logger.error(() => ["Failed to mark orphaned jobs:", error])
     }
   }
+
+  /**
+   * Drops workers that have gone silent past `workerStaleTimeoutMs` (no
+   * heartbeat, ready, or report). A wedged worker keeps its socket open, so the
+   * `close`-based cleanup never fires and its in-flight leases — and the whole
+   * queue — stay stuck until a human notices. Releasing the lost worker's
+   * leases lets its jobs run elsewhere and stops dispatch to it; the worker's
+   * own process lifecycle is the supervisor's concern.
+   * @returns {Promise<void>} - Resolves after the sweep.
+   */
+  async _sweepStaleWorkers() {
+    if (this._stopped) return
+
+    const cutoff = Date.now() - this.workerStaleTimeoutMs
+    /** @type {JsonSocket[]} */
+    const stale = []
+
+    for (const worker of this.workers) {
+      // Only evict heartbeat-capable workers. A legacy worker (e.g. one from the
+      // previous release during a rolling deploy) never heartbeats, so evicting
+      // it on silence would wrongly release the leases of a job it is still
+      // running. Its disconnect is still handled by the socket `close` path.
+      if (!worker.supportsHeartbeat) continue
+
+      const lastSeenAt = typeof worker.lastSeenAt === "number" ? worker.lastSeenAt : 0
+
+      if (lastSeenAt <= cutoff) stale.push(worker)
+    }
+
+    for (const worker of stale) {
+      this.logger.warn(() => ["Dropping stale background jobs worker", {workerId: worker.workerId, lastSeenAt: worker.lastSeenAt}])
+
+      try {
+        worker.close()
+      } catch {
+        // Already closing; the lease release below is what matters.
+      }
+
+      await this._handleWorkerSocketClosed(worker)
+    }
+  }
 }

package/build/background-jobs/store.js

@@ -51,6 +51,7 @@ export default class BackgroundJobsStore {
     this.databaseIdentifier = databaseIdentifier
     this.logger = new Logger(this)
     this._readyPromise = null
+    this._queueConcurrencyReconciled = false
   }
 
   /**
@@ -101,8 +102,26 @@ export default class BackgroundJobsStore {
     const argsJson = JSON.stringify(args || [])
     const queue = this._normalizeQueue(options)
     const concurrency = this._resolveConcurrency(options, queue)
+    /** @type {string} */
+    let resultJobId = jobId
 
     await this._withDb(async (db) => {
+      if (options?.deduplicateWhileQueued && concurrency?.concurrencyKey) {
+        const existing = await db
+          .newQuery()
+          .from(JOBS_TABLE)
+          .select("id")
+          .where({status: "queued", concurrency_key: concurrency.concurrencyKey})
+          .limit(1)
+          .results()
+
+        if (existing[0]) {
+          resultJobId = String(/** @type {Record<string, ?>} */ (existing[0]).id)
+
+          return
+        }
+      }
+
       if (concurrency) {
         if (concurrency.queueDerived) {
           await this._ensureQueueConcurrencyKey(db, concurrency)
@@ -130,7 +149,7 @@ export default class BackgroundJobsStore {
       })
     })
 
-    return jobId
+    return resultJobId
   }
 
   /**
@@ -476,11 +495,25 @@ export default class BackgroundJobsStore {
       for (const row of rows) {
         const job = this._normalizeJobRow(row)
 
+        // Fence the reclaim on the exact handoff this sweep selected, using its
+        // `handed_off_at_ms` rather than its `handoff_id`. Two reasons:
+        //   1. Null-safe. Some rows have a null `handoff_id` (handed off by an
+        //      older velocious before handoff-id fencing). `{handoff_id: null}`
+        //      renders as `handoff_id = NULL`, which matches nothing, so those
+        //      rows would be stranded in `handed_off` forever.
+        //   2. Race-safe. If the row is returned to the queue and re-handed-off
+        //      between the SELECT above and this update, it gets a fresh
+        //      `handed_off_at_ms` (always "now"), so this stale cutoff-era
+        //      timestamp no longer matches and we won't fail/orphan — or
+        //      wrongly release the concurrency reservation of — that new lease.
+        // `handed_off_at_ms` is always set on a handed-off row (and the SELECT
+        // required it `<= cutoff`), so it is a reliable null-safe lease pin.
         const orphanedJob = await this._applyFailure({
           db,
           job,
           error: "Job orphaned after timeout",
-          markOrphaned: true
+          markOrphaned: true,
+          conditions: {id: job.id, status: "handed_off", handed_off_at_ms: job.handedOffAtMs}
         })
 
         if (orphanedJob) orphanedCount += 1
@@ -490,6 +523,78 @@ export default class BackgroundJobsStore {
     })
   }
 
+  /**
+   * Deletes terminal job rows past their retention window so the jobs table
+   * does not grow unbounded (completed rows in particular accumulate forever
+   * otherwise). Batched by id — SELECT a page of ids, then
+   * `DELETE ... WHERE id IN (...)` — rather than `DELETE ... LIMIT`, which not
+   * every driver supports; each batch runs on its own connection so the sweep
+   * yields between batches instead of holding one long transaction.
+   * @param {object} [args] - Options.
+   * @param {number | null} [args.completedTtlMs] - Delete `completed` jobs whose `completed_at_ms` is older than this many ms. Falsy or `<= 0` disables completed pruning.
+   * @param {number | null} [args.failedTtlMs] - Delete terminal `failed`/`orphaned` jobs older than this many ms (by `failed_at_ms`/`orphaned_at_ms`). Falsy or `<= 0` disables.
+   * @param {number} [args.batchSize] - Max rows deleted per batch. Default `1000`.
+   * @returns {Promise<number>} - Total rows deleted.
+   */
+  async pruneTerminalJobs({completedTtlMs = null, failedTtlMs = null, batchSize = 1000} = {}) {
+    await this.ensureReady()
+
+    const now = Date.now()
+    const size = batchSize > 0 ? batchSize : 1000
+    let deleted = 0
+
+    if (completedTtlMs && completedTtlMs > 0) {
+      deleted += await this._pruneStatusBatches({status: "completed", column: "completed_at_ms", cutoff: now - completedTtlMs, batchSize: size})
+    }
+
+    if (failedTtlMs && failedTtlMs > 0) {
+      deleted += await this._pruneStatusBatches({status: "failed", column: "failed_at_ms", cutoff: now - failedTtlMs, batchSize: size})
+      deleted += await this._pruneStatusBatches({status: "orphaned", column: "orphaned_at_ms", cutoff: now - failedTtlMs, batchSize: size})
+    }
+
+    return deleted
+  }
+
+  /**
+   * Deletes rows of one terminal status older than a cutoff, batch by batch,
+   * until a page returns fewer than `batchSize` rows.
+   * @param {object} args - Options.
+   * @param {string} args.status - Terminal status to prune.
+   * @param {string} args.column - Timestamp column compared against the cutoff.
+   * @param {number} args.cutoff - Delete rows whose column value is `<= cutoff`.
+   * @param {number} args.batchSize - Max rows per batch.
+   * @returns {Promise<number>} - Rows deleted for this status.
+   */
+  async _pruneStatusBatches({status, column, cutoff, batchSize}) {
+    let deleted = 0
+
+    for (;;) {
+      const removed = await this._withDb(async (db) => {
+        const rows = await db
+          .newQuery()
+          .from(JOBS_TABLE)
+          .select("id")
+          .where({status})
+          .where(`${db.quoteColumn(column)} <= ${db.quote(cutoff)}`)
+          .limit(batchSize)
+          .results()
+
+        if (rows.length === 0) return 0
+
+        const ids = rows.map((/** @type {Record<string, ?>} */ row) => db.quote(String(row.id))).join(", ")
+
+        await db.query(`DELETE FROM ${db.quoteTable(JOBS_TABLE)} WHERE ${db.quoteColumn("id")} IN (${ids})`)
+
+        return rows.length
+      })
+
+      deleted += removed
+      if (removed < batchSize) break
+    }
+
+    return deleted
+  }
+
   /**
    * Runs clear all.
    * @returns {Promise<void>} - Resolves when cleared.
@@ -562,6 +667,7 @@ export default class BackgroundJobsStore {
       if (alreadyApplied && await db.tableExists(JOBS_TABLE)) {
         await this._ensureJobsTableColumns(db)
         await this._ensureConcurrencyTable(db)
+        await this._reconcileQueueConcurrency(db)
         await this._reconcileConcurrency(db)
         return
       }
@@ -569,6 +675,7 @@ export default class BackgroundJobsStore {
       await this._applyMigrations(db)
       await this._ensureJobsTableColumns(db)
       await this._ensureConcurrencyTable(db)
+      await this._reconcileQueueConcurrency(db)
       await this._reconcileConcurrency(db)
 
       if (alreadyApplied) return
@@ -748,16 +855,15 @@ export default class BackgroundJobsStore {
    * @returns {Promise<void>} - Resolves when ensured.
    */
   async _ensureQueueColumn(db) {
-    const table = await db.getTableByNameOrFail(JOBS_TABLE)
-
-    if (await table.getColumnByName("queue")) return
-
     const lockName = `${MIGRATION_SCOPE}:queue_column`
     const acquired = await db.acquireAdvisoryLock(lockName)
 
     if (!acquired) throw new Error("Failed to acquire background jobs queue schema lock")
 
     try {
+      // SQL Server schema reads can deadlock with a concurrent ALTER TABLE, so
+      // acquire the lock before inspecting the column rather than only
+      // protecting the mutation (mirrors the concurrency-column migration).
       db.clearSchemaCache()
       const lockedTable = await db.getTableByNameOrFail(JOBS_TABLE)
 
@@ -869,9 +975,10 @@ export default class BackgroundJobsStore {
    * @param {import("./types.js").BackgroundJobRow} args.job - Job row.
    * @param {?} args.error - Error.
    * @param {boolean} args.markOrphaned - Whether marking orphaned.
+   * @param {Record<string, ?>} [args.conditions] - Update fencing conditions. Defaults to the active-handoff lease match; the time-based orphan sweep overrides this with an id/status match so it can reclaim rows whose `handoff_id` is null (e.g. handed off by an older velocious before handoff-id fencing existed).
    * @returns {Promise<import("./types.js").BackgroundJobRow | null>} - Updated job row when the lease transition won.
    */
-  async _applyFailure({db, job, error, markOrphaned}) {
+  async _applyFailure({db, job, error, markOrphaned, conditions}) {
     const now = Date.now()
     const nextAttempt = (job.attempts || 0) + 1
     const maxRetries = this._normalizeMaxRetries(job.maxRetries)
@@ -890,7 +997,7 @@ export default class BackgroundJobsStore {
     const affectedRows = await this._updateAffectedRows(db, {
       tableName: JOBS_TABLE,
       data: update,
-      conditions: this._activeHandoffConditions(job)
+      conditions: conditions ?? this._activeHandoffConditions(job)
     })
 
     if (affectedRows !== 1) return null
@@ -1014,6 +1121,9 @@ export default class BackgroundJobsStore {
     if (typeof key !== "string" || key.length === 0 || !Number.isInteger(cap) || Number(cap) <= 0) {
       throw new Error("background job concurrencyKey and maxConcurrency must be paired; concurrencyKey must be non-empty and maxConcurrency must be a positive integer")
     }
+    if (key.startsWith(QUEUE_CONCURRENCY_KEY_PREFIX)) {
+      throw new Error(`background job concurrencyKey must not start with the reserved "${QUEUE_CONCURRENCY_KEY_PREFIX}" prefix, which is reserved for queue-derived concurrency caps`)
+    }
     return {concurrencyKey: key, maxConcurrency: Number(cap)}
   }
 
@@ -1190,6 +1300,65 @@ export default class BackgroundJobsStore {
     )
   }
 
+  /**
+   * Reconciles queue-derived concurrency with the current configuration, once
+   * per process. Enqueue only consults config for new jobs, so a cap added,
+   * removed, or changed while a backlog exists otherwise leaves persisted rows
+   * stale: pre-cap jobs keep a null key and bypass the cap, post-removal jobs
+   * stay capped under a now-unconfigured key, and a changed numeric cap stays
+   * stale until the next enqueue. Bring the durable state in line with config:
+   * sync each configured queue's stored cap, adopt not-yet-keyed non-terminal
+   * jobs onto their queue key, and release non-terminal jobs from queue keys
+   * whose queue is no longer capped. Runs before {@link _reconcileConcurrency}
+   * so the rebuilt active counts reflect the adopted/released keys.
+   * @param {import("../database/drivers/base.js").default} db - Database connection.
+   * @returns {Promise<void>} - Resolves when reconciled.
+   */
+  async _reconcileQueueConcurrency(db) {
+    if (this._queueConcurrencyReconciled) return
+    if (!(await db.tableExists(CONCURRENCY_TABLE))) return
+
+    const queuesConfig = this.configuration.getBackgroundJobsConfig().queues || {}
+    const jobsTable = db.quoteTable(JOBS_TABLE)
+    const keyColumn = db.quoteColumn("concurrency_key")
+    const capColumn = db.quoteColumn("max_concurrency")
+    const queueColumn = db.quoteColumn("queue")
+    const nonTerminal = `${db.quoteColumn("status")} IN (${db.quote("queued")}, ${db.quote("handed_off")})`
+    /** @type {Set<string>} */
+    const cappedQueues = new Set()
+
+    for (const queue of Object.keys(queuesConfig)) {
+      const cap = this._queueMaxConcurrency(queue)
+
+      if (cap === null) continue
+
+      cappedQueues.add(queue)
+      const concurrencyKey = `${QUEUE_CONCURRENCY_KEY_PREFIX}${queue}`
+
+      await this._ensureQueueConcurrencyKey(db, {concurrencyKey, maxConcurrency: cap})
+      await db.query(
+        `UPDATE ${jobsTable} SET ${keyColumn} = ${db.quote(concurrencyKey)}, ${capColumn} = ${Number(cap)} ` +
+        `WHERE ${queueColumn} = ${db.quote(queue)} AND ${keyColumn} IS NULL AND ${nonTerminal}`
+      )
+    }
+
+    const concurrencyRows = await db.newQuery().from(CONCURRENCY_TABLE).select("concurrency_key").results()
+
+    for (const row of concurrencyRows) {
+      const concurrencyKey = String(/** @type {Record<string, ?>} */ (row).concurrency_key)
+
+      if (!concurrencyKey.startsWith(QUEUE_CONCURRENCY_KEY_PREFIX)) continue
+      if (cappedQueues.has(concurrencyKey.slice(QUEUE_CONCURRENCY_KEY_PREFIX.length))) continue
+
+      await db.query(
+        `UPDATE ${jobsTable} SET ${keyColumn} = NULL, ${capColumn} = NULL ` +
+        `WHERE ${keyColumn} = ${db.quote(concurrencyKey)} AND ${nonTerminal}`
+      )
+    }
+
+    this._queueConcurrencyReconciled = true
+  }
+
   /**
    * Runs normalize number.
    * @param {?} value - Input value.

package/build/background-jobs/types.js

@@ -16,6 +16,7 @@
  * @property {string} [queue] - Queue name. Defaults to `"default"`. When the queue has a configured cap in `backgroundJobs.queues`, that cap is enforced cluster-wide.
  * @property {string} [concurrencyKey] - Opaque non-empty key used to share a concurrency cap. Overrides any queue-derived cap.
  * @property {number} [maxConcurrency] - Positive integer cap; must be paired with `concurrencyKey`.
+ * @property {boolean} [deduplicateWhileQueued] - When true (requires `concurrencyKey`), skip the enqueue if a still-queued job with the same `concurrencyKey` already exists, returning that job's id. Keeps an interval-scheduled recurring job (e.g. retention pruning) from piling up redundant queued rows when it runs slower than its interval or no worker is free.
  */
 /**
  * @typedef {object} BackgroundJobPayload
@@ -65,9 +66,10 @@
  * @typedef {"worker" | "client" | "reporter"} BackgroundJobSocketRole
  */
 /**
- * @typedef {{type: "hello", role: BackgroundJobSocketRole, supportsHandoffIdReporting?: boolean, workerId?: string}} BackgroundJobHelloMessage
+ * @typedef {{type: "hello", role: BackgroundJobSocketRole, supportsHandoffIdReporting?: boolean, supportsHeartbeat?: boolean, workerId?: string}} BackgroundJobHelloMessage
  * @typedef {{type: "ready", acceptsForked?: boolean, acceptsInline?: boolean, acceptsSpawned?: boolean}} BackgroundJobReadyMessage
  * @typedef {{type: "draining"}} BackgroundJobDrainingMessage
+ * @typedef {{type: "heartbeat", workerId?: string}} BackgroundJobHeartbeatMessage
  * @typedef {{type: "enqueue", jobName: string, args?: Array<?>, options?: BackgroundJobOptions}} BackgroundJobEnqueueMessage
  * @typedef {{type: "enqueued", jobId: string}} BackgroundJobEnqueuedMessage
  * @typedef {{type: "enqueue-error", error?: string}} BackgroundJobEnqueueErrorMessage
@@ -78,7 +80,7 @@
  * @typedef {{type: "job-update-error", jobId: string, error?: string}} BackgroundJobUpdateErrorMessage
  */
 /**
- * @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
+ * @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobHeartbeatMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
  */
 
 export const nothing = {}