velocious 1.0.538 → 1.0.540

package/package.json

@@ -3,7 +3,7 @@
     "velocious": "build/bin/velocious.js"
   },
   "name": "velocious",
-  "version": "1.0.538",
+  "version": "1.0.540",
   "main": "build/index.js",
   "types": "build/index.d.ts",
   "files": [

package/src/background-jobs/scheduler.js

@@ -261,11 +261,16 @@ export default class BackgroundJobsScheduler {
    */
   async enqueueScheduledJob({jobConfiguration, jobKey}) {
     try {
+      // De-duplicate scheduled enqueues by default: a periodic job still pending from an earlier
+      // tick is not enqueued again, which is what let the background_jobs table fill with thousands
+      // of identical scheduled jobs when the queue backed up. Dedup is by job identity (see the
+      // store), so the job keeps its queue-derived concurrency cap. A schedule can opt out with
+      // `deduplicateWhileQueued: false`.
       await this.enqueueJob({
         args: Array.isArray(jobConfiguration.args) ? jobConfiguration.args : [],
         jobClass: jobConfiguration.class,
         jobKey,
-        options: jobConfiguration.options || {}
+        options: {deduplicateWhileQueued: true, ...(jobConfiguration.options || {})}
       })
     } catch (error) {
       await this.logger.error(() => ["Failed to enqueue scheduled background job", {jobKey, jobName: jobConfiguration.class.jobName()}, error])

package/src/background-jobs/store.js

@@ -144,12 +144,16 @@ export default class BackgroundJobsStore {
     let resultJobId = jobId
 
     await this._withDb(async (db) => {
-      if (options?.deduplicateWhileQueued && concurrency?.concurrencyKey) {
+      if (options?.deduplicateWhileQueued) {
+        // Dedupe on the job's identity (name + args + queue), NOT its concurrency key, so a job
+        // keeps whatever concurrency it resolves to — in particular a scheduled job that relies on
+        // its queue-derived `queue:<name>` cap still participates in that cluster-wide cap instead
+        // of being pulled onto a private per-job key.
         const existing = await db
           .newQuery()
           .from(JOBS_TABLE)
           .select("id")
-          .where({status: "queued", concurrency_key: concurrency.concurrencyKey})
+          .where({status: "queued", job_name: jobName, args_json: argsJson, queue})
           .limit(1)
           .results()
 
@@ -472,6 +476,7 @@ export default class BackgroundJobsStore {
       if (!job) return false
       if (!this._shouldAcceptReport({job, handoffId, workerId, handedOffAtMs})) return false
 
+      await this._lockConcurrencyRow(db, job.concurrencyKey)
       const affectedRows = await this._updateAffectedRows(db, {
         tableName: JOBS_TABLE,
         data: {
@@ -500,6 +505,7 @@ export default class BackgroundJobsStore {
     await this._withDb(async (db) => await db.transaction(async () => {
       const job = await this._getJobRowById(db, jobId)
       if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return
+      await this._lockConcurrencyRow(db, job.concurrencyKey)
       const affectedRows = await this._updateAffectedRows(db, {
         tableName: JOBS_TABLE,
         data: {
@@ -719,6 +725,9 @@ export default class BackgroundJobsStore {
     return await this._withDb(async (db) => await this._transactionResult(db, async () => {
       const job = await this._getJobRowById(db, jobId)
       if (!job || (job.status !== "queued" && job.status !== "handed_off")) return false
+      // Only a handed_off job holds a concurrency reservation, so only that case touches the
+      // shared counter row and needs the concurrency-then-job lock ordering.
+      if (job.status === "handed_off") await this._lockConcurrencyRow(db, job.concurrencyKey)
       const affectedRows = await this._updateAffectedRows(db, {tableName: JOBS_TABLE, data: {status: "cancelled"}, conditions: {id: job.id, status: job.status}})
       if (affectedRows !== 1) return false
       if (job.status === "handed_off") await this._releaseConcurrency(db, job.concurrencyKey)
@@ -1194,6 +1203,7 @@ export default class BackgroundJobsStore {
       shouldRetry
     })
 
+    await this._lockConcurrencyRow(db, job.concurrencyKey)
     const affectedRows = await this._updateAffectedRows(db, {
       tableName: JOBS_TABLE,
       data: update,
@@ -1469,6 +1479,28 @@ export default class BackgroundJobsStore {
     if (this._normalizeNumber(configured.max_concurrency) !== maxConcurrency) throw new Error(`Conflicting maxConcurrency for background job concurrencyKey: ${concurrencyKey}`)
   }
 
+  /**
+   * Locks the concurrency counter row so a job-release transaction acquires it *before* the job
+   * row. {@link markHandedOff} reserves capacity (locking the counter row) before it updates the
+   * job, so it locks concurrency-then-job; the release paths update the job before releasing
+   * capacity, which is job-then-concurrency. Those opposite orders on the same shared counter row
+   * are what deadlock (AB-BA) under a draining worker. Taking this lock first gives every
+   * transaction a single concurrency-then-job order and removes the cycle.
+   *
+   * Uses a value-preserving `UPDATE` rather than `SELECT ... FOR UPDATE` so it stays portable
+   * across drivers without row-level locking reads (e.g. SQLite); on row-locking engines the
+   * matched row is write-locked for the rest of the transaction even though its value is unchanged.
+   * @param {import("../database/drivers/base.js").default} db - Database connection.
+   * @param {string | null} concurrencyKey - Concurrency key.
+   * @returns {Promise<void>} - Resolves when the counter row is locked.
+   */
+  async _lockConcurrencyRow(db, concurrencyKey) {
+    if (!concurrencyKey) return
+    const table = db.quoteTable(CONCURRENCY_TABLE)
+    const count = db.quoteColumn("active_count")
+    await db.query(`UPDATE ${table} SET ${count} = ${count} WHERE ${db.quoteColumn("concurrency_key")} = ${db.quote(concurrencyKey)}`)
+  }
+
   /**
    * Atomically reserves capacity for a key.
    * @param {import("../database/drivers/base.js").default} db - Database connection.

package/src/background-jobs/types.js

@@ -15,7 +15,7 @@
  * @property {string} [queue] - Queue name. Defaults to `"default"`. When the queue has a configured cap in `backgroundJobs.queues`, that cap is enforced cluster-wide.
  * @property {string} [concurrencyKey] - Opaque non-empty key used to share a concurrency cap. Overrides any queue-derived cap.
  * @property {number} [maxConcurrency] - Positive integer cap; must be paired with `concurrencyKey`.
- * @property {boolean} [deduplicateWhileQueued] - When true (requires `concurrencyKey`), skip the enqueue if a still-queued job with the same `concurrencyKey` already exists, returning that job's id. Keeps an interval-scheduled recurring job (e.g. retention pruning) from piling up redundant queued rows when it runs slower than its interval or no worker is free.
+ * @property {boolean} [deduplicateWhileQueued] - When true, skip the enqueue if an identical still-queued job (same job name, args and queue) already exists, returning that job's id. Deduplication is by job identity and independent of `concurrencyKey`, so the job keeps its normal (e.g. queue-derived) concurrency cap. Keeps an interval-scheduled recurring job (e.g. retention pruning) from piling up redundant queued rows when it runs slower than its interval or no worker is free.
  * @property {number} [scheduledAtMs] - Epoch timestamp in milliseconds when the job becomes eligible for dispatch. Defaults to enqueue time.
  */
 /**

package/src/configuration-types.js

@@ -64,6 +64,9 @@
  * @property {string} [databaseCharset] - Default character set applied by `db:create` via mysql/mariadb `CREATE DATABASE ... CHARACTER SET`. Distinct from `charset`, which is the client connection charset forwarded to the mysql2 driver.
  * @property {string} [databaseCollation] - Default collation applied by `db:create` via mysql/mariadb `CREATE DATABASE ... COLLATE`.
  * @property {string} [database] - Database name for this connection.
+ * @property {number} [deadlockMaxRetries] - Maximum attempts for the outermost transaction when it keeps hitting deadlocks. Defaults to 8.
+ * @property {number} [deadlockBaseWaitMs] - Base delay (ms) for the deadlock retry backoff; the per-attempt ceiling doubles from here. Defaults to 50.
+ * @property {number} [deadlockMaxWaitMs] - Cap (ms) on the deadlock retry backoff ceiling so the jittered wait stays bounded. Defaults to 1000.
  * @property {typeof import("./database/drivers/base.js").default} [driver] - Driver class to use for this database.
  * @property {typeof import("./database/pool/base.js").default} [poolType] - Pool class to use for this database.
  * @property {function() : ?} [getConnection] - Custom connection factory override.

package/src/database/drivers/base.js

@@ -122,6 +122,7 @@ import TableData from "../table-data/index.js"
 import TableColumn from "../table-data/table-column.js"
 import TableForeignKey from "../table-data/table-foreign-key.js"
 import wait from "awaitery/build/wait.js"
+import {optionalPositiveInteger} from "typanic"
 
 /**
  * Runs now ms.
@@ -864,7 +865,10 @@ export default class VelociousDatabaseDriversBase {
       return await this._runTransactionAttempt(callback)
     }
 
-    const maxAttempts = 5
+    const args = this.getArgs()
+    const maxAttempts = optionalPositiveInteger(args.deadlockMaxRetries, "deadlockMaxRetries") ?? 8
+    const configuredBaseWaitMs = optionalPositiveInteger(args.deadlockBaseWaitMs, "deadlockBaseWaitMs")
+    const deadlockMaxWaitMs = optionalPositiveInteger(args.deadlockMaxWaitMs, "deadlockMaxWaitMs") ?? 1000
     let attempt = 0
 
     while (true) {
@@ -876,10 +880,22 @@ export default class VelociousDatabaseDriversBase {
         const retryInfo = error instanceof Error ? this.retryableDatabaseError(error) : {retry: false, reconnect: false}
 
         if (retryInfo.deadlock && attempt < maxAttempts && this._transactionsCount == 0) {
-          const baseWaitMs = typeof retryInfo.waitMs == "number" && retryInfo.waitMs > 0 ? retryInfo.waitMs : 50
+          // An explicitly-configured base wins so the tuning knob is effective even on drivers
+          // whose classifier supplies its own `waitMs` (MySQL/MariaDB return a fixed 50ms for
+          // deadlocks); otherwise honor that classifier hint, then fall back to 50ms.
+          const baseWaitMs = configuredBaseWaitMs ?? (typeof retryInfo.waitMs == "number" && retryInfo.waitMs > 0 ? retryInfo.waitMs : 50)
+
+          // Full-jitter exponential backoff: wait a uniform-random duration in
+          // [0, min(base * 2^(attempt-1), cap)]. The doubling ceiling spreads retries out as
+          // contention persists, and the jitter de-correlates transactions that deadlocked in
+          // lockstep so they stop re-colliding on the same wait (the linear `base * attempt`
+          // this replaces had every victim retry after an identical delay). `attempt` is
+          // 1-based here, so 2^(attempt-1) is 1, 2, 4, ... The cap keeps the tail sub-second.
+          const ceilingWaitMs = Math.min(baseWaitMs * (2 ** (attempt - 1)), deadlockMaxWaitMs)
+          const jitteredWaitMs = Math.floor(Math.random() * (ceilingWaitMs + 1))
 
           this.logger.warn(`Retrying transaction after deadlock (attempt ${attempt}/${maxAttempts})`)
-          await wait(baseWaitMs * attempt)
+          await this._waitMs(jitteredWaitMs)
           continue
         }
 
@@ -888,6 +904,16 @@ export default class VelociousDatabaseDriversBase {
     }
   }
 
+  /**
+   * Waits `ms` milliseconds. Isolated in its own method so tests can observe (and skip) the
+   * deadlock-retry backoff without a real timer.
+   * @param {number} ms - Milliseconds to wait.
+   * @returns {Promise<void>} - Resolves after the delay.
+   */
+  async _waitMs(ms) {
+    await wait(ms)
+  }
+
   /**
    * Runs a single transaction attempt: starts a transaction (or a savepoint when nested), runs
    * `callback`, and commits — rolling back on error. {@link transaction} wraps this with deadlock