velocious 1.0.539 → 1.0.541

package/package.json

@@ -3,7 +3,7 @@
     "velocious": "build/bin/velocious.js"
   },
   "name": "velocious",
-  "version": "1.0.539",
+  "version": "1.0.541",
   "main": "build/index.js",
   "types": "build/index.d.ts",
   "files": [

package/src/background-jobs/store.js

@@ -476,6 +476,7 @@ export default class BackgroundJobsStore {
       if (!job) return false
       if (!this._shouldAcceptReport({job, handoffId, workerId, handedOffAtMs})) return false
 
+      await this._lockConcurrencyRow(db, job.concurrencyKey)
       const affectedRows = await this._updateAffectedRows(db, {
         tableName: JOBS_TABLE,
         data: {
@@ -504,6 +505,7 @@ export default class BackgroundJobsStore {
     await this._withDb(async (db) => await db.transaction(async () => {
       const job = await this._getJobRowById(db, jobId)
       if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return
+      await this._lockConcurrencyRow(db, job.concurrencyKey)
       const affectedRows = await this._updateAffectedRows(db, {
         tableName: JOBS_TABLE,
         data: {
@@ -723,6 +725,9 @@ export default class BackgroundJobsStore {
     return await this._withDb(async (db) => await this._transactionResult(db, async () => {
       const job = await this._getJobRowById(db, jobId)
       if (!job || (job.status !== "queued" && job.status !== "handed_off")) return false
+      // Only a handed_off job holds a concurrency reservation, so only that case touches the
+      // shared counter row and needs the concurrency-then-job lock ordering.
+      if (job.status === "handed_off") await this._lockConcurrencyRow(db, job.concurrencyKey)
       const affectedRows = await this._updateAffectedRows(db, {tableName: JOBS_TABLE, data: {status: "cancelled"}, conditions: {id: job.id, status: job.status}})
       if (affectedRows !== 1) return false
       if (job.status === "handed_off") await this._releaseConcurrency(db, job.concurrencyKey)
@@ -1198,6 +1203,7 @@ export default class BackgroundJobsStore {
       shouldRetry
     })
 
+    await this._lockConcurrencyRow(db, job.concurrencyKey)
     const affectedRows = await this._updateAffectedRows(db, {
       tableName: JOBS_TABLE,
       data: update,
@@ -1473,6 +1479,28 @@ export default class BackgroundJobsStore {
     if (this._normalizeNumber(configured.max_concurrency) !== maxConcurrency) throw new Error(`Conflicting maxConcurrency for background job concurrencyKey: ${concurrencyKey}`)
   }
 
+  /**
+   * Locks the concurrency counter row so a job-release transaction acquires it *before* the job
+   * row. {@link markHandedOff} reserves capacity (locking the counter row) before it updates the
+   * job, so it locks concurrency-then-job; the release paths update the job before releasing
+   * capacity, which is job-then-concurrency. Those opposite orders on the same shared counter row
+   * are what deadlock (AB-BA) under a draining worker. Taking this lock first gives every
+   * transaction a single concurrency-then-job order and removes the cycle.
+   *
+   * Uses a value-preserving `UPDATE` rather than `SELECT ... FOR UPDATE` so it stays portable
+   * across drivers without row-level locking reads (e.g. SQLite); on row-locking engines the
+   * matched row is write-locked for the rest of the transaction even though its value is unchanged.
+   * @param {import("../database/drivers/base.js").default} db - Database connection.
+   * @param {string | null} concurrencyKey - Concurrency key.
+   * @returns {Promise<void>} - Resolves when the counter row is locked.
+   */
+  async _lockConcurrencyRow(db, concurrencyKey) {
+    if (!concurrencyKey) return
+    const table = db.quoteTable(CONCURRENCY_TABLE)
+    const count = db.quoteColumn("active_count")
+    await db.query(`UPDATE ${table} SET ${count} = ${count} WHERE ${db.quoteColumn("concurrency_key")} = ${db.quote(concurrencyKey)}`)
+  }
+
   /**
    * Atomically reserves capacity for a key.
    * @param {import("../database/drivers/base.js").default} db - Database connection.

package/src/configuration-types.js

@@ -64,6 +64,9 @@
  * @property {string} [databaseCharset] - Default character set applied by `db:create` via mysql/mariadb `CREATE DATABASE ... CHARACTER SET`. Distinct from `charset`, which is the client connection charset forwarded to the mysql2 driver.
  * @property {string} [databaseCollation] - Default collation applied by `db:create` via mysql/mariadb `CREATE DATABASE ... COLLATE`.
  * @property {string} [database] - Database name for this connection.
+ * @property {number} [deadlockMaxRetries] - Maximum attempts for the outermost transaction when it keeps hitting deadlocks. Defaults to 8.
+ * @property {number} [deadlockBaseWaitMs] - Base delay (ms) for the deadlock retry backoff; the per-attempt ceiling doubles from here. Defaults to 50.
+ * @property {number} [deadlockMaxWaitMs] - Cap (ms) on the deadlock retry backoff ceiling so the jittered wait stays bounded. Defaults to 1000.
  * @property {typeof import("./database/drivers/base.js").default} [driver] - Driver class to use for this database.
  * @property {typeof import("./database/pool/base.js").default} [poolType] - Pool class to use for this database.
  * @property {function() : ?} [getConnection] - Custom connection factory override.

package/src/database/drivers/base.js

@@ -122,6 +122,7 @@ import TableData from "../table-data/index.js"
 import TableColumn from "../table-data/table-column.js"
 import TableForeignKey from "../table-data/table-foreign-key.js"
 import wait from "awaitery/build/wait.js"
+import {optionalPositiveInteger} from "typanic"
 
 /**
  * Runs now ms.
@@ -864,7 +865,10 @@ export default class VelociousDatabaseDriversBase {
       return await this._runTransactionAttempt(callback)
     }
 
-    const maxAttempts = 5
+    const args = this.getArgs()
+    const maxAttempts = optionalPositiveInteger(args.deadlockMaxRetries, "deadlockMaxRetries") ?? 8
+    const configuredBaseWaitMs = optionalPositiveInteger(args.deadlockBaseWaitMs, "deadlockBaseWaitMs")
+    const deadlockMaxWaitMs = optionalPositiveInteger(args.deadlockMaxWaitMs, "deadlockMaxWaitMs") ?? 1000
     let attempt = 0
 
     while (true) {
@@ -876,10 +880,22 @@ export default class VelociousDatabaseDriversBase {
         const retryInfo = error instanceof Error ? this.retryableDatabaseError(error) : {retry: false, reconnect: false}
 
         if (retryInfo.deadlock && attempt < maxAttempts && this._transactionsCount == 0) {
-          const baseWaitMs = typeof retryInfo.waitMs == "number" && retryInfo.waitMs > 0 ? retryInfo.waitMs : 50
+          // An explicitly-configured base wins so the tuning knob is effective even on drivers
+          // whose classifier supplies its own `waitMs` (MySQL/MariaDB return a fixed 50ms for
+          // deadlocks); otherwise honor that classifier hint, then fall back to 50ms.
+          const baseWaitMs = configuredBaseWaitMs ?? (typeof retryInfo.waitMs == "number" && retryInfo.waitMs > 0 ? retryInfo.waitMs : 50)
+
+          // Full-jitter exponential backoff: wait a uniform-random duration in
+          // [0, min(base * 2^(attempt-1), cap)]. The doubling ceiling spreads retries out as
+          // contention persists, and the jitter de-correlates transactions that deadlocked in
+          // lockstep so they stop re-colliding on the same wait (the linear `base * attempt`
+          // this replaces had every victim retry after an identical delay). `attempt` is
+          // 1-based here, so 2^(attempt-1) is 1, 2, 4, ... The cap keeps the tail sub-second.
+          const ceilingWaitMs = Math.min(baseWaitMs * (2 ** (attempt - 1)), deadlockMaxWaitMs)
+          const jitteredWaitMs = Math.floor(Math.random() * (ceilingWaitMs + 1))
 
           this.logger.warn(`Retrying transaction after deadlock (attempt ${attempt}/${maxAttempts})`)
-          await wait(baseWaitMs * attempt)
+          await this._waitMs(jitteredWaitMs)
           continue
         }
 
@@ -888,6 +904,16 @@ export default class VelociousDatabaseDriversBase {
     }
   }
 
+  /**
+   * Waits `ms` milliseconds. Isolated in its own method so tests can observe (and skip) the
+   * deadlock-retry backoff without a real timer.
+   * @param {number} ms - Milliseconds to wait.
+   * @returns {Promise<void>} - Resolves after the delay.
+   */
+  async _waitMs(ms) {
+    await wait(ms)
+  }
+
   /**
    * Runs a single transaction attempt: starts a transaction (or a savepoint when nested), runs
    * `callback`, and commits — rolling back on error. {@link transaction} wraps this with deadlock

package/src/database/record/index.js

@@ -3167,6 +3167,47 @@ class VelociousDatabaseRecord {
     return await this._newQuery().findByOrFail(conditions)
   }
 
+  /**
+   * Returns a scope whose eager finders run against an explicit `tenant` (and
+   * therefore its database) instead of whatever tenant is ambient in
+   * `Current.tenant()`. Use it to read a model that may live in a specific
+   * tenant or in the default database from another tenant context — for example
+   * `GithubWebhook.usingTenant(tenant).findBy({id})` — without depending on
+   * which tenant happens to be active. The target tenant's connections are
+   * ensured for the duration of each query.
+   * @template {typeof VelociousDatabaseRecord} MC
+   * @this {MC}
+   * @param {?} tenant - Tenant descriptor to scope the queries to (as accepted by `configuration.runWithTenant`).
+   * @returns {{find: (recordId: ?) => Promise<InstanceType<MC> | null>, findBy: (conditions: {[key: string]: string | number}) => Promise<InstanceType<MC> | null>, findByOrFail: (conditions: {[key: string]: string | number}) => Promise<InstanceType<MC>>}} - Eager finders scoped to the given tenant.
+   */
+  static usingTenant(tenant) {
+    const ModelClass = this
+
+    return {
+      find: (recordId) => ModelClass._runUsingTenant(tenant, () => ModelClass.find(recordId)),
+      findBy: (conditions) => ModelClass._runUsingTenant(tenant, () => ModelClass.findBy(conditions)),
+      findByOrFail: (conditions) => ModelClass._runUsingTenant(tenant, () => ModelClass.findByOrFail(conditions))
+    }
+  }
+
+  /**
+   * Runs `callback` with the tenant switched to `tenant` and that tenant's
+   * connections ensured. Backs `usingTenant`.
+   * @template T
+   * @param {?} tenant - Tenant descriptor.
+   * @param {() => Promise<T>} callback - Query to run under the tenant.
+   * @returns {Promise<T>} - Resolves with the callback's result.
+   */
+  static async _runUsingTenant(tenant, callback) {
+    const configuration = this._getConfiguration()
+
+    // Do NOT ensureInitialized() out here: for a tenant-switched model whose
+    // first initialization would resolve metadata from the ambient tenant's
+    // database, that must happen under the requested tenant. The finders inside
+    // `callback` call ensureInitialized() themselves, now within this scope.
+    return await configuration.runWithTenant(tenant, () => configuration.ensureConnections({name: `usingTenant: ${this.getModelName()}`}, callback))
+  }
+
   /**
    * Runs find or create by.
    * @template {typeof VelociousDatabaseRecord} MC