velocious 1.0.588 → 1.0.590

package/README.md

@@ -2345,6 +2345,26 @@ await MyJob.performLaterWithOptions({
 
 Until `scheduledAtMs` is reached, the job remains queued but is not eligible for dispatch. The event-driven dispatcher arms its timer for the earliest future job and wakes at that timestamp. Omitting `scheduledAtMs` keeps the immediate-enqueue behavior.
 
+A running job that cannot proceed yet can reschedule its same durable row without
+recording a failure—for example, when a non-blocking lock is busy:
+
+```js
+async perform(accountId) {
+  if (!(await Account.tryAcquireRefreshLock(accountId))) {
+    this.rescheduleIn(30_000)
+  }
+
+  await refreshAccount(accountId)
+}
+```
+
+`rescheduleIn(delayMs)` requires a finite, non-negative safe-integer millisecond
+delay and never returns: it stops the current `perform`, releases its worker and
+concurrency slots, and makes the same job eligible again after the delay. This is
+normal control flow, not failure retry: attempts and failure metadata remain
+unchanged, retries are not consumed, and failure/error events are not emitted.
+See [Rescheduling a running job](docs/background-jobs.md#rescheduling-a-running-job).
+
 Use a durable stable key when the same logical one-off schedule must be moved or cancelled without retaining its transient job id:
 
 ```js
@@ -2470,7 +2490,7 @@ backgroundJobs: {
 }
 ```
 
-A job with no queue runs on `"default"`; a queue with no cap is unlimited. Caps are enforced through the durable per-key concurrency mechanism (the reserved `queue:<name>` key), hold regardless of how many worker processes run, and are reconciled against the existing backlog on startup when you change them. Scheduled jobs honor a job's `static queue` too.
+A job with no queue runs on `"default"`; a queue with no cap is unlimited. Caps are enforced through the durable per-key concurrency mechanism (the reserved `queue:<name>` key) and hold regardless of how many worker processes run. Changing a cap is reconciled against the existing backlog only when `background-jobs-main` starts (serialized across processes with a database advisory lock); `db:migrate`, `db:tenants:*`, and routine store/application initialization never reconcile the backlog and stay read-only regarding queued jobs. Scheduled jobs honor a job's `static queue` too.
 
 Set `priority` (default `0`) to dispatch a queue ahead of lower-priority ones regardless of enqueue order, so a small time-critical queue is never starved by a flood of low-priority work sharing a worker pool. Unlike Sidekiq's strict queue ordering, priority composes with the caps: a higher-priority queue already at its `maxConcurrent` is skipped and dispatch falls through to the next eligible job. See [docs/background-jobs.md](docs/background-jobs.md#queues-per-queue-concurrency-caps).
 

package/build/background-jobs/job-runner.js

@@ -3,6 +3,7 @@
 import configurationResolver from "../configuration-resolver.js"
 import BackgroundJobRegistry from "./job-registry.js"
 import BackgroundJobsStatusReporter from "./status-reporter.js"
+import BackgroundJobRescheduleSignal from "./reschedule-signal.js"
 
 const BEACON_READY_TIMEOUT_MS = 5000
 
@@ -79,7 +80,7 @@ function runnerProcessTitle(JobClass, payload) {
  * @param {object} [options] - Runner options.
  * @param {boolean} [options.closeConnections] - Whether to gracefully close framework connections after the job.
  * @param {boolean} [options.manageProcessTitle] - Whether to set the per-job process title and restore it afterwards. Off for concurrent pooled runners, where interleaved snapshot/restore of the single process-wide `process.title` would corrupt it; the pooled child owns an aggregate title instead.
- * @returns {Promise<void>} - Resolves when complete.
+ * @returns {Promise<"completed" | "rescheduled">} - Acknowledged outcome.
  */
 export default async function runJobPayload(payload, {closeConnections = true, manageProcessTitle = true} = {}) {
   const configuration = await configurationResolver()
@@ -109,6 +110,23 @@ export default async function runJobPayload(payload, {closeConnections = true, m
         await perform.apply(jobInstance, payload.args || [])
       })
     } catch (error) {
+      if (error instanceof BackgroundJobRescheduleSignal) {
+        if (payload.id) {
+          await reporter.reportWithRetry({
+            jobId: payload.id,
+            status: "rescheduled",
+            delayMs: error.delayMs,
+            handoffId: payload.handoffId,
+            workerId: payload.workerId,
+            handedOffAtMs: payload.handedOffAtMs,
+            maxDurationMs: 30000,
+            retryPersistErrors: true
+          })
+        }
+
+        return "rescheduled"
+      }
+
       const performedError = error instanceof Error ? error : new Error(String(error))
       if (payload.id) {
         await reporter.reportWithRetry({
@@ -135,6 +153,7 @@ export default async function runJobPayload(payload, {closeConnections = true, m
         maxDurationMs: 30000
       })
     }
+    return "completed"
   } finally {
     // Restore the runner's base title so a lingering/idle runner (or a reused
     // one) doesn't misreport a finished job as still running.

package/build/background-jobs/job.js

@@ -1,6 +1,7 @@
 // @ts-check
 
 import BackgroundJobsClient from "./client.js"
+import BackgroundJobRescheduleSignal from "./reschedule-signal.js"
 
 /**
  * Base class for background jobs.
@@ -43,6 +44,20 @@ export default class VelociousJob {
    */
   static processTitle = undefined
 
+  /**
+   * Stops this performance and reschedules the same logical job row. This is
+   * normal control flow: it does not count as a failure or consume a retry.
+   * @param {number} delayMs - Non-negative safe-integer delay in milliseconds.
+   * @returns {never} - This method never returns.
+   */
+  rescheduleIn(delayMs) {
+    if (!Number.isSafeInteger(delayMs) || delayMs < 0) {
+      throw new TypeError("background job reschedule delayMs must be a non-negative safe integer")
+    }
+
+    throw new BackgroundJobRescheduleSignal(delayMs)
+  }
+
   /**
    * Runs job name.
    * @returns {string} - Job name.

package/build/background-jobs/main.js

@@ -155,6 +155,12 @@ export default class BackgroundJobsMain {
     await this.configuration.initialize({type: "background-jobs-main"})
     await this.configuration.connectBeacon({peerType: "background-jobs-main"})
     await this.store.ensureReady()
+    // Queue-cap changes are reconciled against the persisted backlog here, at
+    // main-process startup — the explicit lifecycle for applying queue
+    // configuration changes. The store serializes the adoption/release UPDATEs
+    // across processes with a database advisory lock, so concurrently started
+    // mains cannot interleave them.
+    await this.store.reconcileQueueConcurrency()
     const server = net.createServer((socket) => this._handleConnection(socket))
     this.server = server
 
@@ -595,6 +601,11 @@ export default class BackgroundJobsMain {
 
     if (message?.type === "job-failed") {
       this._handleJobFailed({jsonSocket, message})
+      return
+    }
+
+    if (message?.type === "job-reschedule") {
+      this._handleJobReschedule({jsonSocket, message})
     }
   }
 
@@ -879,6 +890,40 @@ export default class BackgroundJobsMain {
     }
   }
 
+  /**
+   * Persists a normal job reschedule outcome and wakes scheduled dispatch.
+   * @param {object} args - Options.
+   * @param {JsonSocket} args.jsonSocket - JSON socket.
+   * @param {import("./types.js").BackgroundJobRescheduleMessage} args.message - Message.
+   * @returns {Promise<void>} - Resolves when handled.
+   */
+  async _handleJobReschedule({jsonSocket, message}) {
+    try {
+      const accepted = await this.store.markRescheduled({
+        jobId: message.jobId,
+        delayMs: message.delayMs,
+        handoffId: message.handoffId,
+        workerId: message.workerId,
+        handedOffAtMs: message.handedOffAtMs
+      })
+      if (accepted && message.handoffId) {
+        this._forgetHandoff({handoffId: message.handoffId, jobId: message.jobId})
+      }
+      jsonSocket.send({type: "job-updated", jobId: message.jobId})
+      this._notifyEnqueued()
+      await this._drain()
+    } catch (error) {
+      const normalizedError = error instanceof Error ? error : new Error(String(error))
+      const payload = {context: {jobId: message.jobId, stage: "background-job-reschedule"}, error: normalizedError}
+      const errorEvents = this.configuration.getErrorEvents()
+
+      this.logger.error(() => ["Failed to update job reschedule:", normalizedError])
+      errorEvents.emit("framework-error", payload)
+      errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
+      jsonSocket.send({type: "job-update-error", jobId: message.jobId, error: "Failed to update job"})
+    }
+  }
+
   /**
    * Runs handle job failed.
    * @param {object} args - Options.

package/build/background-jobs/pooled-runner-child.js

@@ -65,7 +65,7 @@ function isJobMessage(message) {
  * @param {object} args - Outcome.
  * @param {string} args.jobId - Job id.
  * @param {boolean} args.acknowledged - Whether the terminal report was acknowledged.
- * @param {"completed" | "failed"} [args.status] - Acknowledged terminal status.
+ * @param {"completed" | "failed" | "rescheduled"} [args.status] - Acknowledged outcome.
  * @param {Error} [args.error] - Reporting error when acknowledgement was not obtained.
  * @returns {Promise<void>} - Resolves after IPC accepts the message.
  */
@@ -99,8 +99,8 @@ function sendOutcome({jobId, acknowledged, status, error}) {
  */
 async function runJob(payload) {
   try {
-    await runJobPayload(payload, {closeConnections: false, manageProcessTitle: false})
-    await sendOutcome({jobId: payload.id, acknowledged: true, status: "completed"})
+    const status = await runJobPayload(payload, {closeConnections: false, manageProcessTitle: false})
+    await sendOutcome({jobId: payload.id, acknowledged: true, status})
   } catch (error) {
     if (error instanceof BackgroundJobPerformedFailure) {
       await sendOutcome({jobId: payload.id, acknowledged: true, status: "failed"})

package/build/background-jobs/reschedule-signal.js

@@ -0,0 +1,14 @@
+// @ts-check
+
+/** Internal control flow raised by `VelociousJob#rescheduleIn`. */
+export default class BackgroundJobRescheduleSignal extends Error {
+  /**
+   * Creates a reschedule control signal.
+   * @param {number} delayMs - Reschedule delay in milliseconds.
+   */
+  constructor(delayMs) {
+    super(`Reschedule background job in ${delayMs}ms`)
+    this.name = "BackgroundJobRescheduleSignal"
+    this.delayMs = delayMs
+  }
+}

package/build/background-jobs/status-reporter.js

@@ -36,14 +36,15 @@ export default class BackgroundJobsStatusReporter {
    * Runs report.
    * @param {object} args - Options.
    * @param {string} args.jobId - Job id.
-   * @param {"completed" | "failed"} args.status - Status.
+   * @param {"completed" | "failed" | "rescheduled"} args.status - Status.
+   * @param {number} [args.delayMs] - Reschedule delay in milliseconds.
    * @param {ReturnType<typeof JSON.parse>} [args.error] - Error.
    * @param {string} [args.handoffId] - Handoff lease id.
    * @param {number} [args.handedOffAtMs] - Handed off timestamp.
    * @param {string} [args.workerId] - Worker id.
    * @returns {Promise<void>} - Resolves when reported.
    */
-  async report({jobId, status, error, handoffId, handedOffAtMs, workerId}) {
+  async report({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId}) {
     const config = this.configuration.getBackgroundJobsConfig()
     const host = this.host || config.host
     const port = typeof this.port === "number" ? this.port : config.port
@@ -57,8 +58,9 @@ export default class BackgroundJobsStatusReporter {
         signal: control.signal,
         onConnect: (jsonSocket) => {
           jsonSocket.send({
-            type: status === "completed" ? "job-complete" : "job-failed",
+            type: status === "completed" ? "job-complete" : status === "rescheduled" ? "job-reschedule" : "job-failed",
             jobId,
+            delayMs,
             handoffId,
             workerId,
             handedOffAtMs,
@@ -83,7 +85,8 @@ export default class BackgroundJobsStatusReporter {
    * Runs report with retry.
    * @param {object} args - Options.
    * @param {string} args.jobId - Job id.
-   * @param {"completed" | "failed"} args.status - Status.
+   * @param {"completed" | "failed" | "rescheduled"} args.status - Status.
+   * @param {number} [args.delayMs] - Reschedule delay in milliseconds.
    * @param {ReturnType<typeof JSON.parse>} [args.error] - Error.
    * @param {string} [args.handoffId] - Handoff lease id.
    * @param {number} [args.handedOffAtMs] - Handed off timestamp.
@@ -92,13 +95,13 @@ export default class BackgroundJobsStatusReporter {
    * @param {boolean} [args.retryPersistErrors] - Retry a `BackgroundJobUpdateError` (main's `job-update-error`, i.e. a transient DB failure while persisting the terminal status) instead of throwing immediately. Off by default so short-lived forked/spawned runners keep failing loudly and exit non-zero to be reclaimed; on for the long-lived worker, which cannot exit-to-reclaim and would otherwise strand the job in `handed_off`.
    * @returns {Promise<void>} - Resolves when reported.
    */
-  async reportWithRetry({jobId, status, error, handoffId, handedOffAtMs, workerId, maxDurationMs, retryPersistErrors = false}) {
+  async reportWithRetry({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId, maxDurationMs, retryPersistErrors = false}) {
     let attempt = 0
     const startTime = Date.now()
 
     while (true) {
       try {
-        await this.report({jobId, status, error, handoffId, handedOffAtMs, workerId})
+        await this.report({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId})
         return
       } catch (error) {
         // A `BackgroundJobUpdateError` means main answered `job-update-error`, which it

package/build/background-jobs/store.js

@@ -158,6 +158,45 @@ export default class BackgroundJobsStore {
     await this._ensureSchema(db)
   }
 
+  /**
+   * Reconciles queue-derived concurrency with the current configuration: the
+   * explicit lifecycle path that adopts/releases persisted queued jobs onto
+   * queue concurrency keys when `queues[name].maxConcurrent` is added, removed,
+   * or changed. Called by the background-jobs main process on startup — the
+   * deploy-time moment queue configuration changes take effect. Schema/tenant
+   * checks and routine connection initialization deliberately never run this:
+   * they stay read-only regarding queued job rows, because the broad
+   * adoption/release UPDATEs deadlock against active job processes under
+   * concurrent tenant initialization. Serialized across processes with a
+   * database advisory lock so concurrently started mains cannot interleave the
+   * UPDATEs; the per-instance memo only skips repeat work within this process.
+   * @returns {Promise<void>} - Resolves when reconciled.
+   */
+  async reconcileQueueConcurrency() {
+    if (this._queueConcurrencyReconciled) return
+
+    await this.ensureReady()
+
+    await this._withDb(async (db) => {
+      const lockName = "background-jobs:queue-concurrency-reconcile"
+      const acquired = await db.acquireAdvisoryLock(lockName)
+
+      if (!acquired) throw new Error("Failed to acquire background job queue-concurrency reconcile lock")
+
+      try {
+        await this._reconcileQueueConcurrency(db)
+        await this._reconcileConcurrency(db)
+
+        // Latch the memo only after BOTH steps succeed: if the count rebuild
+        // fails after adoption, a retry on this store must re-enter and repair
+        // the counts (adoption itself is idempotent).
+        this._queueConcurrencyReconciled = true
+      } finally {
+        await db.releaseAdvisoryLock(lockName)
+      }
+    })
+  }
+
   /**
    * Runs enqueue.
    * @param {object} args - Options.
@@ -655,6 +694,48 @@ export default class BackgroundJobsStore {
     }))
   }
 
+  /**
+   * Returns an active handoff to the queue at a caller-requested future time.
+   * This is normal job control flow: it preserves failure attempts and metadata.
+   * @param {object} args - Options.
+   * @param {string} args.jobId - Job id.
+   * @param {number} args.delayMs - Delay from persistence time in milliseconds.
+   * @param {string} [args.handoffId] - Handoff lease id.
+   * @param {string} [args.workerId] - Worker id.
+   * @param {number} [args.handedOffAtMs] - Handed off timestamp.
+   * @returns {Promise<boolean>} - Whether the fenced report was accepted.
+   */
+  async markRescheduled({jobId, delayMs, handoffId, workerId, handedOffAtMs}) {
+    await this.ensureReady()
+    this._validateRescheduleDelayMs(delayMs)
+
+    return await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
+      const job = await this._getJobRowById(db, jobId)
+
+      if (!job) return false
+      if (!this._shouldAcceptReport({job, handoffId, workerId, handedOffAtMs})) return false
+
+      await this._lockConcurrencyRow(db, job.concurrencyKey)
+      const scheduledAtMs = this._rescheduledAtMs(delayMs)
+      const affectedRows = await this._updateAffectedRows(db, {
+        tableName: JOBS_TABLE,
+        data: {
+          status: "queued",
+          scheduled_at_ms: scheduledAtMs,
+          handed_off_at_ms: null,
+          handoff_id: null,
+          worker_id: null
+        },
+        conditions: this._activeHandoffConditions(job)
+      })
+
+      if (affectedRows !== 1) return false
+      await this._releaseConcurrency(db, job.concurrencyKey)
+      await this._recordStatusTransition(db, "handed_off", "queued")
+      return true
+    }))
+  }
+
   /**
    * Runs mark returned to queue.
    * @param {object} args - Options.
@@ -1028,6 +1109,33 @@ export default class BackgroundJobsStore {
     throw VelociousError.safe("background job scheduledAtMs must be a non-negative safe integer")
   }
 
+  /**
+   * Resolves a reschedule delay against persistence time.
+   * @param {number} delayMs - Delay in milliseconds.
+   * @returns {number} - Future eligibility timestamp.
+   */
+  _rescheduledAtMs(delayMs) {
+    this._validateRescheduleDelayMs(delayMs)
+
+    const scheduledAtMs = Date.now() + delayMs
+    if (!Number.isSafeInteger(scheduledAtMs)) {
+      throw VelociousError.safe("background job reschedule scheduledAtMs must be a safe integer")
+    }
+
+    return scheduledAtMs
+  }
+
+  /**
+   * Validates a public reschedule delay before persistence work begins.
+   * @param {number} delayMs - Delay in milliseconds.
+   * @returns {void}
+   */
+  _validateRescheduleDelayMs(delayMs) {
+    if (!Number.isSafeInteger(delayMs) || delayMs < 0) {
+      throw VelociousError.safe("background job reschedule delayMs must be a non-negative safe integer")
+    }
+  }
+
   /**
    * Validates a stable schedule key at the public storage boundary.
    * @param {string} scheduleKey - Stable logical schedule key.
@@ -1119,7 +1227,6 @@ export default class BackgroundJobsStore {
       await this._ensureScheduleKeysTable(db)
       await this._ensureConcurrencyTable(db)
       await this._ensureCountRevisionTable(db)
-      await this._reconcileQueueConcurrency(db)
       await this._reconcileConcurrency(db)
 
       return
@@ -1130,7 +1237,6 @@ export default class BackgroundJobsStore {
     await this._ensureScheduleKeysTable(db)
     await this._ensureConcurrencyTable(db)
     await this._ensureCountRevisionTable(db)
-    await this._reconcileQueueConcurrency(db)
     await this._reconcileConcurrency(db)
 
     if (alreadyApplied) return
@@ -2114,16 +2220,22 @@ export default class BackgroundJobsStore {
   }
 
   /**
-   * Reconciles queue-derived concurrency with the current configuration, once
-   * per process. Enqueue only consults config for new jobs, so a cap added,
-   * removed, or changed while a backlog exists otherwise leaves persisted rows
-   * stale: pre-cap jobs keep a null key and bypass the cap, post-removal jobs
-   * stay capped under a now-unconfigured key, and a changed numeric cap stays
-   * stale until the next enqueue. Bring the durable state in line with config:
-   * sync each configured queue's stored cap, adopt not-yet-keyed non-terminal
-   * jobs onto their queue key, and release non-terminal jobs from queue keys
-   * whose queue is no longer capped. Runs before {@link _reconcileConcurrency}
-   * so the rebuilt active counts reflect the adopted/released keys.
+   * Reconciles queue-derived concurrency with the current configuration. Only
+   * invoked through {@link reconcileQueueConcurrency} — the explicit lifecycle
+   * path run at main-process startup under a cross-process advisory lock —
+   * never from schema/tenant checks or routine connection initialization,
+   * which stay read-only regarding queued job rows. The per-process memo is
+   * latched by {@link reconcileQueueConcurrency} only after the following
+   * count rebuild also succeeds, so a failed rebuild re-enters here on retry
+   * (the adoption UPDATEs below are idempotent). Enqueue only consults config for new jobs, so a cap added, removed, or changed
+   * while a backlog exists otherwise leaves persisted rows stale: pre-cap jobs
+   * keep a null key and bypass the cap, post-removal jobs stay capped under a
+   * now-unconfigured key, and a changed numeric cap stays stale until the next
+   * enqueue. Bring the durable state in line with config: sync each configured
+   * queue's stored cap, adopt not-yet-keyed non-terminal jobs onto their queue
+   * key, and release non-terminal jobs from queue keys whose queue is no
+   * longer capped. Runs before {@link _reconcileConcurrency} so the rebuilt
+   * active counts reflect the adopted/released keys.
    * @param {import("../database/drivers/base.js").default} db - Database connection.
    * @returns {Promise<void>} - Resolves when reconciled.
    */
@@ -2168,8 +2280,6 @@ export default class BackgroundJobsStore {
         `WHERE ${keyColumn} = ${db.quote(concurrencyKey)} AND ${nonTerminal}`
       )
     }
-
-    this._queueConcurrencyReconciled = true
   }
 
   /**

package/build/background-jobs/types.js

@@ -99,11 +99,12 @@
  * @typedef {{type: "job", payload: BackgroundJobPayload}} BackgroundJobJobMessage
  * @typedef {{type: "job-complete", jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number}} BackgroundJobCompleteMessage
  * @typedef {{type: "job-failed", jobId: string, error?: ReturnType<typeof JSON.parse>, handoffId?: string, workerId?: string, handedOffAtMs?: number}} BackgroundJobFailedMessage
+ * @typedef {{type: "job-reschedule", jobId: string, delayMs: number, handoffId?: string, workerId?: string, handedOffAtMs?: number}} BackgroundJobRescheduleMessage
  * @typedef {{type: "job-updated", jobId: string}} BackgroundJobUpdatedMessage
  * @typedef {{type: "job-update-error", jobId: string, error?: string}} BackgroundJobUpdateErrorMessage
  */
 /**
- * @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobHeartbeatMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobReplaceScheduledMessage | BackgroundJobScheduleReplacedMessage | BackgroundJobReplaceScheduledErrorMessage | BackgroundJobCancelScheduledMessage | BackgroundJobScheduleCancelledMessage | BackgroundJobCancelScheduledErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
+ * @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobHeartbeatMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobReplaceScheduledMessage | BackgroundJobScheduleReplacedMessage | BackgroundJobReplaceScheduledErrorMessage | BackgroundJobCancelScheduledMessage | BackgroundJobScheduleCancelledMessage | BackgroundJobCancelScheduledErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobRescheduleMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
  */
 
 export const nothing = {}

package/build/background-jobs/worker.js

@@ -9,6 +9,7 @@ import BackgroundJobsStatusReporter from "./status-reporter.js"
 import {randomUUID} from "crypto"
 import {fileURLToPath} from "node:url"
 import shutdownLifecycle from "../utils/shutdown-lifecycle.js"
+import BackgroundJobRescheduleSignal from "./reschedule-signal.js"
 
 /**
  * Per-forked-child timeout bookkeeping.
@@ -574,6 +575,18 @@ export default class BackgroundJobsWorker {
         workerId: payload.workerId || this.workerId
       })
     } catch (error) {
+      if (error instanceof BackgroundJobRescheduleSignal) {
+        this._reportJobResultInBackground({
+          jobId: payload.id,
+          status: "rescheduled",
+          delayMs: error.delayMs,
+          handoffId: payload.handoffId,
+          handedOffAtMs: payload.handedOffAtMs,
+          workerId: payload.workerId || this.workerId
+        })
+        return
+      }
+
       this._reportJobResultInBackground({
         jobId: payload.id,
         status: "failed",
@@ -1233,14 +1246,15 @@ export default class BackgroundJobsWorker {
    * Runs report job result.
    * @param {object} args - Options.
    * @param {string} args.jobId - Job id.
-   * @param {"completed" | "failed"} args.status - Status.
+   * @param {"completed" | "failed" | "rescheduled"} args.status - Status.
+   * @param {number} [args.delayMs] - Reschedule delay in milliseconds.
    * @param {ReturnType<typeof JSON.parse>} [args.error] - Error.
    * @param {string} [args.handoffId] - Handoff lease id.
    * @param {number} [args.handedOffAtMs] - Handed off timestamp.
    * @param {string} [args.workerId] - Worker id.
    * @returns {Promise<void>} - Resolves when reported.
    */
-  async _reportJobResult({jobId, status, error, handoffId, handedOffAtMs, workerId}) {
+  async _reportJobResult({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId}) {
     if (!this.statusReporter) return
 
     try {
@@ -1248,7 +1262,7 @@ export default class BackgroundJobsWorker {
       // long-lived and cannot exit to trigger orphan reclaim, so dropping the
       // completion here would strand the job in `handed_off` forever — fatal for a
       // `max_concurrency: 1` job (a stranded row blocks every future run).
-      await this.statusReporter.reportWithRetry({jobId, status, error, handoffId, handedOffAtMs, workerId, retryPersistErrors: true})
+      await this.statusReporter.reportWithRetry({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId, retryPersistErrors: true})
     } catch (reportError) {
       console.error("Background job status reporting failed:", reportError)
     }
@@ -1260,20 +1274,21 @@ export default class BackgroundJobsWorker {
    * graceful `stop()` can drain in-flight reports before closing the socket.
    * @param {object} args - Options.
    * @param {string} args.jobId - Job id.
-   * @param {"completed" | "failed"} args.status - Status.
+   * @param {"completed" | "failed" | "rescheduled"} args.status - Status.
+   * @param {number} [args.delayMs] - Reschedule delay in milliseconds.
    * @param {ReturnType<typeof JSON.parse>} [args.error] - Error.
    * @param {string} [args.handoffId] - Handoff lease id.
    * @param {number} [args.handedOffAtMs] - Handed off timestamp.
    * @param {string} [args.workerId] - Worker id.
    * @returns {void}
    */
-  _reportJobResultInBackground({jobId, status, error, handoffId, handedOffAtMs, workerId}) {
+  _reportJobResultInBackground({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId}) {
     /**
      * Defines report.
      * @type {Promise<void>} */
     let report
 
-    report = this._reportJobResult({jobId, status, error, handoffId, handedOffAtMs, workerId}).finally(() => {
+    report = this._reportJobResult({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId}).finally(() => {
       this.inflightReports.delete(report)
     })
 

package/build/environment-handlers/node.js

@@ -1052,12 +1052,11 @@ export default class VelociousEnvironmentHandlerNode extends Base{
     // `db:tenants:migrate <tenant>`, which migrates only tenant databases — the
     // framework store lives elsewhere (typically the default DB) and was already
     // ensured by the plain `db:migrate` that precedes it. Reaching into it here would
-    // open a fresh connection to that shared database and re-run the concurrency
-    // reconcile UPDATEs; under `db:tenants:migrate --parallel N` that happens once per
-    // tenant worker, and the concurrent auto-committed UPDATEs on the shared
-    // background_jobs / background_job_concurrency rows InnoDB-deadlock
-    // (ER_LOCK_DEADLOCK). So skip when the framework DB isn't in this set; the runtime
-    // store still creates it lazily if a plain migrate never ran.
+    // open a fresh connection to that shared database once per tenant worker for
+    // schema work that is already applied. So skip when the framework DB isn't in
+    // this set; the runtime store still creates it lazily if a plain migrate never
+    // ran. Queue-cap reconciliation never runs on this path at all — it belongs to
+    // main-process startup (`BackgroundJobsStore#reconcileQueueConcurrency`).
     if (!frameworkDb) return
 
     // Reuse the connection db:migrate already holds for this database; opening a

package/build/src/background-jobs/job-runner.d.ts

@@ -11,10 +11,10 @@ export declare class BackgroundJobPerformedFailure extends Error {
  * @param {object} [options] - Runner options.
  * @param {boolean} [options.closeConnections] - Whether to gracefully close framework connections after the job.
  * @param {boolean} [options.manageProcessTitle] - Whether to set the per-job process title and restore it afterwards. Off for concurrent pooled runners, where interleaved snapshot/restore of the single process-wide `process.title` would corrupt it; the pooled child owns an aggregate title instead.
- * @returns {Promise<void>} - Resolves when complete.
+ * @returns {Promise<"completed" | "rescheduled">} - Acknowledged outcome.
  */
 export default function runJobPayload(payload: import("./types.js").BackgroundJobPayload, { closeConnections, manageProcessTitle }?: {
     closeConnections?: boolean;
     manageProcessTitle?: boolean;
-}): Promise<void>;
+}): Promise<"completed" | "rescheduled">;
 //# sourceMappingURL=job-runner.d.ts.map

package/build/src/background-jobs/job-runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"job-runner.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/job-runner.js"],"names":[],"mappings":"AAQA,qBAAa,6BAA8B,SAAQ,KAAK;IACtD;;;OAGG;IACH,YAAY,KAAK,EAFN,KAEM,EAGhB;CACF;AA0DD;;;;;;;GAOG;AACH,wBAA8B,aAAa,CAAC,OAAO,EANxC,OAAO,YAAY,EAAE,oBAMmB,EAAE,EAAC,gBAAuB,EAAE,kBAAyB,EAAC,AALtG,CAGA,EAFA;IAA0B,gBAAgB,AAA1C,CACA,EADQ,OAAO,CACf;IAA0B,kBAAkB,AAA5C,CACA,EADQ,OAAO,CACf;CAE2G,GAFjG,OAAO,CAAC,IAAI,CAAC,CAoEzB"}
+{"version":3,"file":"job-runner.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/job-runner.js"],"names":[],"mappings":"AASA,qBAAa,6BAA8B,SAAQ,KAAK;IACtD;;;OAGG;IACH,YAAY,KAAK,EAFN,KAEM,EAGhB;CACF;AA0DD;;;;;;;GAOG;AACH,wBAA8B,aAAa,CAAC,OAAO,EANxC,OAAO,YAAY,EAAE,oBAMmB,EAAE,EAAC,gBAAuB,EAAE,kBAAyB,EAAC,AALtG,CAGA,EAFA;IAA0B,gBAAgB,AAA1C,CACA,EADQ,OAAO,CACf;IAA0B,kBAAkB,AAA5C,CACA,EADQ,OAAO,CACf;CAE2G,GAFjG,OAAO,CAAC,WAAW,GAAG,aAAa,CAAC,CAsFhD"}