velocious 1.0.589 → 1.0.591

package/README.md

@@ -2072,6 +2072,10 @@ Tests default to a 60-second timeout. Override per test with `{timeoutSeconds: 5
 
 Request tests share transaction-active, non-tenant database connections with their in-process HTTP handlers. Eligibility is evaluated when each request is dispatched, so a hook can start a transaction and issue a request in the same callback. This makes uncommitted setup visible to handlers while preserving rollback isolation. Without an active transaction, handlers use independent pooled connections, so concurrency and locking tests can opt out of transaction cleanup and exercise production-style connections. Shared connection state is scoped to the test lifecycle and cleared around each test. See [docs/testing-guidelines.md](docs/testing-guidelines.md#request-test-database-connections).
 
+Transactional tests also share active non-tenant connections with real forked, reusable pooled, and spawned background-job child runners through a per-attempt test-only loopback broker. Parent setup and child writes therefore occupy the same physical transaction and roll back together, including background-job persistence. Multiple configured databases route by identifier; tenant-only databases remain excluded. Tests using `{transaction: false, truncate: true}` retain ordinary independent physical connections for concurrency and locking coverage. See [docs/testing-guidelines.md](docs/testing-guidelines.md#request-test-database-connections).
+
+Warm pooled children receive the active broker capability per job, discard retained proxy state when the capability changes, and fail closed if a transactional dispatch lacks coordinates. Child transaction/savepoint work holds a FIFO lease on the parent physical connection until the matching root release or rollback.
+
 # Writing a request test
 
 First create a test file under something like the following path 'src/routes/accounts/create-test.js' with something like the following content:
@@ -2345,6 +2349,26 @@ await MyJob.performLaterWithOptions({
 
 Until `scheduledAtMs` is reached, the job remains queued but is not eligible for dispatch. The event-driven dispatcher arms its timer for the earliest future job and wakes at that timestamp. Omitting `scheduledAtMs` keeps the immediate-enqueue behavior.
 
+A running job that cannot proceed yet can reschedule its same durable row without
+recording a failure—for example, when a non-blocking lock is busy:
+
+```js
+async perform(accountId) {
+  if (!(await Account.tryAcquireRefreshLock(accountId))) {
+    this.rescheduleIn(30_000)
+  }
+
+  await refreshAccount(accountId)
+}
+```
+
+`rescheduleIn(delayMs)` requires a finite, non-negative safe-integer millisecond
+delay and never returns: it stops the current `perform`, releases its worker and
+concurrency slots, and makes the same job eligible again after the delay. This is
+normal control flow, not failure retry: attempts and failure metadata remain
+unchanged, retries are not consumed, and failure/error events are not emitted.
+See [Rescheduling a running job](docs/background-jobs.md#rescheduling-a-running-job).
+
 Use a durable stable key when the same logical one-off schedule must be moved or cancelled without retaining its transient job id:
 
 ```js

package/build/background-jobs/job-runner.js

@@ -3,6 +3,7 @@
 import configurationResolver from "../configuration-resolver.js"
 import BackgroundJobRegistry from "./job-registry.js"
 import BackgroundJobsStatusReporter from "./status-reporter.js"
+import BackgroundJobRescheduleSignal from "./reschedule-signal.js"
 
 const BEACON_READY_TIMEOUT_MS = 5000
 
@@ -79,7 +80,7 @@ function runnerProcessTitle(JobClass, payload) {
  * @param {object} [options] - Runner options.
  * @param {boolean} [options.closeConnections] - Whether to gracefully close framework connections after the job.
  * @param {boolean} [options.manageProcessTitle] - Whether to set the per-job process title and restore it afterwards. Off for concurrent pooled runners, where interleaved snapshot/restore of the single process-wide `process.title` would corrupt it; the pooled child owns an aggregate title instead.
- * @returns {Promise<void>} - Resolves when complete.
+ * @returns {Promise<"completed" | "rescheduled">} - Acknowledged outcome.
  */
 export default async function runJobPayload(payload, {closeConnections = true, manageProcessTitle = true} = {}) {
   const configuration = await configurationResolver()
@@ -109,6 +110,23 @@ export default async function runJobPayload(payload, {closeConnections = true, m
         await perform.apply(jobInstance, payload.args || [])
       })
     } catch (error) {
+      if (error instanceof BackgroundJobRescheduleSignal) {
+        if (payload.id) {
+          await reporter.reportWithRetry({
+            jobId: payload.id,
+            status: "rescheduled",
+            delayMs: error.delayMs,
+            handoffId: payload.handoffId,
+            workerId: payload.workerId,
+            handedOffAtMs: payload.handedOffAtMs,
+            maxDurationMs: 30000,
+            retryPersistErrors: true
+          })
+        }
+
+        return "rescheduled"
+      }
+
       const performedError = error instanceof Error ? error : new Error(String(error))
       if (payload.id) {
         await reporter.reportWithRetry({
@@ -135,6 +153,7 @@ export default async function runJobPayload(payload, {closeConnections = true, m
         maxDurationMs: 30000
       })
     }
+    return "completed"
   } finally {
     // Restore the runner's base title so a lingering/idle runner (or a reused
     // one) doesn't misreport a finished job as still running.

package/build/background-jobs/job.js

@@ -1,6 +1,7 @@
 // @ts-check
 
 import BackgroundJobsClient from "./client.js"
+import BackgroundJobRescheduleSignal from "./reschedule-signal.js"
 
 /**
  * Base class for background jobs.
@@ -43,6 +44,20 @@ export default class VelociousJob {
    */
   static processTitle = undefined
 
+  /**
+   * Stops this performance and reschedules the same logical job row. This is
+   * normal control flow: it does not count as a failure or consume a retry.
+   * @param {number} delayMs - Non-negative safe-integer delay in milliseconds.
+   * @returns {never} - This method never returns.
+   */
+  rescheduleIn(delayMs) {
+    if (!Number.isSafeInteger(delayMs) || delayMs < 0) {
+      throw new TypeError("background job reschedule delayMs must be a non-negative safe integer")
+    }
+
+    throw new BackgroundJobRescheduleSignal(delayMs)
+  }
+
   /**
    * Runs job name.
    * @returns {string} - Job name.

package/build/background-jobs/main.js

@@ -601,6 +601,11 @@ export default class BackgroundJobsMain {
 
     if (message?.type === "job-failed") {
       this._handleJobFailed({jsonSocket, message})
+      return
+    }
+
+    if (message?.type === "job-reschedule") {
+      this._handleJobReschedule({jsonSocket, message})
     }
   }
 
@@ -885,6 +890,40 @@ export default class BackgroundJobsMain {
     }
   }
 
+  /**
+   * Persists a normal job reschedule outcome and wakes scheduled dispatch.
+   * @param {object} args - Options.
+   * @param {JsonSocket} args.jsonSocket - JSON socket.
+   * @param {import("./types.js").BackgroundJobRescheduleMessage} args.message - Message.
+   * @returns {Promise<void>} - Resolves when handled.
+   */
+  async _handleJobReschedule({jsonSocket, message}) {
+    try {
+      const accepted = await this.store.markRescheduled({
+        jobId: message.jobId,
+        delayMs: message.delayMs,
+        handoffId: message.handoffId,
+        workerId: message.workerId,
+        handedOffAtMs: message.handedOffAtMs
+      })
+      if (accepted && message.handoffId) {
+        this._forgetHandoff({handoffId: message.handoffId, jobId: message.jobId})
+      }
+      jsonSocket.send({type: "job-updated", jobId: message.jobId})
+      this._notifyEnqueued()
+      await this._drain()
+    } catch (error) {
+      const normalizedError = error instanceof Error ? error : new Error(String(error))
+      const payload = {context: {jobId: message.jobId, stage: "background-job-reschedule"}, error: normalizedError}
+      const errorEvents = this.configuration.getErrorEvents()
+
+      this.logger.error(() => ["Failed to update job reschedule:", normalizedError])
+      errorEvents.emit("framework-error", payload)
+      errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
+      jsonSocket.send({type: "job-update-error", jobId: message.jobId, error: "Failed to update job"})
+    }
+  }
+
   /**
    * Runs handle job failed.
    * @param {object} args - Options.

package/build/background-jobs/pooled-runner-broker-identity.js

@@ -0,0 +1,76 @@
+// @ts-check
+
+export default class PooledRunnerBrokerIdentity {
+  /**
+   * Creates a pooled runner identity coordinator.
+   * @param {{closeConnections: () => Promise<void>}} args - Connection cleanup hook.
+   */
+  constructor({closeConnections}) {
+    this.closeConnections = closeConnections
+    /** @type {string | undefined} */
+    this.activeIdentity = undefined
+    /** @type {{identity: string, promise: Promise<void>} | undefined} */
+    this.pending = undefined
+    this.activeUsers = 0
+  }
+
+  /**
+   * Gets the current prepared identity.
+   * @returns {string | undefined} - Current prepared identity.
+   */
+  current() { return this.activeIdentity }
+
+  /**
+   * Prepares one identity, sharing an in-flight same-identity rotation.
+   * @param {import("../testing/shared-transaction-proxy-driver.js").SharedTransactionBrokerJobConfig} config - Dispatch configuration.
+   * @returns {Promise<void>} - Resolves after stale connections close.
+   */
+  async prepare(config) {
+    const identity = JSON.stringify(config)
+    if (this.pending) {
+      if (this.pending.identity !== identity) throw new Error("Pooled runner cannot mix shared transaction broker capabilities concurrently")
+      return await this.pending.promise
+    }
+    if (this.activeIdentity === identity) return
+    if (this.activeUsers > 0) throw new Error("Pooled runner cannot mix shared transaction broker capabilities concurrently")
+    if (this.activeIdentity === undefined) {
+      this.activeIdentity = identity
+      return
+    }
+
+    const promise = this.rotate(identity)
+    this.pending = {identity, promise}
+    try {
+      await promise
+    } finally {
+      this.pending = undefined
+    }
+  }
+
+  /**
+   * Runs work while preventing a different identity from replacing its connections.
+   * @template T
+   * @param {import("../testing/shared-transaction-proxy-driver.js").SharedTransactionBrokerJobConfig} config - Dispatch configuration.
+   * @param {() => Promise<T>} callback - Job callback.
+   * @returns {Promise<T>} - Job result.
+   */
+  async run(config, callback) {
+    await this.prepare(config)
+    this.activeUsers++
+    try {
+      return await callback()
+    } finally {
+      this.activeUsers--
+    }
+  }
+
+  /**
+   * Rotates retained connection state to an identity.
+   * @param {string} identity - Target identity.
+   * @returns {Promise<void>} - Resolves after rotation.
+   */
+  async rotate(identity) {
+    await this.closeConnections()
+    this.activeIdentity = identity
+  }
+}

package/build/background-jobs/pooled-runner-child.js

@@ -3,6 +3,8 @@
 import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
 import { closeRunnerConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
 import setRunnerProcessTitle from "./runner-process-title.js"
+import PooledRunnerBrokerIdentity from "./pooled-runner-broker-identity.js"
+import { runWithSharedTransactionBrokerConfig } from "../testing/shared-transaction-proxy-driver.js"
 
 const BASE_PROCESS_TITLE = "velocious background-jobs-runner"
 
@@ -33,6 +35,9 @@ async function shutdownRunner(exitCode) {
  * @type {Set<string>}
  */
 const runningJobIds = new Set()
+const brokerIdentity = new PooledRunnerBrokerIdentity({
+  closeConnections: async () => await closeRunnerConnections(currentConfigurationOrNull())
+})
 
 /**
  * Sets an aggregate process title from the current in-flight count. A child runs
@@ -51,11 +56,11 @@ function updateProcessTitle() {
 /**
  * Checks whether an IPC value is a runnable pooled job message.
  * @param {ReturnType<typeof JSON.parse>} message - IPC message.
- * @returns {message is {type: "job", payload: import("./types.js").BackgroundJobPayload & {id: string}}} - Whether this is a valid job message.
+ * @returns {message is {type: "job", payload: import("./types.js").BackgroundJobPayload & {id: string}, sharedTransactionBroker?: import("../testing/shared-transaction-proxy-driver.js").SharedTransactionBrokerJobConfig}} - Whether this is a valid job message.
  */
 function isJobMessage(message) {
   if (!message || typeof message !== "object") return false
-  const record = /** @type {{type?: ReturnType<typeof JSON.parse>, payload?: ReturnType<typeof JSON.parse>}} */ (message)
+  const record = /** @type {{type?: ReturnType<typeof JSON.parse>, payload?: ReturnType<typeof JSON.parse>, sharedTransactionBroker?: ReturnType<typeof JSON.parse>}} */ (message)
 
   return record.type === "job" && !!record.payload && typeof record.payload === "object" && typeof record.payload.id === "string"
 }
@@ -65,7 +70,7 @@ function isJobMessage(message) {
  * @param {object} args - Outcome.
  * @param {string} args.jobId - Job id.
  * @param {boolean} args.acknowledged - Whether the terminal report was acknowledged.
- * @param {"completed" | "failed"} [args.status] - Acknowledged terminal status.
+ * @param {"completed" | "failed" | "rescheduled"} [args.status] - Acknowledged outcome.
  * @param {Error} [args.error] - Reporting error when acknowledgement was not obtained.
  * @returns {Promise<void>} - Resolves after IPC accepts the message.
  */
@@ -95,12 +100,17 @@ function sendOutcome({jobId, acknowledged, status, error}) {
  * per-job try/catch) ends the child, which the worker sees as an exit and
  * reclaims for the whole in-flight set.
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Job payload.
+ * @param {import("../testing/shared-transaction-proxy-driver.js").SharedTransactionBrokerJobConfig} sharedTransactionBroker - Per-job broker configuration.
  * @returns {Promise<void>} - Resolves after reporting.
  */
-async function runJob(payload) {
+async function runJob(payload, sharedTransactionBroker) {
   try {
-    await runJobPayload(payload, {closeConnections: false, manageProcessTitle: false})
-    await sendOutcome({jobId: payload.id, acknowledged: true, status: "completed"})
+    const status = await runWithSharedTransactionBrokerConfig(sharedTransactionBroker, async () => {
+      return await brokerIdentity.run(sharedTransactionBroker, async () => {
+        return await runJobPayload(payload, {closeConnections: false, manageProcessTitle: false})
+      })
+    })
+    await sendOutcome({jobId: payload.id, acknowledged: true, status})
   } catch (error) {
     if (error instanceof BackgroundJobPerformedFailure) {
       await sendOutcome({jobId: payload.id, acknowledged: true, status: "failed"})
@@ -125,7 +135,7 @@ function handleMessage(message) {
 
   runningJobIds.add(message.payload.id)
   updateProcessTitle()
-  void runJob(message.payload)
+  void runJob(message.payload, message.sharedTransactionBroker || {expected: false})
 }
 
 process.on("message", (message) => handleMessage(message))

package/build/background-jobs/reschedule-signal.js

@@ -0,0 +1,14 @@
+// @ts-check
+
+/** Internal control flow raised by `VelociousJob#rescheduleIn`. */
+export default class BackgroundJobRescheduleSignal extends Error {
+  /**
+   * Creates a reschedule control signal.
+   * @param {number} delayMs - Reschedule delay in milliseconds.
+   */
+  constructor(delayMs) {
+    super(`Reschedule background job in ${delayMs}ms`)
+    this.name = "BackgroundJobRescheduleSignal"
+    this.delayMs = delayMs
+  }
+}

package/build/background-jobs/status-reporter.js

@@ -36,14 +36,15 @@ export default class BackgroundJobsStatusReporter {
    * Runs report.
    * @param {object} args - Options.
    * @param {string} args.jobId - Job id.
-   * @param {"completed" | "failed"} args.status - Status.
+   * @param {"completed" | "failed" | "rescheduled"} args.status - Status.
+   * @param {number} [args.delayMs] - Reschedule delay in milliseconds.
    * @param {ReturnType<typeof JSON.parse>} [args.error] - Error.
    * @param {string} [args.handoffId] - Handoff lease id.
    * @param {number} [args.handedOffAtMs] - Handed off timestamp.
    * @param {string} [args.workerId] - Worker id.
    * @returns {Promise<void>} - Resolves when reported.
    */
-  async report({jobId, status, error, handoffId, handedOffAtMs, workerId}) {
+  async report({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId}) {
     const config = this.configuration.getBackgroundJobsConfig()
     const host = this.host || config.host
     const port = typeof this.port === "number" ? this.port : config.port
@@ -57,8 +58,9 @@ export default class BackgroundJobsStatusReporter {
         signal: control.signal,
         onConnect: (jsonSocket) => {
           jsonSocket.send({
-            type: status === "completed" ? "job-complete" : "job-failed",
+            type: status === "completed" ? "job-complete" : status === "rescheduled" ? "job-reschedule" : "job-failed",
             jobId,
+            delayMs,
             handoffId,
             workerId,
             handedOffAtMs,
@@ -83,7 +85,8 @@ export default class BackgroundJobsStatusReporter {
    * Runs report with retry.
    * @param {object} args - Options.
    * @param {string} args.jobId - Job id.
-   * @param {"completed" | "failed"} args.status - Status.
+   * @param {"completed" | "failed" | "rescheduled"} args.status - Status.
+   * @param {number} [args.delayMs] - Reschedule delay in milliseconds.
    * @param {ReturnType<typeof JSON.parse>} [args.error] - Error.
    * @param {string} [args.handoffId] - Handoff lease id.
    * @param {number} [args.handedOffAtMs] - Handed off timestamp.
@@ -92,13 +95,13 @@ export default class BackgroundJobsStatusReporter {
    * @param {boolean} [args.retryPersistErrors] - Retry a `BackgroundJobUpdateError` (main's `job-update-error`, i.e. a transient DB failure while persisting the terminal status) instead of throwing immediately. Off by default so short-lived forked/spawned runners keep failing loudly and exit non-zero to be reclaimed; on for the long-lived worker, which cannot exit-to-reclaim and would otherwise strand the job in `handed_off`.
    * @returns {Promise<void>} - Resolves when reported.
    */
-  async reportWithRetry({jobId, status, error, handoffId, handedOffAtMs, workerId, maxDurationMs, retryPersistErrors = false}) {
+  async reportWithRetry({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId, maxDurationMs, retryPersistErrors = false}) {
     let attempt = 0
     const startTime = Date.now()
 
     while (true) {
       try {
-        await this.report({jobId, status, error, handoffId, handedOffAtMs, workerId})
+        await this.report({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId})
         return
       } catch (error) {
         // A `BackgroundJobUpdateError` means main answered `job-update-error`, which it

package/build/background-jobs/store.js

@@ -6,6 +6,7 @@ import TableData from "../database/table-data/index.js"
 import VelociousError from "../velocious-error.js"
 import BackgroundJobRecord from "./job-record.js"
 import normalizeBackgroundJobError from "./normalize-error.js"
+import {coordinateSharedTransactionConnection} from "../testing/shared-transaction-connection-coordinator.js"
 
 /**
  * PreparedBackgroundJob type.
@@ -694,6 +695,48 @@ export default class BackgroundJobsStore {
     }))
   }
 
+  /**
+   * Returns an active handoff to the queue at a caller-requested future time.
+   * This is normal job control flow: it preserves failure attempts and metadata.
+   * @param {object} args - Options.
+   * @param {string} args.jobId - Job id.
+   * @param {number} args.delayMs - Delay from persistence time in milliseconds.
+   * @param {string} [args.handoffId] - Handoff lease id.
+   * @param {string} [args.workerId] - Worker id.
+   * @param {number} [args.handedOffAtMs] - Handed off timestamp.
+   * @returns {Promise<boolean>} - Whether the fenced report was accepted.
+   */
+  async markRescheduled({jobId, delayMs, handoffId, workerId, handedOffAtMs}) {
+    await this.ensureReady()
+    this._validateRescheduleDelayMs(delayMs)
+
+    return await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
+      const job = await this._getJobRowById(db, jobId)
+
+      if (!job) return false
+      if (!this._shouldAcceptReport({job, handoffId, workerId, handedOffAtMs})) return false
+
+      await this._lockConcurrencyRow(db, job.concurrencyKey)
+      const scheduledAtMs = this._rescheduledAtMs(delayMs)
+      const affectedRows = await this._updateAffectedRows(db, {
+        tableName: JOBS_TABLE,
+        data: {
+          status: "queued",
+          scheduled_at_ms: scheduledAtMs,
+          handed_off_at_ms: null,
+          handoff_id: null,
+          worker_id: null
+        },
+        conditions: this._activeHandoffConditions(job)
+      })
+
+      if (affectedRows !== 1) return false
+      await this._releaseConcurrency(db, job.concurrencyKey)
+      await this._recordStatusTransition(db, "handed_off", "queued")
+      return true
+    }))
+  }
+
   /**
    * Runs mark returned to queue.
    * @param {object} args - Options.
@@ -1067,6 +1110,33 @@ export default class BackgroundJobsStore {
     throw VelociousError.safe("background job scheduledAtMs must be a non-negative safe integer")
   }
 
+  /**
+   * Resolves a reschedule delay against persistence time.
+   * @param {number} delayMs - Delay in milliseconds.
+   * @returns {number} - Future eligibility timestamp.
+   */
+  _rescheduledAtMs(delayMs) {
+    this._validateRescheduleDelayMs(delayMs)
+
+    const scheduledAtMs = Date.now() + delayMs
+    if (!Number.isSafeInteger(scheduledAtMs)) {
+      throw VelociousError.safe("background job reschedule scheduledAtMs must be a safe integer")
+    }
+
+    return scheduledAtMs
+  }
+
+  /**
+   * Validates a public reschedule delay before persistence work begins.
+   * @param {number} delayMs - Delay in milliseconds.
+   * @returns {void}
+   */
+  _validateRescheduleDelayMs(delayMs) {
+    if (!Number.isSafeInteger(delayMs) || delayMs < 0) {
+      throw VelociousError.safe("background job reschedule delayMs must be a non-negative safe integer")
+    }
+  }
+
   /**
    * Validates a stable schedule key at the public storage boundary.
    * @param {string} scheduleKey - Stable logical schedule key.
@@ -2307,23 +2377,19 @@ export default class BackgroundJobsStore {
    * @returns {Promise<T>} - Callback result.
    */
   async _withDb(callback) {
-    const pool = this.configuration.getDatabasePool(this.getDatabaseIdentifier())
-    let callbackCalled = false
-    /**
-     * Defines result.
-     * @type {T | undefined} */
-    let result
-
-    await pool.withConnection({name: "Background jobs store"}, async (db) => {
-      callbackCalled = true
-      result = await callback(db)
-    })
+    const databaseIdentifier = this.getDatabaseIdentifier()
+    const pool = this.configuration.getDatabasePool(databaseIdentifier)
 
-    if (!callbackCalled) {
-      throw new Error("Background jobs store callback was not invoked")
+    if (!pool.testSharedConnection()) {
+      return await pool.withConnection({name: "Background jobs store"}, callback)
     }
 
-    return /** @type {T} */ (result)
+    return await this.configuration.runWithTestSharedConnectionContexts(async () => {
+      return await this.configuration.ensureConnections({databaseIdentifiers: [databaseIdentifier], name: "Background jobs store"}, async (dbs) => {
+        const connection = dbs[databaseIdentifier]
+        return await coordinateSharedTransactionConnection(connection, async () => await callback(connection))
+      })
+    })
   }
 
   /**

package/build/background-jobs/types.js

@@ -99,11 +99,12 @@
  * @typedef {{type: "job", payload: BackgroundJobPayload}} BackgroundJobJobMessage
  * @typedef {{type: "job-complete", jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number}} BackgroundJobCompleteMessage
  * @typedef {{type: "job-failed", jobId: string, error?: ReturnType<typeof JSON.parse>, handoffId?: string, workerId?: string, handedOffAtMs?: number}} BackgroundJobFailedMessage
+ * @typedef {{type: "job-reschedule", jobId: string, delayMs: number, handoffId?: string, workerId?: string, handedOffAtMs?: number}} BackgroundJobRescheduleMessage
  * @typedef {{type: "job-updated", jobId: string}} BackgroundJobUpdatedMessage
  * @typedef {{type: "job-update-error", jobId: string, error?: string}} BackgroundJobUpdateErrorMessage
  */
 /**
- * @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobHeartbeatMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobReplaceScheduledMessage | BackgroundJobScheduleReplacedMessage | BackgroundJobReplaceScheduledErrorMessage | BackgroundJobCancelScheduledMessage | BackgroundJobScheduleCancelledMessage | BackgroundJobCancelScheduledErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
+ * @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobHeartbeatMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobReplaceScheduledMessage | BackgroundJobScheduleReplacedMessage | BackgroundJobReplaceScheduledErrorMessage | BackgroundJobCancelScheduledMessage | BackgroundJobScheduleCancelledMessage | BackgroundJobCancelScheduledErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobRescheduleMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
  */
 
 export const nothing = {}

package/build/background-jobs/worker.js

@@ -9,6 +9,7 @@ import BackgroundJobsStatusReporter from "./status-reporter.js"
 import {randomUUID} from "crypto"
 import {fileURLToPath} from "node:url"
 import shutdownLifecycle from "../utils/shutdown-lifecycle.js"
+import BackgroundJobRescheduleSignal from "./reschedule-signal.js"
 
 /**
  * Per-forked-child timeout bookkeeping.
@@ -574,6 +575,18 @@ export default class BackgroundJobsWorker {
         workerId: payload.workerId || this.workerId
       })
     } catch (error) {
+      if (error instanceof BackgroundJobRescheduleSignal) {
+        this._reportJobResultInBackground({
+          jobId: payload.id,
+          status: "rescheduled",
+          delayMs: error.delayMs,
+          handoffId: payload.handoffId,
+          handedOffAtMs: payload.handedOffAtMs,
+          workerId: payload.workerId || this.workerId
+        })
+        return
+      }
+
       this._reportJobResultInBackground({
         jobId: payload.id,
         status: "failed",
@@ -680,13 +693,26 @@ export default class BackgroundJobsWorker {
 
       state.inflight.set(payload.id, {payload, resolve, timeoutTimer})
       try {
-        child.send({type: "job", payload})
+        child.send({type: "job", payload, sharedTransactionBroker: this._pooledJobSharedTransactionBrokerConfig()})
       } catch (error) {
         void this._handlePooledChildFailure({child, error})
       }
     })
   }
 
+  /**
+   * Captures the current test attempt's broker mode at dispatch time. A warm
+   * pooled child must never rely on its immutable fork-time environment.
+   * @returns {import("../testing/shared-transaction-proxy-driver.js").SharedTransactionBrokerJobConfig} - Per-job broker configuration.
+   */
+  _pooledJobSharedTransactionBrokerConfig() {
+    const serialized = process.env.VELOCIOUS_TEST_SHARED_TRANSACTION_BROKER
+    if (!serialized) return {expected: false}
+
+    const config = JSON.parse(Buffer.from(serialized, "base64url").toString("utf8"))
+    return {...config, expected: true}
+  }
+
   /**
    * Selects a pooled child to run the next job, or undefined when every non-retiring
    * child is already full (the caller then lazily spawns one). Among children with a
@@ -779,7 +805,7 @@ export default class BackgroundJobsWorker {
     const config = configuration.getBackgroundJobsConfig()
     const child = fork(POOLED_RUNNER_ENTRY_PATH, [], {
       cwd: configuration.getDirectory(), execArgv: [], stdio: ["ignore", "ignore", "ignore", "ipc"],
-      env: Object.assign({}, process.env, {VELOCIOUS_ENV: configuration.getEnvironment(), VELOCIOUS_BACKGROUND_JOBS_HOST: config.host, VELOCIOUS_BACKGROUND_JOBS_PORT: `${config.port}`})
+      env: Object.assign({}, process.env, {VELOCIOUS_BACKGROUND_JOB_CHILD: "1", VELOCIOUS_ENV: configuration.getEnvironment(), VELOCIOUS_BACKGROUND_JOBS_HOST: config.host, VELOCIOUS_BACKGROUND_JOBS_PORT: `${config.port}`})
     })
     this.pooledChildren.add(child)
     this.inflightProcessChildren.add(child)
@@ -976,6 +1002,7 @@ export default class BackgroundJobsWorker {
       execArgv: [],
       stdio: ["ignore", "ignore", "ignore", "ipc"],
       env: Object.assign({}, process.env, {
+        VELOCIOUS_BACKGROUND_JOB_CHILD: "1",
         VELOCIOUS_ENV: configuration.getEnvironment(),
         VELOCIOUS_BACKGROUND_JOBS_HOST: backgroundJobsConfig.host,
         VELOCIOUS_BACKGROUND_JOBS_PORT: `${backgroundJobsConfig.port}`
@@ -1203,6 +1230,7 @@ export default class BackgroundJobsWorker {
       detached: true,
       stdio: "ignore",
       env: Object.assign({}, process.env, {
+        VELOCIOUS_BACKGROUND_JOB_CHILD: "1",
         VELOCIOUS_ENV: configuration.getEnvironment(),
         VELOCIOUS_BACKGROUND_JOBS_HOST: backgroundJobsConfig.host,
         VELOCIOUS_BACKGROUND_JOBS_PORT: `${backgroundJobsConfig.port}`,
@@ -1233,14 +1261,15 @@ export default class BackgroundJobsWorker {
    * Runs report job result.
    * @param {object} args - Options.
    * @param {string} args.jobId - Job id.
-   * @param {"completed" | "failed"} args.status - Status.
+   * @param {"completed" | "failed" | "rescheduled"} args.status - Status.
+   * @param {number} [args.delayMs] - Reschedule delay in milliseconds.
    * @param {ReturnType<typeof JSON.parse>} [args.error] - Error.
    * @param {string} [args.handoffId] - Handoff lease id.
    * @param {number} [args.handedOffAtMs] - Handed off timestamp.
    * @param {string} [args.workerId] - Worker id.
    * @returns {Promise<void>} - Resolves when reported.
    */
-  async _reportJobResult({jobId, status, error, handoffId, handedOffAtMs, workerId}) {
+  async _reportJobResult({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId}) {
     if (!this.statusReporter) return
 
     try {
@@ -1248,7 +1277,7 @@ export default class BackgroundJobsWorker {
       // long-lived and cannot exit to trigger orphan reclaim, so dropping the
       // completion here would strand the job in `handed_off` forever — fatal for a
       // `max_concurrency: 1` job (a stranded row blocks every future run).
-      await this.statusReporter.reportWithRetry({jobId, status, error, handoffId, handedOffAtMs, workerId, retryPersistErrors: true})
+      await this.statusReporter.reportWithRetry({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId, retryPersistErrors: true})
     } catch (reportError) {
       console.error("Background job status reporting failed:", reportError)
     }
@@ -1260,20 +1289,21 @@ export default class BackgroundJobsWorker {
    * graceful `stop()` can drain in-flight reports before closing the socket.
    * @param {object} args - Options.
    * @param {string} args.jobId - Job id.
-   * @param {"completed" | "failed"} args.status - Status.
+   * @param {"completed" | "failed" | "rescheduled"} args.status - Status.
+   * @param {number} [args.delayMs] - Reschedule delay in milliseconds.
    * @param {ReturnType<typeof JSON.parse>} [args.error] - Error.
    * @param {string} [args.handoffId] - Handoff lease id.
    * @param {number} [args.handedOffAtMs] - Handed off timestamp.
    * @param {string} [args.workerId] - Worker id.
    * @returns {void}
    */
-  _reportJobResultInBackground({jobId, status, error, handoffId, handedOffAtMs, workerId}) {
+  _reportJobResultInBackground({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId}) {
     /**
      * Defines report.
      * @type {Promise<void>} */
     let report
 
-    report = this._reportJobResult({jobId, status, error, handoffId, handedOffAtMs, workerId}).finally(() => {
+    report = this._reportJobResult({jobId, status, delayMs, error, handoffId, handedOffAtMs, workerId}).finally(() => {
       this.inflightReports.delete(report)
     })