velocious 1.0.556 → 1.0.558

package/src/background-jobs/pooled-runner-child.js

@@ -1,12 +1,30 @@
 // @ts-check
 
 import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
+import { closeRunnerConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
 import setRunnerProcessTitle from "./runner-process-title.js"
 
 const BASE_PROCESS_TITLE = "velocious background-jobs-runner"
 
 setRunnerProcessTitle()
 
+let shuttingDown = false
+
+/**
+ * Closes the runner's connections — releasing any advisory lock a killed-mid-pass
+ * job still holds — before exiting, instead of leaving a half-open session that
+ * keeps the lock until the DB server's `wait_timeout`.
+ * @param {number} exitCode - Process exit code.
+ * @returns {Promise<void>}
+ */
+async function shutdownRunner(exitCode) {
+  if (shuttingDown) return
+  shuttingDown = true
+
+  await closeRunnerConnections(currentConfigurationOrNull())
+  process.exit(exitCode)
+}
+
 /**
  * Ids of jobs currently running in this child. A pooled child runs up to
  * `pooledRunnerConcurrency` jobs at once (the worker only dispatches within that
@@ -111,5 +129,5 @@ function handleMessage(message) {
 }
 
 process.on("message", (message) => handleMessage(message))
-process.once("disconnect", () => process.exit(0))
-for (const signal of ["SIGTERM", "SIGINT"]) process.once(signal, () => process.exit(1))
+process.once("disconnect", () => void shutdownRunner(0))
+for (const signal of ["SIGTERM", "SIGINT"]) process.once(signal, () => void shutdownRunner(1))

package/src/background-jobs/runner-graceful-shutdown.js

@@ -0,0 +1,73 @@
+// @ts-check
+
+import timeout from "awaitery/build/timeout.js"
+
+import Configuration, {CurrentConfigurationNotSetError} from "../configuration.js"
+
+/** Bounded grace for closing framework connections on shutdown before forcing exit. */
+const SHUTDOWN_CLOSE_TIMEOUT_MS = 5000
+
+/**
+ * The subset of a configuration a runner closes on shutdown. Typed structurally so
+ * the shutdown path stays typechecked without a broad cast, and a future signature
+ * drift surfaces at the call sites (and in tests) instead of hiding behind `any`.
+ * @typedef {{disconnectBeacon: () => Promise<void>, closeDatabaseConnections: () => Promise<void>}} RunnerCloseableConfiguration
+ */
+
+/**
+ * Gracefully closes a background-job runner's framework connections (beacon +
+ * database) on shutdown, so the database server ends the session and releases any
+ * advisory lock the runner still holds *immediately* — instead of leaving a
+ * half-open session that keeps the lock (e.g. the build-planner lock) held until
+ * the server's idle `wait_timeout` (hours).
+ *
+ * A runner child otherwise exits abruptly on shutdown (`process.exit`) and relies
+ * on the OS tearing down its sockets. A named lock releases only when its owning
+ * session ends, and an abrupt exit / container teardown does not reliably deliver a
+ * clean disconnect to the server — so the lock leaks. Sending a real close here
+ * (`COM_QUIT` via `closeDatabaseConnections`) makes the server end the session and
+ * release the lock deterministically.
+ *
+ * The beacon disconnect and the database close run independently, each bounded by
+ * `closeTimeoutMs`: the database close releases the locks and must still run even if
+ * the beacon disconnect hangs (a wedged beacon socket during teardown), and neither
+ * may block the exit forever. A failed or timed-out close is thrown, not swallowed —
+ * the caller surfaces it (a lock that failed to release must be visible).
+ * @param {RunnerCloseableConfiguration | null} configuration - Configuration whose connections to close; null when none is set (nothing to close).
+ * @param {number} [closeTimeoutMs] - Max time to spend on each close before giving up.
+ * @returns {Promise<void>} - Resolves once both closes have settled; rejects if either failed.
+ */
+export async function closeRunnerConnections(configuration, closeTimeoutMs = SHUTDOWN_CLOSE_TIMEOUT_MS) {
+  if (!configuration) return
+
+  const [beaconResult, databaseResult] = await Promise.allSettled([
+    timeout({timeout: closeTimeoutMs}, () => configuration.disconnectBeacon()),
+    timeout({timeout: closeTimeoutMs}, () => configuration.closeDatabaseConnections())
+  ])
+
+  /** @type {unknown[]} */
+  const errors = []
+
+  if (beaconResult.status == "rejected") errors.push(beaconResult.reason)
+  if (databaseResult.status == "rejected") errors.push(databaseResult.reason)
+
+  if (errors.length == 1) throw errors[0]
+  if (errors.length > 1) throw new AggregateError(errors, "Failed to close background-job runner connections on shutdown")
+}
+
+/**
+ * The current configuration, or null when none has been set yet — a runner that is
+ * signalled before it runs any job holds no connections (and no locks) to close.
+ * Only that expected "not set yet" case is treated as null; any other error is a real
+ * fault and is rethrown rather than masked.
+ * @returns {Configuration | null} - The current configuration, or null when none is set.
+ */
+export function currentConfigurationOrNull() {
+  try {
+    return Configuration.current()
+  } catch (error) {
+    if (error instanceof CurrentConfigurationNotSetError) return null
+
+    throw error
+  }
+}

package/src/background-jobs/worker.js

@@ -8,6 +8,7 @@ import configurationResolver from "../configuration-resolver.js"
 import BackgroundJobsStatusReporter from "./status-reporter.js"
 import {randomUUID} from "crypto"
 import {fileURLToPath} from "node:url"
+import shutdownLifecycle from "../utils/shutdown-lifecycle.js"
 
 /** Grace period after SIGTERM before a lingering process runner is SIGKILLed. */
 const FORKED_CHILD_SIGKILL_GRACE_MS = 5000
@@ -73,8 +74,10 @@ export default class BackgroundJobsWorker {
    * @param {number} [args.forkedChildSigkillGraceMs] - Override the grace period between SIGTERM and SIGKILL when reaping lingering process runners on stop.
    * @param {number} [args.heartbeatIntervalMs] - Override the liveness heartbeat interval (default 15000ms).
    * @param {number} [args.jobTimeoutMs] - Override the wall-clock timeout for forked and pooled jobs from `configuration.getBackgroundJobsConfig()`. `0` disables it.
+   * @param {boolean} [args.closeDatabaseConnectionsOnStop] - Whether stop owns closing the configuration's database pools (default true).
+   * @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the worker finishes stopping.
    */
-  constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, forkedChildSigkillGraceMs, heartbeatIntervalMs, jobTimeoutMs} = {}) {
+  constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, forkedChildSigkillGraceMs, heartbeatIntervalMs, jobTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped} = {}) {
     /**
      * Narrows the runtime value to the documented type.
      * @type {Promise<import("../configuration.js").default>} */
@@ -85,6 +88,8 @@ export default class BackgroundJobsWorker {
     this.configuration = undefined
     this.host = host
     this.port = port
+    this.closeDatabaseConnectionsOnStop = closeDatabaseConnectionsOnStop
+    this.onStopped = onStopped
     /**
      * Constructor override for the inline-job concurrency cap. When unset
      * the cap is read from `configuration.getBackgroundJobsConfig()` in
@@ -136,6 +141,8 @@ export default class BackgroundJobsWorker {
      */
     this.jobTimeoutMsOverride = typeof jobTimeoutMs === "number" ? jobTimeoutMs : undefined
     this.shouldStop = false
+    /** @type {Promise<void> | undefined} */
+    this.stopPromise = undefined
     this.workerId = randomUUID()
     this.heartbeatIntervalMs = typeof heartbeatIntervalMs === "number" && heartbeatIntervalMs >= 1
       ? heartbeatIntervalMs
@@ -195,6 +202,8 @@ export default class BackgroundJobsWorker {
    * @returns {Promise<void>} - Resolves when connected.
    */
   async start() {
+    this.shouldStop = false
+    this.stopPromise = undefined
     this.configuration = await this.configurationPromise
     this.configuration.setCurrent()
     await this.configuration.initialize({type: "background-jobs-worker"})
@@ -241,37 +250,53 @@ export default class BackgroundJobsWorker {
    * @param {number} [args.timeoutMs] - Max wait for in-flight jobs (per phase) in ms.
    * @returns {Promise<void>} - Resolves when stopped.
    */
-  async stop({timeoutMs} = {}) {
-    if (this.shouldStop) return
-    this.shouldStop = true
-    this._stopHeartbeat()
+  stop({timeoutMs} = {}) {
+    if (!this.stopPromise) this.stopPromise = this._stop({timeoutMs})
 
-    // Announce drain so main stops dispatching but keeps the connection
-    // open until we close it ourselves below.
-    if (this.jsonSocket) {
-      try {
-        this.jsonSocket.send({type: "draining"})
-      } catch {
-        // Socket may already be closing; nothing to do.
-      }
-    }
+    return this.stopPromise
+  }
 
-    await this._drainInflight(this.inflightInlineJobs, timeoutMs)
-    await this._drainInflight(this.inflightPooledJobs, timeoutMs)
-    await this._drainInflight(this.inflightProcessJobs, timeoutMs)
-    await this._terminateProcessChildren()
-    // Give in-flight result reports (now decoupled from job slots) a bounded
-    // chance to land before the socket closes.
-    await this._drainInflight(this.inflightReports, timeoutMs)
+  /**
+   * Runs the worker shutdown lifecycle once.
+   * @param {object} [args] - Options.
+   * @param {number} [args.timeoutMs] - Max wait for in-flight jobs (per phase) in ms.
+   * @returns {Promise<void>} - Resolves when stopped.
+   */
+  async _stop({timeoutMs} = {}) {
+    this.shouldStop = true
+    this._stopHeartbeat()
 
-    if (this.jsonSocket) this.jsonSocket.close()
-    if (this.configuration) {
-      try {
-        await this.configuration.disconnectBeacon()
-      } finally {
-        await this.configuration.closeDatabaseConnections()
+    await shutdownLifecycle({
+      onStopped: this.onStopped,
+      shutdown: async () => {
+        // Announce drain so main stops dispatching but keeps the connection
+        // open until we close it ourselves below.
+        if (this.jsonSocket) {
+          try {
+            this.jsonSocket.send({type: "draining"})
+          } catch {
+            // Socket may already be closing; nothing to do.
+          }
+        }
+
+        await this._drainInflight(this.inflightInlineJobs, timeoutMs)
+        await this._drainInflight(this.inflightPooledJobs, timeoutMs)
+        await this._drainInflight(this.inflightProcessJobs, timeoutMs)
+        await this._terminateProcessChildren()
+        // Give in-flight result reports (now decoupled from job slots) a bounded
+        // chance to land before the socket closes.
+        await this._drainInflight(this.inflightReports, timeoutMs)
+
+        if (this.jsonSocket) this.jsonSocket.close()
+        if (!this.configuration) return
+
+        try {
+          await this.configuration.disconnectBeacon()
+        } finally {
+          if (this.closeDatabaseConnectionsOnStop) await this.configuration.closeDatabaseConnections()
+        }
       }
-    }
+    })
   }
 
   /**

package/src/configuration.js

@@ -119,6 +119,15 @@ export default class VelociousConfiguration {
    * Close database connections promise.
    * @type {Promise<void> | null} */
   _closeDatabaseConnectionsPromise = null
+
+  /**
+   * Dedicated advisory-lock connections currently holding a lock. These are spawned
+   * outside the pools' tracked sets (so a hold-timeout lock survives pool checkouts),
+   * so `closeDatabaseConnections` would otherwise walk past them; tracking them here
+   * lets a shutdown close them and release the lock instead of orphaning it.
+   * @type {Set<import("./database/drivers/base.js").default>} */
+  _advisoryLockConnections = new Set()
+
   /**
    * Runs current.
    * @returns {VelociousConfiguration} - The current.
@@ -2986,6 +2995,53 @@ export default class VelociousConfiguration {
     })
   }
 
+  /**
+   * Registers a dedicated connection that currently holds an advisory lock, so a
+   * shutdown can close it and release the lock. See `_advisoryLockConnections`.
+   * @param {import("./database/drivers/base.js").default} connection - The dedicated lock connection.
+   * @returns {void}
+   */
+  registerAdvisoryLockConnection(connection) {
+    this._advisoryLockConnections.add(connection)
+  }
+
+  /**
+   * Unregisters a dedicated advisory-lock connection once its lock scope ends and the
+   * connection has been (or is about to be) closed by its owner.
+   * @param {import("./database/drivers/base.js").default} connection - The dedicated lock connection.
+   * @returns {void}
+   */
+  unregisterAdvisoryLockConnection(connection) {
+    this._advisoryLockConnections.delete(connection)
+  }
+
+  /**
+   * Closes every registered dedicated advisory-lock connection, ending its session so
+   * the DB server releases the lock. Every connection is attempted before any failure
+   * is surfaced, so one stuck close does not leave the others' locks held; a failure is
+   * then thrown (never swallowed), aggregated when more than one connection failed.
+   * @returns {Promise<void>} - Resolves once all have been closed; rejects if any failed.
+   */
+  async _closeAdvisoryLockConnections() {
+    const connections = [...this._advisoryLockConnections]
+
+    this._advisoryLockConnections.clear()
+
+    /** @type {unknown[]} */
+    const errors = []
+
+    for (const connection of connections) {
+      try {
+        await connection.close()
+      } catch (error) {
+        errors.push(error)
+      }
+    }
+
+    if (errors.length == 1) throw errors[0]
+    if (errors.length > 1) throw new AggregateError(errors, "Failed to close dedicated advisory-lock connections")
+  }
+
   /**
    * Closes active database connections and clears global connections.
    * @returns {Promise<void>} - Resolves when complete.
@@ -3000,21 +3056,30 @@ export default class VelociousConfiguration {
     const constructors = new Set()
 
     this._closeDatabaseConnectionsPromise = (async () => {
-      for (const pool of Object.values(this.databasePools)) {
-        if (!pool) continue
-
-        await pool.closeAll()
+      try {
+        // Close dedicated advisory-lock connections first: they are spawned outside the
+        // pools' tracked sets, so `pool.closeAll()` would not reach them and a lock held
+        // by a runner torn down mid-pass would leak until the DB server's `wait_timeout`.
+        // Still close the pools even if this throws, so a stuck lock connection does not
+        // leave the rest of the connections open.
+        await this._closeAdvisoryLockConnections()
+      } finally {
+        for (const pool of Object.values(this.databasePools)) {
+          if (!pool) continue
+
+          await pool.closeAll()
+
+          const PoolClass = /** @type {typeof import("./database/pool/base.js").default} */ (pool.constructor)
+          constructors.add(PoolClass)
+        }
 
-        const PoolClass = /** @type {typeof import("./database/pool/base.js").default} */ (pool.constructor)
-        constructors.add(PoolClass)
-      }
+        for (const PoolClass of constructors) {
+          PoolClass.clearGlobalConnections(this)
+        }
 
-      for (const PoolClass of constructors) {
-        PoolClass.clearGlobalConnections(this)
+        // Allow models to be re-initialized after connections are closed.
+        this._modelsInitialized = false
       }
-
-      // Allow models to be re-initialized after connections are closed.
-      this._modelsInitialized = false
     })()
 
     try {

package/src/database/advisory-lock-runner.js

@@ -150,13 +150,22 @@ export default class AdvisoryLockRunner {
   async withDedicatedConnection(callback) {
     const connection = await this.configuration.getDatabasePool(this.databaseIdentifier).spawnConnection()
 
+    // The spawned driver owns its physical connection unless it borrows a shared one
+    // via `getConnection`. An owned connection lives outside the pools' tracked sets,
+    // so register it while the lock is held: a shutdown then closes it (releasing the
+    // lock) instead of orphaning a half-open session until the DB `wait_timeout`.
+    const ownsConnection = !connection.getArgs().getConnection
+
+    if (ownsConnection) this.configuration.registerAdvisoryLockConnection(connection)
+
     try {
       return await callback(connection)
     } finally {
-      if (connection.getArgs().getConnection) {
-        await connection.releaseHeldAdvisoryLocks()
-      } else {
+      if (ownsConnection) {
+        this.configuration.unregisterAdvisoryLockConnection(connection)
         await connection.close()
+      } else {
+        await connection.releaseHeldAdvisoryLocks()
       }
     }
   }

package/src/database/record/index.js

@@ -1710,6 +1710,12 @@ class VelociousDatabaseRecord {
    */
   static switchesTenantDatabase(databaseIdentifierOrResolver) {
     this._tenantDatabaseIdentifierResolver = databaseIdentifierOrResolver
+
+    if (this._translationClass) {
+      const translatedModelClass = this
+
+      this._translationClass.switchesTenantDatabase(({tenant}) => translatedModelClass.getTenantDatabaseIdentifier(tenant))
+    }
   }
 
   /**
@@ -2814,6 +2820,12 @@ class VelociousDatabaseRecord {
     TranslationClass.setTableName(this.getTranslationsTableName())
     TranslationClass.belongsTo(belongsTo)
 
+    if (this.hasTenantDatabaseIdentifierResolver()) {
+      const translatedModelClass = this
+
+      TranslationClass.switchesTenantDatabase(({tenant}) => translatedModelClass.getTenantDatabaseIdentifier(tenant))
+    }
+
     this._translationClass = TranslationClass
 
     return this._translationClass