velocious 1.0.690 → 1.0.692

package/src/background-jobs/worker.js

@@ -1617,7 +1617,11 @@ export default class BackgroundJobsWorker {
    * whole drain window, which can stretch to the longest in-flight job. The
    * retiring child stops receiving new jobs and is terminated only once its
    * in-flight set drains, so a long-running job (e.g. a build) is never cut
-   * off.
+   * off. When the pool already carries its bounded over-cap child, retiring
+   * the last working child is deferred instead of emptying the working pool:
+   * the child keeps serving and a later observation retries the retirement
+   * once a draining exit has freed a cap slot, so advertised capacity cannot
+   * collapse to zero inside a wave.
    * @param {import("node:child_process").ChildProcess} child - Child to retire.
    * @returns {void}
    */
@@ -1625,6 +1629,17 @@ export default class BackgroundJobsWorker {
     const state = this.pooledChildStates.get(child)
     if (!state || state.retiring) return
 
+    // Retiring the last non-retiring child while the pool already carries its
+    // bounded over-cap child would leave advertised capacity at zero until the
+    // next draining exit — up to the longest in-flight job (tens of minutes
+    // with builds), stalling dispatch cluster-wide. Defer this retirement
+    // instead: the child keeps serving and a later observation retries once a
+    // drain has freed a cap slot for its replacement.
+    if (!this.shouldStop && this.configuration && this._nonRetiringPooledChildCount() <= 1 && this.pooledChildren.size > this.pooledRunnerCount) {
+      this._logPooledCapacityEvent("pooled-capacity-retirement-deferred")
+      return
+    }
+
     state.retiring = true
     // Best-effort pre-warm: skip when stopping (no new work) or before the
     // worker is initialized (no configuration to fork a child from). The cap
@@ -1688,9 +1703,11 @@ export default class BackgroundJobsWorker {
    * surface that reaches the production log). Used by the bounded liveness
    * replacement in `_ensureNonRetiringChild`: the spawn that keeps dispatch
    * alive through a full retirement wave (at a retirement mark or a child
-   * exit), or the bounded starvation when an over-cap child is already
-   * draining and the pool must wait for an exit.
-   * @param {"pooled-capacity-liveness-spawn" | "pooled-capacity-starved"} event - Event name.
+   * exit), the deferred retirement of the last working child while the pool
+   * already carries its bounded over-cap child (retried after the next drain
+   * frees a cap slot), or the bounded starvation when an over-cap child is
+   * already draining and the pool must wait for an exit.
+   * @param {"pooled-capacity-liveness-spawn" | "pooled-capacity-starved" | "pooled-capacity-retirement-deferred"} event - Event name.
    * @returns {void}
    */
   _logPooledCapacityEvent(event) {

package/src/configuration-types.js

@@ -93,6 +93,7 @@
  * @property {boolean} [readOnly] - Whether writes should be blocked for this database.
  * @property {string} [schema] - Default schema for unqualified table lookups (MSSQL).
  * @property {boolean} [schemaCache] - Whether schema metadata should be cached on the driver. Defaults to true.
+ * @property {number} [schemaMetadataRetryDeadlineMs] - Bounded deadline (ms) for outlasting a cached schema-metadata read the server skipped because concurrent online DDL is modifying the table. Defaults to 60000.
  * @property {object} [record] - Record-level configuration.
  * @property {boolean} [record.transactions] - Whether record operations should use transactions.
  * @property {boolean} [reset] - Whether to reset the database on startup.

package/src/database/drivers/base.js

@@ -183,6 +183,12 @@ const SCHEMA_INVALIDATION_SCAN_LIMIT = 8192
 /** Maximum checkout-name characters inspected by retry diagnostics. */
 const OPERATION_NAME_SCAN_LIMIT = 1024
 const REDACTED_DIAGNOSTIC_LABEL = "[REDACTED]"
+/** First backoff between retries of a schema-metadata read blocked by concurrent online DDL. */
+const SCHEMA_METADATA_RETRY_INITIAL_DELAY_MS = 250
+/** Largest backoff between retries of a schema-metadata read blocked by concurrent online DDL. */
+const SCHEMA_METADATA_RETRY_MAX_DELAY_MS = 2000
+/** Default bounded deadline for outlasting a concurrent online-DDL metadata skip. */
+const SCHEMA_METADATA_RETRY_DEADLINE_MS = 60000
 
 /**
  * Builds a non-reversible, stable SQL fingerprint without retaining SQL text. Literal spelling is
@@ -777,6 +783,52 @@ export default class VelociousDatabaseDriversBase {
     return this.getArgs().schemaCache !== false
   }
 
+  /**
+   * Runs schema metadata retry deadline ms.
+   * @returns {number} - Bounded deadline for outlasting transient schema-metadata contention.
+   */
+  _schemaMetadataRetryDeadlineMs() {
+    const configured = this.getArgs().schemaMetadataRetryDeadlineMs
+
+    return typeof configured === "number" ? configured : SCHEMA_METADATA_RETRY_DEADLINE_MS
+  }
+
+  /**
+   * Whether an error raised by a schema-metadata read is transient contention that a bounded
+   * retry can outlast. Drivers override this for their own contention signatures; the default
+   * treats every failure as final so only drivers that prove the transient shape retry it.
+   * @param {unknown} _error - Error raised by a schema-metadata read.
+   * @returns {boolean} - Whether the read may be retried.
+   */
+  _isTransientSchemaMetadataError(_error) {
+    return false
+  }
+
+  /**
+   * Runs a schema-metadata read, retrying transient concurrent-DDL contention until the bounded
+   * deadline. Non-transient failures and exhausted deadlines propagate the original error.
+   * @template T
+   * @param {() => Promise<T>} callback - Cache-miss schema read.
+   * @returns {Promise<T>} - Resolves with the read result.
+   */
+  async _withSchemaMetadataDdlRetry(callback) {
+    const deadlineMs = this._schemaMetadataRetryDeadlineMs()
+    const startedAt = Date.now()
+    let delayMs = SCHEMA_METADATA_RETRY_INITIAL_DELAY_MS
+
+    for (;;) {
+      try {
+        return await callback()
+      } catch (error) {
+        if (!this._isTransientSchemaMetadataError(error)) throw error
+        if (Date.now() - startedAt + delayMs > deadlineMs) throw error
+
+        await wait(delayMs)
+        delayMs = Math.min(delayMs * 2, SCHEMA_METADATA_RETRY_MAX_DELAY_MS)
+      }
+    }
+  }
+
   /**
    * Runs cached schema metadata.
    * @template T
@@ -785,7 +837,7 @@ export default class VelociousDatabaseDriversBase {
    * @returns {Promise<T>} - Resolves with the cached metadata.
    */
   async _cachedSchemaMetadata(cacheKey, callback) {
-    if (!this._schemaCacheEnabled()) return await callback()
+    if (!this._schemaCacheEnabled()) return await this._withSchemaMetadataDdlRetry(callback)
 
     const existingPromise = this._schemaCache.get(cacheKey)
 
@@ -793,7 +845,7 @@ export default class VelociousDatabaseDriversBase {
       return /** @type {T} */ (this._schemaCacheReturnValue(await existingPromise))
     }
 
-    const promise = (async () => await callback())()
+    const promise = (async () => await this._withSchemaMetadataDdlRetry(callback))()
 
     this._schemaCache.set(cacheKey, promise)
 

package/src/database/drivers/mysql/index.js

@@ -723,6 +723,29 @@ export default class VelociousDatabaseDriversMysql extends Base{
     return insert.toSql()
   }
 
+  /**
+   * MariaDB skips information-schema/SHOW reads for a table whose definition is being modified
+   * by concurrent online DDL (ER_WARN_I_S_SKIPPED_TABLE). The build completes in seconds, so the
+   * skip is transient contention that the bounded schema-metadata retry can outlast. Query
+   * helpers wrap the driver error, so the code is searched through the cause chain.
+   * @param {unknown} error - Error raised by a schema-metadata read.
+   * @returns {boolean} - Whether the read hit concurrent-DDL metadata contention.
+   */
+  _isTransientSchemaMetadataError(error) {
+    let currentError = /** @type {{code?: string, cause?: unknown} | undefined} */ (
+      /** @type {unknown} */ (error)
+    )
+
+    for (let depth = 0; depth < 5 && currentError; depth++) {
+      if (currentError.code === "ER_WARN_I_S_SKIPPED_TABLE") return true
+      currentError = /** @type {{code?: string, cause?: unknown} | undefined} */ (
+        /** @type {unknown} */ (currentError.cause)
+      )
+    }
+
+    return false
+  }
+
   /**
    * Runs get tables.
    * @returns {Promise<Array<import("../base-table.js").default>>} - Resolves with the tables.

package/src/database/drivers/mysql/query.js

@@ -171,7 +171,7 @@ export default async function query(pool, sql, {signal} = {}) {
           connection.release()
         }
 
-        reject(new Error(`Query failed because of ${error}: ${sql}`))
+        reject(new Error(`Query failed because of ${error}: ${sql}`, {cause: error}))
 
         return
       }