velocious 1.0.714 → 1.0.715

package/build/src/utils/bounded-task-runner.js

@@ -0,0 +1,44 @@
+// @ts-check
+/**
+ * Runs one callback per item with a bounded number of callbacks in flight.
+ *
+ * Used where a startup pass has to touch many independent records (model
+ * metadata, per-table schema reads) and running them sequentially multiplies the
+ * round-trip latency by the item count, while running them all at once would
+ * open an unbounded number of database connections.
+ */
+export default class BoundedTaskRunner {
+    /**
+     * Runs the worker for every item, keeping at most `concurrency` callbacks in
+     * flight and returning the results in input order.
+     * @template T, R
+     * @param {object} args - Run args.
+     * @param {number} [args.concurrency] - Maximum callbacks in flight.
+     * @param {T[]} args.items - Items to process.
+     * @param {(item: T, index: number) => Promise<R>} args.worker - Callback per item.
+     * @returns {Promise<R[]>} - Results in input order.
+     */
+    static async run({ concurrency = 8, items, worker }) {
+        if (!Number.isInteger(concurrency) || concurrency < 1) {
+            throw new Error(`Concurrency must be a positive integer but got ${String(concurrency)}`);
+        }
+        /** @type {R[]} */
+        const results = new Array(items.length);
+        let nextIndex = 0;
+        const runNext = async () => {
+            while (nextIndex < items.length) {
+                const index = nextIndex;
+                nextIndex += 1;
+                results[index] = await worker(items[index], index);
+            }
+        };
+        /** @type {Promise<void>[]} */
+        const runners = [];
+        for (let index = 0; index < Math.min(concurrency, items.length); index++) {
+            runners.push(runNext());
+        }
+        await Promise.all(runners);
+        return results;
+    }
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYm91bmRlZC10YXNrLXJ1bm5lci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy91dGlscy9ib3VuZGVkLXRhc2stcnVubmVyLmpzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLFlBQVk7QUFFWjs7Ozs7OztHQU9HO0FBQ0gsTUFBTSxDQUFDLE9BQU8sT0FBTyxpQkFBaUI7SUFDcEM7Ozs7Ozs7OztPQVNHO0lBQ0gsTUFBTSxDQUFDLEtBQUssQ0FBQyxHQUFHLENBQUMsRUFBQyxXQUFXLEdBQUcsQ0FBQyxFQUFFLEtBQUssRUFBRSxNQUFNLEVBQUM7UUFDL0MsSUFBSSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsV0FBVyxDQUFDLElBQUksV0FBVyxHQUFHLENBQUMsRUFBRSxDQUFDO1lBQ3RELE1BQU0sSUFBSSxLQUFLLENBQUMsa0RBQWtELE1BQU0sQ0FBQyxXQUFXLENBQUMsRUFBRSxDQUFDLENBQUE7UUFDMUYsQ0FBQztRQUVELGtCQUFrQjtRQUNsQixNQUFNLE9BQU8sR0FBRyxJQUFJLEtBQUssQ0FBQyxLQUFLLENBQUMsTUFBTSxDQUFDLENBQUE7UUFDdkMsSUFBSSxTQUFTLEdBQUcsQ0FBQyxDQUFBO1FBRWpCLE1BQU0sT0FBTyxHQUFHLEtBQUssSUFBSSxFQUFFO1lBQ3pCLE9BQU8sU0FBUyxHQUFHLEtBQUssQ0FBQyxNQUFNLEVBQUUsQ0FBQztnQkFDaEMsTUFBTSxLQUFLLEdBQUcsU0FBUyxDQUFBO2dCQUV2QixTQUFTLElBQUksQ0FBQyxDQUFBO2dCQUNkLE9BQU8sQ0FBQyxLQUFLLENBQUMsR0FBRyxNQUFNLE1BQU0sQ0FBQyxLQUFLLENBQUMsS0FBSyxDQUFDLEVBQUUsS0FBSyxDQUFDLENBQUE7WUFDcEQsQ0FBQztRQUNILENBQUMsQ0FBQTtRQUVELDhCQUE4QjtRQUM5QixNQUFNLE9BQU8sR0FBRyxFQUFFLENBQUE7UUFFbEIsS0FBSyxJQUFJLEtBQUssR0FBRyxDQUFDLEVBQUUsS0FBSyxHQUFHLElBQUksQ0FBQyxHQUFHLENBQUMsV0FBVyxFQUFFLEtBQUssQ0FBQyxNQUFNLENBQUMsRUFBRSxLQUFLLEVBQUUsRUFBRSxDQUFDO1lBQ3pFLE9BQU8sQ0FBQyxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUMsQ0FBQTtRQUN6QixDQUFDO1FBRUQsTUFBTSxPQUFPLENBQUMsR0FBRyxDQUFDLE9BQU8sQ0FBQyxDQUFBO1FBRTFCLE9BQU8sT0FBTyxDQUFBO0lBQ2hCLENBQUM7Q0FDRiIsInNvdXJjZXNDb250ZW50IjpbIi8vIEB0cy1jaGVja1xuXG4vKipcbiAqIFJ1bnMgb25lIGNhbGxiYWNrIHBlciBpdGVtIHdpdGggYSBib3VuZGVkIG51bWJlciBvZiBjYWxsYmFja3MgaW4gZmxpZ2h0LlxuICpcbiAqIFVzZWQgd2hlcmUgYSBzdGFydHVwIHBhc3MgaGFzIHRvIHRvdWNoIG1hbnkgaW5kZXBlbmRlbnQgcmVjb3JkcyAobW9kZWxcbiAqIG1ldGFkYXRhLCBwZXItdGFibGUgc2NoZW1hIHJlYWRzKSBhbmQgcnVubmluZyB0aGVtIHNlcXVlbnRpYWxseSBtdWx0aXBsaWVzIHRoZVxuICogcm91bmQtdHJpcCBsYXRlbmN5IGJ5IHRoZSBpdGVtIGNvdW50LCB3aGlsZSBydW5uaW5nIHRoZW0gYWxsIGF0IG9uY2Ugd291bGRcbiAqIG9wZW4gYW4gdW5ib3VuZGVkIG51bWJlciBvZiBkYXRhYmFzZSBjb25uZWN0aW9ucy5cbiAqL1xuZXhwb3J0IGRlZmF1bHQgY2xhc3MgQm91bmRlZFRhc2tSdW5uZXIge1xuICAvKipcbiAgICogUnVucyB0aGUgd29ya2VyIGZvciBldmVyeSBpdGVtLCBrZWVwaW5nIGF0IG1vc3QgYGNvbmN1cnJlbmN5YCBjYWxsYmFja3MgaW5cbiAgICogZmxpZ2h0IGFuZCByZXR1cm5pbmcgdGhlIHJlc3VsdHMgaW4gaW5wdXQgb3JkZXIuXG4gICAqIEB0ZW1wbGF0ZSBULCBSXG4gICAqIEBwYXJhbSB7b2JqZWN0fSBhcmdzIC0gUnVuIGFyZ3MuXG4gICAqIEBwYXJhbSB7bnVtYmVyfSBbYXJncy5jb25jdXJyZW5jeV0gLSBNYXhpbXVtIGNhbGxiYWNrcyBpbiBmbGlnaHQuXG4gICAqIEBwYXJhbSB7VFtdfSBhcmdzLml0ZW1zIC0gSXRlbXMgdG8gcHJvY2Vzcy5cbiAgICogQHBhcmFtIHsoaXRlbTogVCwgaW5kZXg6IG51bWJlcikgPT4gUHJvbWlzZTxSPn0gYXJncy53b3JrZXIgLSBDYWxsYmFjayBwZXIgaXRlbS5cbiAgICogQHJldHVybnMge1Byb21pc2U8UltdPn0gLSBSZXN1bHRzIGluIGlucHV0IG9yZGVyLlxuICAgKi9cbiAgc3RhdGljIGFzeW5jIHJ1bih7Y29uY3VycmVuY3kgPSA4LCBpdGVtcywgd29ya2VyfSkge1xuICAgIGlmICghTnVtYmVyLmlzSW50ZWdlcihjb25jdXJyZW5jeSkgfHwgY29uY3VycmVuY3kgPCAxKSB7XG4gICAgICB0aHJvdyBuZXcgRXJyb3IoYENvbmN1cnJlbmN5IG11c3QgYmUgYSBwb3NpdGl2ZSBpbnRlZ2VyIGJ1dCBnb3QgJHtTdHJpbmcoY29uY3VycmVuY3kpfWApXG4gICAgfVxuXG4gICAgLyoqIEB0eXBlIHtSW119ICovXG4gICAgY29uc3QgcmVzdWx0cyA9IG5ldyBBcnJheShpdGVtcy5sZW5ndGgpXG4gICAgbGV0IG5leHRJbmRleCA9IDBcblxuICAgIGNvbnN0IHJ1bk5leHQgPSBhc3luYyAoKSA9PiB7XG4gICAgICB3aGlsZSAobmV4dEluZGV4IDwgaXRlbXMubGVuZ3RoKSB7XG4gICAgICAgIGNvbnN0IGluZGV4ID0gbmV4dEluZGV4XG5cbiAgICAgICAgbmV4dEluZGV4ICs9IDFcbiAgICAgICAgcmVzdWx0c1tpbmRleF0gPSBhd2FpdCB3b3JrZXIoaXRlbXNbaW5kZXhdLCBpbmRleClcbiAgICAgIH1cbiAgICB9XG5cbiAgICAvKiogQHR5cGUge1Byb21pc2U8dm9pZD5bXX0gKi9cbiAgICBjb25zdCBydW5uZXJzID0gW11cblxuICAgIGZvciAobGV0IGluZGV4ID0gMDsgaW5kZXggPCBNYXRoLm1pbihjb25jdXJyZW5jeSwgaXRlbXMubGVuZ3RoKTsgaW5kZXgrKykge1xuICAgICAgcnVubmVycy5wdXNoKHJ1bk5leHQoKSlcbiAgICB9XG5cbiAgICBhd2FpdCBQcm9taXNlLmFsbChydW5uZXJzKVxuXG4gICAgcmV0dXJuIHJlc3VsdHNcbiAgfVxufVxuIl19

package/build/utils/bounded-task-runner.js

@@ -0,0 +1,51 @@
+// @ts-check
+
+/**
+ * Runs one callback per item with a bounded number of callbacks in flight.
+ *
+ * Used where a startup pass has to touch many independent records (model
+ * metadata, per-table schema reads) and running them sequentially multiplies the
+ * round-trip latency by the item count, while running them all at once would
+ * open an unbounded number of database connections.
+ */
+export default class BoundedTaskRunner {
+  /**
+   * Runs the worker for every item, keeping at most `concurrency` callbacks in
+   * flight and returning the results in input order.
+   * @template T, R
+   * @param {object} args - Run args.
+   * @param {number} [args.concurrency] - Maximum callbacks in flight.
+   * @param {T[]} args.items - Items to process.
+   * @param {(item: T, index: number) => Promise<R>} args.worker - Callback per item.
+   * @returns {Promise<R[]>} - Results in input order.
+   */
+  static async run({concurrency = 8, items, worker}) {
+    if (!Number.isInteger(concurrency) || concurrency < 1) {
+      throw new Error(`Concurrency must be a positive integer but got ${String(concurrency)}`)
+    }
+
+    /** @type {R[]} */
+    const results = new Array(items.length)
+    let nextIndex = 0
+
+    const runNext = async () => {
+      while (nextIndex < items.length) {
+        const index = nextIndex
+
+        nextIndex += 1
+        results[index] = await worker(items[index], index)
+      }
+    }
+
+    /** @type {Promise<void>[]} */
+    const runners = []
+
+    for (let index = 0; index < Math.min(concurrency, items.length); index++) {
+      runners.push(runNext())
+    }
+
+    await Promise.all(runners)
+
+    return results
+  }
+}

package/package.json

@@ -3,7 +3,7 @@
     "velocious": "build/bin/velocious.js"
   },
   "name": "velocious",
-  "version": "1.0.714",
+  "version": "1.0.715",
   "main": "build/index.js",
   "types": "build/index.d.ts",
   "files": [

package/src/background-jobs/local-store.js

@@ -1598,6 +1598,9 @@ export default class LocalBackgroundJobsStore {
       failedAtMs: this._numberOrNull(row.failed_at_ms),
       handedOffAtMs: this._numberOrNull(row.handed_off_at_ms),
       handoffId: row.handoff_id === null || row.handoff_id === undefined ? null : String(row.handoff_id),
+      // Local (browser/Expo) dispatch is strictly FIFO and has no priority column,
+      // so every row reports the default priority.
+      priority: 0,
       id: String(row.id),
       jobName: String(row.job_name),
       lastError: row.last_error === null || row.last_error === undefined ? null : String(row.last_error),

package/src/background-jobs/store.js

@@ -41,6 +41,7 @@ import {
  * @property {string} jobId - New job id.
  * @property {string} jobName - Job name.
  * @property {number} maxRetries - Retry cap.
+ * @property {number} priority - Dispatch priority within the queue.
  * @property {string} queue - Queue name.
  * @property {number} scheduledAtMs - Eligibility timestamp.
  * @property {number | null} timeoutMs - Per-job timeout override, or null when omitted.
@@ -137,8 +138,22 @@ export const BACKGROUND_JOB_COUNT_BUCKETS = ["all", "queued", "handed_off", "com
 const COUNTED_JOB_STATUSES = BACKGROUND_JOB_COUNT_BUCKETS.slice(1)
 const MAX_JOB_TIMEOUT_MS = 2_147_483_647
 const JOB_TIMEOUT_VALIDATION_MESSAGE = `background job timeoutMs must be a finite non-positive number or an integer between 1 and ${MAX_JOB_TIMEOUT_MS}`
+const BACKGROUND_JOB_PRIORITY_VALIDATION_MESSAGE = "background job priority must be a non-negative integer"
 const ORPHANED_AFTER_MS = 2 * 60 * 60 * 1000
 
+/**
+ * Priority a job is demoted to when it is re-queued after a failed attempt.
+ * A retry has already consumed capacity and time, so it must never overtake
+ * work that has not had its first chance yet. Enqueue priorities are validated
+ * to be non-negative integers, which makes `-1` the lowest reachable priority.
+ */
+export const RETRY_BACKGROUND_JOB_PRIORITY = -1
+
+/**
+ * Priority a job gets when the producer does not ask for one.
+ */
+export const DEFAULT_BACKGROUND_JOB_PRIORITY = 0
+
 /**
  * Columns the dashboard is allowed to sort job listings by, mapped to their
  * database column names. Restricting to this set keeps the sort parameter
@@ -1159,6 +1174,12 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
       const priorityOrder = this._queuePriorityOrderSql(db)
 
       if (priorityOrder) query = query.order(`${priorityOrder} DESC`)
+
+      // Job-level priority applies inside the queue ranking above. A retry is
+      // stored at RETRY_BACKGROUND_JOB_PRIORITY (-1), so it sorts behind every
+      // first attempt without delaying itself further than the backoff already
+      // does. COALESCE covers rows written before the column existed.
+      query = query.order(`COALESCE(${db.quoteColumn("priority")}, ${DEFAULT_BACKGROUND_JOB_PRIORITY}) DESC`)
     }
 
     query = query
@@ -1965,12 +1986,33 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
       jobId: randomUUID(),
       jobName,
       maxRetries: this._normalizeMaxRetries(options?.maxRetries),
+      priority: this._normalizeJobPriority(options),
       queue,
       scheduledAtMs: this._normalizeScheduledAtMs(options?.scheduledAtMs, createdAtMs),
       timeoutMs: this._normalizeJobTimeoutMs(options)
     }
   }
 
+  /**
+   * Normalizes a requested dispatch priority. Priorities order jobs within their
+   * queue on the dispatch path only: higher goes first, and the default keeps
+   * FIFO order. Negative priorities are rejected because the store reserves them
+   * for {@link RETRY_BACKGROUND_JOB_PRIORITY}, which must stay the lowest.
+   * @param {import("./types.js").BackgroundJobOptions | undefined} options - Job options.
+   * @returns {number} - Priority.
+   */
+  _normalizeJobPriority(options) {
+    const priority = options?.priority
+
+    if (priority === undefined) return DEFAULT_BACKGROUND_JOB_PRIORITY
+
+    if (typeof priority !== "number" || !Number.isInteger(priority) || priority < 0) {
+      throw VelociousError.safe(BACKGROUND_JOB_PRIORITY_VALIDATION_MESSAGE)
+    }
+
+    return priority
+  }
+
   /**
    * Normalizes a per-job timeout while preserving omitted (worker fallback)
    * separately from explicitly disabled.
@@ -2024,6 +2066,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
         execution_mode: preparedJob.executionMode,
         queue: preparedJob.queue,
         max_retries: preparedJob.maxRetries,
+        priority: preparedJob.priority,
         attempts: 0,
         status: "queued",
         scheduled_at_ms: preparedJob.scheduledAtMs,
@@ -2334,6 +2377,30 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
       }
     }
 
+    const priorityLockName = `${MIGRATION_SCOPE}:priority_column`
+    const priorityLockAcquired = await db.acquireAdvisoryLock(priorityLockName)
+
+    if (!priorityLockAcquired) throw new Error("Failed to acquire background jobs priority schema lock")
+
+    try {
+      db.clearSchemaCache()
+      const priorityTable = await db.getTableByNameOrFail(JOBS_TABLE)
+
+      if (!(await priorityTable.getColumnByName("priority"))) {
+        const tableData = new TableData(JOBS_TABLE)
+        tableData.bigint("priority", {null: true})
+        const sqls = await db.alterTableSQLs(tableData)
+
+        for (const sql of sqls) {
+          await db.query(sql)
+        }
+
+        db.clearSchemaCache()
+      }
+    } finally {
+      await db.releaseAdvisoryLock(priorityLockName)
+    }
+
     await this._backfillExecutionModesOnce(db)
     await this._dropForkedColumnOnce(db)
 
@@ -3109,6 +3176,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
 
     if (markOrphaned) transitionedJob.orphanedAtMs = now
     if (shouldRetry) {
+      transitionedJob.priority = RETRY_BACKGROUND_JOB_PRIORITY
       transitionedJob.scheduledAtMs = scheduledAt
     } else if (!markOrphaned) {
       transitionedJob.failedAtMs = now
@@ -3142,7 +3210,14 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
     // A retry starts a fresh handoff with a possibly different runner, so the
     // previous child's acceptance evidence must not leak into the next attempt.
     // Terminal failures keep it as historical evidence for the lost attempt.
-    if (shouldRetry) Object.assign(update, this._clearedChildAcceptanceData())
+    if (shouldRetry) {
+      Object.assign(update, this._clearedChildAcceptanceData())
+
+      // Retried work goes to the back of the dispatch order. It has already had
+      // a turn, and letting it overtake first attempts is how a failing producer
+      // starves healthy jobs of the same queue.
+      update.priority = RETRY_BACKGROUND_JOB_PRIORITY
+    }
 
     this._applyOrphanedFailureUpdate({markOrphaned, now, update})
     this._applyFailureStatusUpdate({markOrphaned, now, scheduledAt, shouldRetry, update})
@@ -3211,6 +3286,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
       status: normalizeBackgroundJobStatus(row.status ? String(row.status) : "queued"),
       attempts: this._normalizeNumber(row.attempts),
       maxRetries: this._normalizeNumber(row.max_retries),
+      priority: this._normalizeNumber(row.priority) ?? DEFAULT_BACKGROUND_JOB_PRIORITY,
       scheduledAtMs: this._normalizeNumber(row.scheduled_at_ms),
       createdAtMs: this._normalizeNumber(row.created_at_ms),
       handedOffAtMs: this._normalizeNumber(row.handed_off_at_ms),

package/src/background-jobs/types.js

@@ -169,6 +169,7 @@
  * @typedef {object} BackgroundJobOptions
  * @property {BackgroundJobExecutionMode} [executionMode] - How the job should run. Node defaults to `"pooled"` (a warm, reused local runner process). Browser/Expo local dispatch defaults to and only accepts `"inline"`. `"forked"` runs a Node job in a fresh `child_process.fork()` child, and `"spawned"` in a detached CLI runner.
  * @property {number} [maxRetries] - Max retries for a failed job before it is marked failed.
+ * @property {number} [priority] - Dispatch priority within the queue: higher runs first, default `0`, retries always run last.
  * @property {string} [queue] - Queue name. Defaults to `"default"`. When the queue has a configured cap in `backgroundJobs.queues`, that cap is enforced cluster-wide.
  * @property {string} [concurrencyKey] - Opaque non-empty key used to share a concurrency cap. Overrides any queue-derived cap.
  * @property {number} [maxConcurrency] - Positive integer cap; must be paired with `concurrencyKey`.
@@ -207,6 +208,7 @@
  * @property {BackgroundJobStatus} status - Current job status.
  * @property {number | null} attempts - Failure attempts count.
  * @property {number | null} maxRetries - Max retry attempts.
+ * @property {number} priority - Dispatch priority; `-1` means retried and therefore queued last.
  * @property {number | null} scheduledAtMs - Next scheduled time in ms.
  * @property {number | null} createdAtMs - Creation time in ms.
  * @property {number | null} handedOffAtMs - Time handed to worker in ms.

package/src/database/drivers/base.js

@@ -157,6 +157,7 @@
  */
 
 import BacktraceCleaner from "../../utils/backtrace-cleaner.js"
+import BoundedTaskRunner from "../../utils/bounded-task-runner.js"
 import { getDatabaseAnnotations } from "../annotations.js"
 import { formatDateForDatabase } from "../datetime-storage.js"
 import isDate from "../../utils/is-date.js"
@@ -911,6 +912,65 @@ export default class VelociousDatabaseDriversBase {
     return await this._cachedSchemaMetadata(`table:${tableName}:${metadataName}`, callback)
   }
 
+  /**
+   * Seeds schema metadata that was loaded in one batch for many tables, so a later
+   * per-table read resolves from the cache instead of issuing its own query.
+   * @template T
+   * @param {string} tableName - Table name.
+   * @param {string} metadataName - Metadata name.
+   * @param {T} value - Already-resolved metadata.
+   * @returns {void}
+   */
+  seedTableSchemaMetadata(tableName, metadataName, value) {
+    if (!this._schemaCacheEnabled()) return
+
+    this._schemaCache.set(`table:${tableName}:${metadataName}`, Promise.resolve(value))
+  }
+
+  /**
+   * Loads column metadata for several tables, combining the reads into one query
+   * when the driver can answer for all of them at once and seeding the per-table
+   * cache so the models that follow initialize without querying.
+   * @param {Array<import("./base-table.js").default>} tables - Tables to load columns for.
+   * @param {{concurrency?: number}} [args] - Concurrency for drivers that can only read per table.
+   * @returns {Promise<void>}
+   */
+  async prefetchTableColumns(tables, {concurrency = 8} = {}) {
+    if (!this._schemaCacheEnabled()) return
+    if (tables.length === 0) return
+
+    const batchedColumns = await this.batchTableColumns(tables)
+
+    if (batchedColumns) {
+      for (const [tableName, columns] of batchedColumns) {
+        this.seedTableSchemaMetadata(tableName, "columns", columns)
+      }
+
+      return
+    }
+
+    await BoundedTaskRunner.run({
+      concurrency,
+      items: tables,
+      worker: async (table) => {
+        await table.getColumns()
+      }
+    })
+  }
+
+  /**
+   * Batched column metadata for many tables at once. Drivers whose engine cannot
+   * answer for several tables in one statement return null, leaving the caller to
+   * read them per table.
+   * @param {Array<import("./base-table.js").default>} tables - Tables to load columns for.
+   * @returns {Promise<Map<string, Array<import("./base-column.js").default>> | null>} - Columns keyed by table name, or null when unsupported.
+   */
+  async batchTableColumns(tables) {
+    void tables
+
+    return null
+  }
+
   /**
    * Runs schema cache return value.
    * @param {ReturnType<typeof JSON.parse>} value - Cached value.

package/src/database/drivers/mssql/index.js

@@ -2,6 +2,7 @@
 
 import AlterTable from "./sql/alter-table.js"
 import Base from "../base.js"
+import Column from "./column.js"
 import CreateDatabase from "./sql/create-database.js"
 import CreateIndex from "./sql/create-index.js"
 import CreateTable from "./sql/create-table.js"
@@ -516,6 +517,64 @@ export default class VelociousDatabaseDriversMssql extends Base{
     })
   }
 
+  /**
+   * Reads column metadata for many tables in one query. The per-table read of the
+   * same three-way `INFORMATION_SCHEMA` join is one of the slowest things a boot
+   * does against the legacy SQL Server, so it is worth combining regardless of the
+   * number of tables.
+   * @param {Array<import("../base-table.js").default>} tables - Tables to load columns for.
+   * @returns {Promise<Map<string, Array<import("../base-column.js").default>>>} - Columns keyed by table name.
+   */
+  async batchTableColumns(tables) {
+    /** @type {Map<string, Array<import("../base-column.js").default>>} */
+    const columnsByTable = new Map()
+
+    if (tables.length === 0) return columnsByTable
+
+    const schema = this.getArgs()?.schema || this.getArgs()?.sqlConfig?.options?.schema
+    const schemaClause = schema
+      ? ` AND columns.[TABLE_SCHEMA] = ${this.quote(schema)}`
+      : " AND columns.[TABLE_SCHEMA] = SCHEMA_NAME()"
+    const tablesByName = new Map(tables.map((table) => [table.getName(), table]))
+    const tableNamesSql = [...tablesByName.keys()].map((tableName) => this.quote(tableName)).join(", ")
+    const rows = await this.query(`
+      SELECT
+        columns.*,
+        COLUMNPROPERTY(object_id(columns.TABLE_SCHEMA + '.' + columns.TABLE_NAME), columns.COLUMN_NAME, 'IsIdentity') AS isIdentity,
+        CASE WHEN keyColumnUsage.COLUMN_NAME IS NOT NULL THEN 1 ELSE 0 END AS isPrimaryKey
+      FROM [INFORMATION_SCHEMA].[COLUMNS] AS columns
+      LEFT JOIN [INFORMATION_SCHEMA].[TABLE_CONSTRAINTS] AS tableConstraints ON
+        tableConstraints.CONSTRAINT_TYPE = 'PRIMARY KEY' AND
+        tableConstraints.TABLE_CATALOG = columns.TABLE_CATALOG AND
+        tableConstraints.TABLE_SCHEMA = columns.TABLE_SCHEMA AND
+        tableConstraints.TABLE_NAME = columns.TABLE_NAME
+      LEFT JOIN [INFORMATION_SCHEMA].[KEY_COLUMN_USAGE] AS keyColumnUsage ON
+        keyColumnUsage.CONSTRAINT_CATALOG = tableConstraints.CONSTRAINT_CATALOG AND
+        keyColumnUsage.CONSTRAINT_SCHEMA = tableConstraints.CONSTRAINT_SCHEMA AND
+        keyColumnUsage.CONSTRAINT_NAME = tableConstraints.CONSTRAINT_NAME AND
+        keyColumnUsage.TABLE_CATALOG = columns.TABLE_CATALOG AND
+        keyColumnUsage.TABLE_SCHEMA = columns.TABLE_SCHEMA AND
+        keyColumnUsage.TABLE_NAME = columns.TABLE_NAME AND
+        keyColumnUsage.COLUMN_NAME = columns.COLUMN_NAME
+      WHERE columns.TABLE_NAME IN (${tableNamesSql})${schemaClause}
+      ORDER BY columns.TABLE_NAME, columns.ORDINAL_POSITION
+    `)
+
+    for (const row of rows) {
+      const tableName = /** @type {string} */ (row.TABLE_NAME)
+      const table = tablesByName.get(tableName)
+
+      if (!table) continue
+
+      const columns = columnsByTable.get(tableName) || []
+
+      columns.push(new Column(table, /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)))
+      columnsByTable.set(tableName, columns)
+    }
+
+    return columnsByTable
+  }
+
   /**
    * Keeps constraint toggles and cleanup inside each physical SQL Server
    * request so a pool cannot split lifecycle ownership across sessions.

package/src/database/drivers/mysql/index.js

@@ -2,6 +2,7 @@
 
 import AlterTable from "./sql/alter-table.js"
 import Base from "../base.js"
+import Column from "./column.js"
 import CreateDatabase from "./sql/create-database.js"
 import CreateIndex from "./sql/create-index.js"
 import CreateTable from "./sql/create-table.js"
@@ -765,6 +766,56 @@ export default class VelociousDatabaseDriversMysql extends Base{
     })
   }
 
+  /**
+   * Reads column metadata for many tables in one query. `SHOW FULL COLUMNS` only
+   * describes a single table, but `information_schema.COLUMNS` carries the same
+   * fields, so the result is aliased to the shape `Column` already reads.
+   *
+   * `NULLIF(COLUMN_DEFAULT, 'NULL')` compensates for MariaDB reporting a SQL NULL
+   * default as the literal string 'NULL', where `SHOW FULL COLUMNS` reports NULL.
+   * Verified column-for-column against `SHOW FULL COLUMNS` on a 44-table MariaDB
+   * schema; the only divergence was that sentinel.
+   * @param {Array<import("../base-table.js").default>} tables - Tables to load columns for.
+   * @returns {Promise<Map<string, Array<import("../base-column.js").default>>>} - Columns keyed by table name.
+   */
+  async batchTableColumns(tables) {
+    /** @type {Map<string, Array<import("../base-column.js").default>>} */
+    const columnsByTable = new Map()
+
+    if (tables.length === 0) return columnsByTable
+
+    const tablesByName = new Map(tables.map((table) => [table.getName(), table]))
+    const tableNamesSql = [...tablesByName.keys()].map((tableName) => this.quote(tableName)).join(", ")
+    const rows = await this.query(`
+      SELECT
+        TABLE_NAME AS tableName,
+        COLUMN_NAME AS Field,
+        COLUMN_TYPE AS Type,
+        IS_NULLABLE AS \`Null\`,
+        COLUMN_KEY AS \`Key\`,
+        NULLIF(COLUMN_DEFAULT, 'NULL') AS \`Default\`,
+        EXTRA AS Extra,
+        COLUMN_COMMENT AS Comment
+      FROM information_schema.COLUMNS
+      WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (${tableNamesSql})
+      ORDER BY TABLE_NAME, ORDINAL_POSITION
+    `)
+
+    for (const row of rows) {
+      const tableName = /** @type {string} */ (row.tableName)
+      const table = tablesByName.get(tableName)
+
+      if (!table) continue
+
+      const columns = columnsByTable.get(tableName) || []
+
+      columns.push(new Column(table, /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)))
+      columnsByTable.set(tableName, columns)
+    }
+
+    return columnsByTable
+  }
+
   /**
    * Runs structure sql.
    * @returns {Promise<string | null>} - Resolves with SQL string.

package/src/database/drivers/pgsql/index.js

@@ -3,6 +3,7 @@
 import AlterTable from "./sql/alter-table.js"
 import wait from "awaitery/build/wait.js"
 import Base from "../base.js"
+import Column from "./column.js"
 import { Client, types as pgTypes } from "pg"
 import CreateDatabase from "./sql/create-database.js"
 import CreateIndex from "./sql/create-index.js"
@@ -329,6 +330,10 @@ export default class VelociousDatabaseDriversPgsql extends Base{
     return insert.toSql()
   }
 
+  /**
+   * Runs get tables.
+   * @returns {Promise<Array<import("../base-table.js").default>>} - Resolves with the tables.
+   */
   async getTables() {
     return await this._cachedSchemaMetadata("tables", async () => {
       const result = await this.query("SELECT * FROM information_schema.tables WHERE table_catalog = CURRENT_DATABASE() AND table_schema = 'public'")
@@ -344,6 +349,62 @@ export default class VelociousDatabaseDriversPgsql extends Base{
     })
   }
 
+  /**
+   * Reads column metadata for many tables in one query instead of one catalog
+   * join per table.
+   * @param {Array<import("../base-table.js").default>} tables - Tables to load columns for.
+   * @returns {Promise<Map<string, Array<import("../base-column.js").default>>>} - Columns keyed by table name.
+   */
+  async batchTableColumns(tables) {
+    /** @type {Map<string, Array<import("../base-column.js").default>>} */
+    const columnsByTable = new Map()
+
+    if (tables.length === 0) return columnsByTable
+
+    const tablesByName = new Map(tables.map((table) => [table.getName(), table]))
+    const tableNamesSql = [...tablesByName.keys()].map((tableName) => this.quote(tableName)).join(", ")
+    const rows = await this.query(`
+      SELECT
+        columns.*,
+        CASE WHEN key_column_usage.column_name IS NOT NULL THEN 1 ELSE 0 END AS is_primary_key,
+        col_description((columns.table_schema || '.' || columns.table_name)::regclass, columns.ordinal_position) AS column_comment
+
+      FROM
+        information_schema.columns AS columns
+
+      LEFT JOIN information_schema.table_constraints AS table_constraints ON
+        table_constraints.table_name = columns.table_name AND
+        table_constraints.table_schema = columns.table_schema AND
+        table_constraints.constraint_type = 'PRIMARY KEY'
+
+      LEFT JOIN information_schema.key_column_usage AS key_column_usage ON
+        key_column_usage.constraint_name = table_constraints.constraint_name AND
+        key_column_usage.table_schema = table_constraints.table_schema AND
+        key_column_usage.table_name = columns.table_name AND
+        key_column_usage.column_name = columns.column_name
+
+      WHERE
+        columns.table_catalog = CURRENT_DATABASE() AND
+        columns.table_schema = 'public' AND
+        columns.table_name IN (${tableNamesSql})
+      ORDER BY columns.table_name, columns.ordinal_position
+    `)
+
+    for (const row of rows) {
+      const tableName = /** @type {string} */ (row.table_name)
+      const table = tablesByName.get(tableName)
+
+      if (!table) continue
+
+      const columns = columnsByTable.get(tableName) || []
+
+      columns.push(new Column(table, /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)))
+      columnsByTable.set(tableName, columns)
+    }
+
+    return columnsByTable
+  }
+
   /**
    * Truncates all eligible tables in one PostgreSQL request.
    * @param {Array<import("../base-table.js").default>} tables - Eligible tables.

package/src/database/initializer-from-require-context.js

@@ -10,7 +10,15 @@
  */
 
 import Logger from "../logger.js"
+import BoundedTaskRunner from "../utils/bounded-task-runner.js"
 import restArgsError from "../utils/rest-args-error.js"
+import SchemaMetadataPrefetcher from "./schema-metadata-prefetcher.js"
+
+/**
+ * How many model classes initialize at once. Metadata comes from the prefetched
+ * schema cache, so this is CPU-bound attribute mapping rather than database work.
+ */
+const MODEL_INITIALIZATION_CONCURRENCY = 8
 
 export default class VelociousDatabaseInitializerFromRequireContext {
   /**
@@ -34,6 +42,9 @@ export default class VelociousDatabaseInitializerFromRequireContext {
   async initialize({configuration, ...restArgs}) {
     restArgsError(restArgs)
 
+    /** @type {Array<typeof import("./record/index.js").default>} */
+    const modelClasses = []
+
     for (const fileName of this.requireContext.keys()) {
       const modelClassImport = this.requireContext(fileName)
 
@@ -50,14 +61,27 @@ export default class VelociousDatabaseInitializerFromRequireContext {
         continue
       }
 
-      if (!modelClass.getEagerLoadRecordMetadata()) {
-        modelClass.registerRecordClass({configuration})
-        await this._bestEffortInitializeDeferredModel({configuration, modelClass})
-        continue
-      }
-
-      await this._initializeModelRecord({configuration, modelClass})
+      modelClass.registerRecordClass({configuration})
+      modelClasses.push(modelClass)
     }
+
+    // One catalog read per database instead of one per model, shared by every
+    // model initialized below through the driver's schema cache.
+    await new SchemaMetadataPrefetcher({configuration, logger: this.logger}).prefetch({modelClasses})
+
+    await BoundedTaskRunner.run({
+      concurrency: MODEL_INITIALIZATION_CONCURRENCY,
+      items: modelClasses,
+      worker: async (modelClass) => {
+        if (!modelClass.getEagerLoadRecordMetadata()) {
+          await this._bestEffortInitializeDeferredModel({configuration, modelClass})
+
+          return
+        }
+
+        await this._initializeModelRecord({configuration, modelClass})
+      }
+    })
   }
 
   /**