velocious 1.0.714 → 1.0.715

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/build/background-jobs/local-store.js +3 -0
  2. package/build/background-jobs/store.js +77 -1
  3. package/build/background-jobs/types.js +2 -0
  4. package/build/database/drivers/base.js +60 -0
  5. package/build/database/drivers/mssql/index.js +59 -0
  6. package/build/database/drivers/mysql/index.js +51 -0
  7. package/build/database/drivers/pgsql/index.js +61 -0
  8. package/build/database/initializer-from-require-context.js +31 -7
  9. package/build/database/schema-metadata-prefetcher.js +108 -0
  10. package/build/src/background-jobs/local-store.d.ts.map +1 -1
  11. package/build/src/background-jobs/local-store.js +4 -1
  12. package/build/src/background-jobs/store.d.ts +24 -0
  13. package/build/src/background-jobs/store.d.ts.map +1 -1
  14. package/build/src/background-jobs/store.js +66 -2
  15. package/build/src/background-jobs/types.d.ts +10 -0
  16. package/build/src/background-jobs/types.d.ts.map +1 -1
  17. package/build/src/background-jobs/types.js +3 -1
  18. package/build/src/database/drivers/base.d.ts +29 -0
  19. package/build/src/database/drivers/base.d.ts.map +1 -1
  20. package/build/src/database/drivers/base.js +55 -1
  21. package/build/src/database/drivers/mssql/index.d.ts +9 -0
  22. package/build/src/database/drivers/mssql/index.d.ts.map +1 -1
  23. package/build/src/database/drivers/mssql/index.js +54 -1
  24. package/build/src/database/drivers/mysql/index.d.ts +13 -0
  25. package/build/src/database/drivers/mysql/index.d.ts.map +1 -1
  26. package/build/src/database/drivers/mysql/index.js +46 -1
  27. package/build/src/database/drivers/pgsql/index.d.ts +12 -2
  28. package/build/src/database/drivers/pgsql/index.d.ts.map +1 -1
  29. package/build/src/database/drivers/pgsql/index.js +56 -1
  30. package/build/src/database/initializer-from-require-context.d.ts.map +1 -1
  31. package/build/src/database/initializer-from-require-context.js +26 -7
  32. package/build/src/database/schema-metadata-prefetcher.d.ts +53 -0
  33. package/build/src/database/schema-metadata-prefetcher.d.ts.map +1 -0
  34. package/build/src/database/schema-metadata-prefetcher.js +95 -0
  35. package/build/src/utils/bounded-task-runner.d.ts +26 -0
  36. package/build/src/utils/bounded-task-runner.d.ts.map +1 -0
  37. package/build/src/utils/bounded-task-runner.js +44 -0
  38. package/build/utils/bounded-task-runner.js +51 -0
  39. package/package.json +1 -1
  40. package/src/background-jobs/local-store.js +3 -0
  41. package/src/background-jobs/store.js +77 -1
  42. package/src/background-jobs/types.js +2 -0
  43. package/src/database/drivers/base.js +60 -0
  44. package/src/database/drivers/mssql/index.js +59 -0
  45. package/src/database/drivers/mysql/index.js +51 -0
  46. package/src/database/drivers/pgsql/index.js +61 -0
  47. package/src/database/initializer-from-require-context.js +31 -7
  48. package/src/database/schema-metadata-prefetcher.js +108 -0
  49. package/src/utils/bounded-task-runner.js +51 -0
@@ -1598,6 +1598,9 @@ export default class LocalBackgroundJobsStore {
1598
1598
  failedAtMs: this._numberOrNull(row.failed_at_ms),
1599
1599
  handedOffAtMs: this._numberOrNull(row.handed_off_at_ms),
1600
1600
  handoffId: row.handoff_id === null || row.handoff_id === undefined ? null : String(row.handoff_id),
1601
+ // Local (browser/Expo) dispatch is strictly FIFO and has no priority column,
1602
+ // so every row reports the default priority.
1603
+ priority: 0,
1601
1604
  id: String(row.id),
1602
1605
  jobName: String(row.job_name),
1603
1606
  lastError: row.last_error === null || row.last_error === undefined ? null : String(row.last_error),
@@ -41,6 +41,7 @@ import {
41
41
  * @property {string} jobId - New job id.
42
42
  * @property {string} jobName - Job name.
43
43
  * @property {number} maxRetries - Retry cap.
44
+ * @property {number} priority - Dispatch priority within the queue.
44
45
  * @property {string} queue - Queue name.
45
46
  * @property {number} scheduledAtMs - Eligibility timestamp.
46
47
  * @property {number | null} timeoutMs - Per-job timeout override, or null when omitted.
@@ -137,8 +138,22 @@ export const BACKGROUND_JOB_COUNT_BUCKETS = ["all", "queued", "handed_off", "com
137
138
  const COUNTED_JOB_STATUSES = BACKGROUND_JOB_COUNT_BUCKETS.slice(1)
138
139
  const MAX_JOB_TIMEOUT_MS = 2_147_483_647
139
140
  const JOB_TIMEOUT_VALIDATION_MESSAGE = `background job timeoutMs must be a finite non-positive number or an integer between 1 and ${MAX_JOB_TIMEOUT_MS}`
141
+ const BACKGROUND_JOB_PRIORITY_VALIDATION_MESSAGE = "background job priority must be a non-negative integer"
140
142
  const ORPHANED_AFTER_MS = 2 * 60 * 60 * 1000
141
143
 
144
+ /**
145
+ * Priority a job is demoted to when it is re-queued after a failed attempt.
146
+ * A retry has already consumed capacity and time, so it must never overtake
147
+ * work that has not had its first chance yet. Enqueue priorities are validated
148
+ * to be non-negative integers, which makes `-1` the lowest reachable priority.
149
+ */
150
+ export const RETRY_BACKGROUND_JOB_PRIORITY = -1
151
+
152
+ /**
153
+ * Priority a job gets when the producer does not ask for one.
154
+ */
155
+ export const DEFAULT_BACKGROUND_JOB_PRIORITY = 0
156
+
142
157
  /**
143
158
  * Columns the dashboard is allowed to sort job listings by, mapped to their
144
159
  * database column names. Restricting to this set keeps the sort parameter
@@ -1159,6 +1174,12 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1159
1174
  const priorityOrder = this._queuePriorityOrderSql(db)
1160
1175
 
1161
1176
  if (priorityOrder) query = query.order(`${priorityOrder} DESC`)
1177
+
1178
+ // Job-level priority applies inside the queue ranking above. A retry is
1179
+ // stored at RETRY_BACKGROUND_JOB_PRIORITY (-1), so it sorts behind every
1180
+ // first attempt without delaying itself further than the backoff already
1181
+ // does. COALESCE covers rows written before the column existed.
1182
+ query = query.order(`COALESCE(${db.quoteColumn("priority")}, ${DEFAULT_BACKGROUND_JOB_PRIORITY}) DESC`)
1162
1183
  }
1163
1184
 
1164
1185
  query = query
@@ -1965,12 +1986,33 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1965
1986
  jobId: randomUUID(),
1966
1987
  jobName,
1967
1988
  maxRetries: this._normalizeMaxRetries(options?.maxRetries),
1989
+ priority: this._normalizeJobPriority(options),
1968
1990
  queue,
1969
1991
  scheduledAtMs: this._normalizeScheduledAtMs(options?.scheduledAtMs, createdAtMs),
1970
1992
  timeoutMs: this._normalizeJobTimeoutMs(options)
1971
1993
  }
1972
1994
  }
1973
1995
 
1996
+ /**
1997
+ * Normalizes a requested dispatch priority. Priorities order jobs within their
1998
+ * queue on the dispatch path only: higher goes first, and the default keeps
1999
+ * FIFO order. Negative priorities are rejected because the store reserves them
2000
+ * for {@link RETRY_BACKGROUND_JOB_PRIORITY}, which must stay the lowest.
2001
+ * @param {import("./types.js").BackgroundJobOptions | undefined} options - Job options.
2002
+ * @returns {number} - Priority.
2003
+ */
2004
+ _normalizeJobPriority(options) {
2005
+ const priority = options?.priority
2006
+
2007
+ if (priority === undefined) return DEFAULT_BACKGROUND_JOB_PRIORITY
2008
+
2009
+ if (typeof priority !== "number" || !Number.isInteger(priority) || priority < 0) {
2010
+ throw VelociousError.safe(BACKGROUND_JOB_PRIORITY_VALIDATION_MESSAGE)
2011
+ }
2012
+
2013
+ return priority
2014
+ }
2015
+
1974
2016
  /**
1975
2017
  * Normalizes a per-job timeout while preserving omitted (worker fallback)
1976
2018
  * separately from explicitly disabled.
@@ -2024,6 +2066,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
2024
2066
  execution_mode: preparedJob.executionMode,
2025
2067
  queue: preparedJob.queue,
2026
2068
  max_retries: preparedJob.maxRetries,
2069
+ priority: preparedJob.priority,
2027
2070
  attempts: 0,
2028
2071
  status: "queued",
2029
2072
  scheduled_at_ms: preparedJob.scheduledAtMs,
@@ -2334,6 +2377,30 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
2334
2377
  }
2335
2378
  }
2336
2379
 
2380
+ const priorityLockName = `${MIGRATION_SCOPE}:priority_column`
2381
+ const priorityLockAcquired = await db.acquireAdvisoryLock(priorityLockName)
2382
+
2383
+ if (!priorityLockAcquired) throw new Error("Failed to acquire background jobs priority schema lock")
2384
+
2385
+ try {
2386
+ db.clearSchemaCache()
2387
+ const priorityTable = await db.getTableByNameOrFail(JOBS_TABLE)
2388
+
2389
+ if (!(await priorityTable.getColumnByName("priority"))) {
2390
+ const tableData = new TableData(JOBS_TABLE)
2391
+ tableData.bigint("priority", {null: true})
2392
+ const sqls = await db.alterTableSQLs(tableData)
2393
+
2394
+ for (const sql of sqls) {
2395
+ await db.query(sql)
2396
+ }
2397
+
2398
+ db.clearSchemaCache()
2399
+ }
2400
+ } finally {
2401
+ await db.releaseAdvisoryLock(priorityLockName)
2402
+ }
2403
+
2337
2404
  await this._backfillExecutionModesOnce(db)
2338
2405
  await this._dropForkedColumnOnce(db)
2339
2406
 
@@ -3109,6 +3176,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
3109
3176
 
3110
3177
  if (markOrphaned) transitionedJob.orphanedAtMs = now
3111
3178
  if (shouldRetry) {
3179
+ transitionedJob.priority = RETRY_BACKGROUND_JOB_PRIORITY
3112
3180
  transitionedJob.scheduledAtMs = scheduledAt
3113
3181
  } else if (!markOrphaned) {
3114
3182
  transitionedJob.failedAtMs = now
@@ -3142,7 +3210,14 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
3142
3210
  // A retry starts a fresh handoff with a possibly different runner, so the
3143
3211
  // previous child's acceptance evidence must not leak into the next attempt.
3144
3212
  // Terminal failures keep it as historical evidence for the lost attempt.
3145
- if (shouldRetry) Object.assign(update, this._clearedChildAcceptanceData())
3213
+ if (shouldRetry) {
3214
+ Object.assign(update, this._clearedChildAcceptanceData())
3215
+
3216
+ // Retried work goes to the back of the dispatch order. It has already had
3217
+ // a turn, and letting it overtake first attempts is how a failing producer
3218
+ // starves healthy jobs of the same queue.
3219
+ update.priority = RETRY_BACKGROUND_JOB_PRIORITY
3220
+ }
3146
3221
 
3147
3222
  this._applyOrphanedFailureUpdate({markOrphaned, now, update})
3148
3223
  this._applyFailureStatusUpdate({markOrphaned, now, scheduledAt, shouldRetry, update})
@@ -3211,6 +3286,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
3211
3286
  status: normalizeBackgroundJobStatus(row.status ? String(row.status) : "queued"),
3212
3287
  attempts: this._normalizeNumber(row.attempts),
3213
3288
  maxRetries: this._normalizeNumber(row.max_retries),
3289
+ priority: this._normalizeNumber(row.priority) ?? DEFAULT_BACKGROUND_JOB_PRIORITY,
3214
3290
  scheduledAtMs: this._normalizeNumber(row.scheduled_at_ms),
3215
3291
  createdAtMs: this._normalizeNumber(row.created_at_ms),
3216
3292
  handedOffAtMs: this._normalizeNumber(row.handed_off_at_ms),
@@ -169,6 +169,7 @@
169
169
  * @typedef {object} BackgroundJobOptions
170
170
  * @property {BackgroundJobExecutionMode} [executionMode] - How the job should run. Node defaults to `"pooled"` (a warm, reused local runner process). Browser/Expo local dispatch defaults to and only accepts `"inline"`. `"forked"` runs a Node job in a fresh `child_process.fork()` child, and `"spawned"` in a detached CLI runner.
171
171
  * @property {number} [maxRetries] - Max retries for a failed job before it is marked failed.
172
+ * @property {number} [priority] - Dispatch priority within the queue: higher runs first, default `0`, retries always run last.
172
173
  * @property {string} [queue] - Queue name. Defaults to `"default"`. When the queue has a configured cap in `backgroundJobs.queues`, that cap is enforced cluster-wide.
173
174
  * @property {string} [concurrencyKey] - Opaque non-empty key used to share a concurrency cap. Overrides any queue-derived cap.
174
175
  * @property {number} [maxConcurrency] - Positive integer cap; must be paired with `concurrencyKey`.
@@ -207,6 +208,7 @@
207
208
  * @property {BackgroundJobStatus} status - Current job status.
208
209
  * @property {number | null} attempts - Failure attempts count.
209
210
  * @property {number | null} maxRetries - Max retry attempts.
211
+ * @property {number} priority - Dispatch priority; `-1` means retried and therefore queued last.
210
212
  * @property {number | null} scheduledAtMs - Next scheduled time in ms.
211
213
  * @property {number | null} createdAtMs - Creation time in ms.
212
214
  * @property {number | null} handedOffAtMs - Time handed to worker in ms.
@@ -157,6 +157,7 @@
157
157
  */
158
158
 
159
159
  import BacktraceCleaner from "../../utils/backtrace-cleaner.js"
160
+ import BoundedTaskRunner from "../../utils/bounded-task-runner.js"
160
161
  import { getDatabaseAnnotations } from "../annotations.js"
161
162
  import { formatDateForDatabase } from "../datetime-storage.js"
162
163
  import isDate from "../../utils/is-date.js"
@@ -911,6 +912,65 @@ export default class VelociousDatabaseDriversBase {
911
912
  return await this._cachedSchemaMetadata(`table:${tableName}:${metadataName}`, callback)
912
913
  }
913
914
 
915
+ /**
916
+ * Seeds schema metadata that was loaded in one batch for many tables, so a later
917
+ * per-table read resolves from the cache instead of issuing its own query.
918
+ * @template T
919
+ * @param {string} tableName - Table name.
920
+ * @param {string} metadataName - Metadata name.
921
+ * @param {T} value - Already-resolved metadata.
922
+ * @returns {void}
923
+ */
924
+ seedTableSchemaMetadata(tableName, metadataName, value) {
925
+ if (!this._schemaCacheEnabled()) return
926
+
927
+ this._schemaCache.set(`table:${tableName}:${metadataName}`, Promise.resolve(value))
928
+ }
929
+
930
+ /**
931
+ * Loads column metadata for several tables, combining the reads into one query
932
+ * when the driver can answer for all of them at once and seeding the per-table
933
+ * cache so the models that follow initialize without querying.
934
+ * @param {Array<import("./base-table.js").default>} tables - Tables to load columns for.
935
+ * @param {{concurrency?: number}} [args] - Concurrency for drivers that can only read per table.
936
+ * @returns {Promise<void>}
937
+ */
938
+ async prefetchTableColumns(tables, {concurrency = 8} = {}) {
939
+ if (!this._schemaCacheEnabled()) return
940
+ if (tables.length === 0) return
941
+
942
+ const batchedColumns = await this.batchTableColumns(tables)
943
+
944
+ if (batchedColumns) {
945
+ for (const [tableName, columns] of batchedColumns) {
946
+ this.seedTableSchemaMetadata(tableName, "columns", columns)
947
+ }
948
+
949
+ return
950
+ }
951
+
952
+ await BoundedTaskRunner.run({
953
+ concurrency,
954
+ items: tables,
955
+ worker: async (table) => {
956
+ await table.getColumns()
957
+ }
958
+ })
959
+ }
960
+
961
+ /**
962
+ * Batched column metadata for many tables at once. Drivers whose engine cannot
963
+ * answer for several tables in one statement return null, leaving the caller to
964
+ * read them per table.
965
+ * @param {Array<import("./base-table.js").default>} tables - Tables to load columns for.
966
+ * @returns {Promise<Map<string, Array<import("./base-column.js").default>> | null>} - Columns keyed by table name, or null when unsupported.
967
+ */
968
+ async batchTableColumns(tables) {
969
+ void tables
970
+
971
+ return null
972
+ }
973
+
914
974
  /**
915
975
  * Runs schema cache return value.
916
976
  * @param {ReturnType<typeof JSON.parse>} value - Cached value.
@@ -2,6 +2,7 @@
2
2
 
3
3
  import AlterTable from "./sql/alter-table.js"
4
4
  import Base from "../base.js"
5
+ import Column from "./column.js"
5
6
  import CreateDatabase from "./sql/create-database.js"
6
7
  import CreateIndex from "./sql/create-index.js"
7
8
  import CreateTable from "./sql/create-table.js"
@@ -516,6 +517,64 @@ export default class VelociousDatabaseDriversMssql extends Base{
516
517
  })
517
518
  }
518
519
 
520
+ /**
521
+ * Reads column metadata for many tables in one query. The per-table read of the
522
+ * same three-way `INFORMATION_SCHEMA` join is one of the slowest things a boot
523
+ * does against the legacy SQL Server, so it is worth combining regardless of the
524
+ * number of tables.
525
+ * @param {Array<import("../base-table.js").default>} tables - Tables to load columns for.
526
+ * @returns {Promise<Map<string, Array<import("../base-column.js").default>>>} - Columns keyed by table name.
527
+ */
528
+ async batchTableColumns(tables) {
529
+ /** @type {Map<string, Array<import("../base-column.js").default>>} */
530
+ const columnsByTable = new Map()
531
+
532
+ if (tables.length === 0) return columnsByTable
533
+
534
+ const schema = this.getArgs()?.schema || this.getArgs()?.sqlConfig?.options?.schema
535
+ const schemaClause = schema
536
+ ? ` AND columns.[TABLE_SCHEMA] = ${this.quote(schema)}`
537
+ : " AND columns.[TABLE_SCHEMA] = SCHEMA_NAME()"
538
+ const tablesByName = new Map(tables.map((table) => [table.getName(), table]))
539
+ const tableNamesSql = [...tablesByName.keys()].map((tableName) => this.quote(tableName)).join(", ")
540
+ const rows = await this.query(`
541
+ SELECT
542
+ columns.*,
543
+ COLUMNPROPERTY(object_id(columns.TABLE_SCHEMA + '.' + columns.TABLE_NAME), columns.COLUMN_NAME, 'IsIdentity') AS isIdentity,
544
+ CASE WHEN keyColumnUsage.COLUMN_NAME IS NOT NULL THEN 1 ELSE 0 END AS isPrimaryKey
545
+ FROM [INFORMATION_SCHEMA].[COLUMNS] AS columns
546
+ LEFT JOIN [INFORMATION_SCHEMA].[TABLE_CONSTRAINTS] AS tableConstraints ON
547
+ tableConstraints.CONSTRAINT_TYPE = 'PRIMARY KEY' AND
548
+ tableConstraints.TABLE_CATALOG = columns.TABLE_CATALOG AND
549
+ tableConstraints.TABLE_SCHEMA = columns.TABLE_SCHEMA AND
550
+ tableConstraints.TABLE_NAME = columns.TABLE_NAME
551
+ LEFT JOIN [INFORMATION_SCHEMA].[KEY_COLUMN_USAGE] AS keyColumnUsage ON
552
+ keyColumnUsage.CONSTRAINT_CATALOG = tableConstraints.CONSTRAINT_CATALOG AND
553
+ keyColumnUsage.CONSTRAINT_SCHEMA = tableConstraints.CONSTRAINT_SCHEMA AND
554
+ keyColumnUsage.CONSTRAINT_NAME = tableConstraints.CONSTRAINT_NAME AND
555
+ keyColumnUsage.TABLE_CATALOG = columns.TABLE_CATALOG AND
556
+ keyColumnUsage.TABLE_SCHEMA = columns.TABLE_SCHEMA AND
557
+ keyColumnUsage.TABLE_NAME = columns.TABLE_NAME AND
558
+ keyColumnUsage.COLUMN_NAME = columns.COLUMN_NAME
559
+ WHERE columns.TABLE_NAME IN (${tableNamesSql})${schemaClause}
560
+ ORDER BY columns.TABLE_NAME, columns.ORDINAL_POSITION
561
+ `)
562
+
563
+ for (const row of rows) {
564
+ const tableName = /** @type {string} */ (row.TABLE_NAME)
565
+ const table = tablesByName.get(tableName)
566
+
567
+ if (!table) continue
568
+
569
+ const columns = columnsByTable.get(tableName) || []
570
+
571
+ columns.push(new Column(table, /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)))
572
+ columnsByTable.set(tableName, columns)
573
+ }
574
+
575
+ return columnsByTable
576
+ }
577
+
519
578
  /**
520
579
  * Keeps constraint toggles and cleanup inside each physical SQL Server
521
580
  * request so a pool cannot split lifecycle ownership across sessions.
@@ -2,6 +2,7 @@
2
2
 
3
3
  import AlterTable from "./sql/alter-table.js"
4
4
  import Base from "../base.js"
5
+ import Column from "./column.js"
5
6
  import CreateDatabase from "./sql/create-database.js"
6
7
  import CreateIndex from "./sql/create-index.js"
7
8
  import CreateTable from "./sql/create-table.js"
@@ -765,6 +766,56 @@ export default class VelociousDatabaseDriversMysql extends Base{
765
766
  })
766
767
  }
767
768
 
769
+ /**
770
+ * Reads column metadata for many tables in one query. `SHOW FULL COLUMNS` only
771
+ * describes a single table, but `information_schema.COLUMNS` carries the same
772
+ * fields, so the result is aliased to the shape `Column` already reads.
773
+ *
774
+ * `NULLIF(COLUMN_DEFAULT, 'NULL')` compensates for MariaDB reporting a SQL NULL
775
+ * default as the literal string 'NULL', where `SHOW FULL COLUMNS` reports NULL.
776
+ * Verified column-for-column against `SHOW FULL COLUMNS` on a 44-table MariaDB
777
+ * schema; the only divergence was that sentinel.
778
+ * @param {Array<import("../base-table.js").default>} tables - Tables to load columns for.
779
+ * @returns {Promise<Map<string, Array<import("../base-column.js").default>>>} - Columns keyed by table name.
780
+ */
781
+ async batchTableColumns(tables) {
782
+ /** @type {Map<string, Array<import("../base-column.js").default>>} */
783
+ const columnsByTable = new Map()
784
+
785
+ if (tables.length === 0) return columnsByTable
786
+
787
+ const tablesByName = new Map(tables.map((table) => [table.getName(), table]))
788
+ const tableNamesSql = [...tablesByName.keys()].map((tableName) => this.quote(tableName)).join(", ")
789
+ const rows = await this.query(`
790
+ SELECT
791
+ TABLE_NAME AS tableName,
792
+ COLUMN_NAME AS Field,
793
+ COLUMN_TYPE AS Type,
794
+ IS_NULLABLE AS \`Null\`,
795
+ COLUMN_KEY AS \`Key\`,
796
+ NULLIF(COLUMN_DEFAULT, 'NULL') AS \`Default\`,
797
+ EXTRA AS Extra,
798
+ COLUMN_COMMENT AS Comment
799
+ FROM information_schema.COLUMNS
800
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (${tableNamesSql})
801
+ ORDER BY TABLE_NAME, ORDINAL_POSITION
802
+ `)
803
+
804
+ for (const row of rows) {
805
+ const tableName = /** @type {string} */ (row.tableName)
806
+ const table = tablesByName.get(tableName)
807
+
808
+ if (!table) continue
809
+
810
+ const columns = columnsByTable.get(tableName) || []
811
+
812
+ columns.push(new Column(table, /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)))
813
+ columnsByTable.set(tableName, columns)
814
+ }
815
+
816
+ return columnsByTable
817
+ }
818
+
768
819
  /**
769
820
  * Runs structure sql.
770
821
  * @returns {Promise<string | null>} - Resolves with SQL string.
@@ -3,6 +3,7 @@
3
3
  import AlterTable from "./sql/alter-table.js"
4
4
  import wait from "awaitery/build/wait.js"
5
5
  import Base from "../base.js"
6
+ import Column from "./column.js"
6
7
  import { Client, types as pgTypes } from "pg"
7
8
  import CreateDatabase from "./sql/create-database.js"
8
9
  import CreateIndex from "./sql/create-index.js"
@@ -329,6 +330,10 @@ export default class VelociousDatabaseDriversPgsql extends Base{
329
330
  return insert.toSql()
330
331
  }
331
332
 
333
+ /**
334
+ * Runs get tables.
335
+ * @returns {Promise<Array<import("../base-table.js").default>>} - Resolves with the tables.
336
+ */
332
337
  async getTables() {
333
338
  return await this._cachedSchemaMetadata("tables", async () => {
334
339
  const result = await this.query("SELECT * FROM information_schema.tables WHERE table_catalog = CURRENT_DATABASE() AND table_schema = 'public'")
@@ -344,6 +349,62 @@ export default class VelociousDatabaseDriversPgsql extends Base{
344
349
  })
345
350
  }
346
351
 
352
+ /**
353
+ * Reads column metadata for many tables in one query instead of one catalog
354
+ * join per table.
355
+ * @param {Array<import("../base-table.js").default>} tables - Tables to load columns for.
356
+ * @returns {Promise<Map<string, Array<import("../base-column.js").default>>>} - Columns keyed by table name.
357
+ */
358
+ async batchTableColumns(tables) {
359
+ /** @type {Map<string, Array<import("../base-column.js").default>>} */
360
+ const columnsByTable = new Map()
361
+
362
+ if (tables.length === 0) return columnsByTable
363
+
364
+ const tablesByName = new Map(tables.map((table) => [table.getName(), table]))
365
+ const tableNamesSql = [...tablesByName.keys()].map((tableName) => this.quote(tableName)).join(", ")
366
+ const rows = await this.query(`
367
+ SELECT
368
+ columns.*,
369
+ CASE WHEN key_column_usage.column_name IS NOT NULL THEN 1 ELSE 0 END AS is_primary_key,
370
+ col_description((columns.table_schema || '.' || columns.table_name)::regclass, columns.ordinal_position) AS column_comment
371
+
372
+ FROM
373
+ information_schema.columns AS columns
374
+
375
+ LEFT JOIN information_schema.table_constraints AS table_constraints ON
376
+ table_constraints.table_name = columns.table_name AND
377
+ table_constraints.table_schema = columns.table_schema AND
378
+ table_constraints.constraint_type = 'PRIMARY KEY'
379
+
380
+ LEFT JOIN information_schema.key_column_usage AS key_column_usage ON
381
+ key_column_usage.constraint_name = table_constraints.constraint_name AND
382
+ key_column_usage.table_schema = table_constraints.table_schema AND
383
+ key_column_usage.table_name = columns.table_name AND
384
+ key_column_usage.column_name = columns.column_name
385
+
386
+ WHERE
387
+ columns.table_catalog = CURRENT_DATABASE() AND
388
+ columns.table_schema = 'public' AND
389
+ columns.table_name IN (${tableNamesSql})
390
+ ORDER BY columns.table_name, columns.ordinal_position
391
+ `)
392
+
393
+ for (const row of rows) {
394
+ const tableName = /** @type {string} */ (row.table_name)
395
+ const table = tablesByName.get(tableName)
396
+
397
+ if (!table) continue
398
+
399
+ const columns = columnsByTable.get(tableName) || []
400
+
401
+ columns.push(new Column(table, /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)))
402
+ columnsByTable.set(tableName, columns)
403
+ }
404
+
405
+ return columnsByTable
406
+ }
407
+
347
408
  /**
348
409
  * Truncates all eligible tables in one PostgreSQL request.
349
410
  * @param {Array<import("../base-table.js").default>} tables - Eligible tables.
@@ -10,7 +10,15 @@
10
10
  */
11
11
 
12
12
  import Logger from "../logger.js"
13
+ import BoundedTaskRunner from "../utils/bounded-task-runner.js"
13
14
  import restArgsError from "../utils/rest-args-error.js"
15
+ import SchemaMetadataPrefetcher from "./schema-metadata-prefetcher.js"
16
+
17
+ /**
18
+ * How many model classes initialize at once. Metadata comes from the prefetched
19
+ * schema cache, so this is CPU-bound attribute mapping rather than database work.
20
+ */
21
+ const MODEL_INITIALIZATION_CONCURRENCY = 8
14
22
 
15
23
  export default class VelociousDatabaseInitializerFromRequireContext {
16
24
  /**
@@ -34,6 +42,9 @@ export default class VelociousDatabaseInitializerFromRequireContext {
34
42
  async initialize({configuration, ...restArgs}) {
35
43
  restArgsError(restArgs)
36
44
 
45
+ /** @type {Array<typeof import("./record/index.js").default>} */
46
+ const modelClasses = []
47
+
37
48
  for (const fileName of this.requireContext.keys()) {
38
49
  const modelClassImport = this.requireContext(fileName)
39
50
 
@@ -50,14 +61,27 @@ export default class VelociousDatabaseInitializerFromRequireContext {
50
61
  continue
51
62
  }
52
63
 
53
- if (!modelClass.getEagerLoadRecordMetadata()) {
54
- modelClass.registerRecordClass({configuration})
55
- await this._bestEffortInitializeDeferredModel({configuration, modelClass})
56
- continue
57
- }
58
-
59
- await this._initializeModelRecord({configuration, modelClass})
64
+ modelClass.registerRecordClass({configuration})
65
+ modelClasses.push(modelClass)
60
66
  }
67
+
68
+ // One catalog read per database instead of one per model, shared by every
69
+ // model initialized below through the driver's schema cache.
70
+ await new SchemaMetadataPrefetcher({configuration, logger: this.logger}).prefetch({modelClasses})
71
+
72
+ await BoundedTaskRunner.run({
73
+ concurrency: MODEL_INITIALIZATION_CONCURRENCY,
74
+ items: modelClasses,
75
+ worker: async (modelClass) => {
76
+ if (!modelClass.getEagerLoadRecordMetadata()) {
77
+ await this._bestEffortInitializeDeferredModel({configuration, modelClass})
78
+
79
+ return
80
+ }
81
+
82
+ await this._initializeModelRecord({configuration, modelClass})
83
+ }
84
+ })
61
85
  }
62
86
 
63
87
  /**
@@ -0,0 +1,108 @@
1
+ // @ts-check
2
+
3
+ import Logger from "../logger.js"
4
+
5
+ /**
6
+ * Loads table and column metadata for a whole set of model classes up front.
7
+ *
8
+ * Model initialization otherwise reads each table's columns on its own, which
9
+ * turns a boot into one catalog query per model (and per HTTP worker process).
10
+ * Drivers that can answer for many tables in one statement are asked to do so;
11
+ * the resulting columns are seeded into the driver's per-table schema cache, so
12
+ * the initialization pass that follows resolves without querying at all.
13
+ */
14
+ export default class SchemaMetadataPrefetcher {
15
+ /**
16
+ * Builds a prefetcher for one configuration.
17
+ * @param {object} args - Prefetcher args.
18
+ * @param {import("../configuration.js").default} args.configuration - Configuration owning the database connections.
19
+ * @param {number} [args.concurrency] - Concurrency for drivers that can only read one table per query.
20
+ * @param {import("../logger.js").default} [args.logger] - Logger for debug output.
21
+ */
22
+ constructor({configuration, concurrency = 8, logger}) {
23
+ this.configuration = configuration
24
+ this.concurrency = concurrency
25
+ this.logger = logger || new Logger(this)
26
+ }
27
+
28
+ /**
29
+ * Prefetches column metadata for every model class that has a reachable table.
30
+ *
31
+ * Models whose table or connection is unavailable are skipped: their
32
+ * initialization stays deferred and will initialize lazily on first use, which
33
+ * is the same contract the per-model initialization applies.
34
+ * @param {{modelClasses: Array<typeof import("./record/index.js").default>}} args - Model classes to prefetch for.
35
+ * @returns {Promise<void>}
36
+ */
37
+ async prefetch({modelClasses}) {
38
+ /** @type {Map<import("./drivers/base.js").default, Map<string, import("./drivers/base-table.js").default>>} */
39
+ const tablesByDriver = new Map()
40
+
41
+ for (const modelClass of modelClasses) {
42
+ const table = await this.tableFor(modelClass)
43
+
44
+ if (!table) continue
45
+
46
+ const driver = table.getDriver()
47
+ let tables = tablesByDriver.get(driver)
48
+
49
+ if (!tables) {
50
+ tables = new Map()
51
+ tablesByDriver.set(driver, tables)
52
+ }
53
+
54
+ tables.set(table.getName(), table)
55
+
56
+ // Translated models initialize their translation class as part of the same
57
+ // startup pass, so its table has to be part of the batch as well - otherwise
58
+ // every translated model still costs its own catalog read.
59
+ const translationTable = await this.tableByName(driver, modelClass.getTranslationsTableName())
60
+
61
+ if (translationTable) tables.set(translationTable.getName(), translationTable)
62
+ }
63
+
64
+ const startedAt = Date.now()
65
+ let tableCount = 0
66
+
67
+ for (const [driver, tables] of tablesByDriver) {
68
+ await driver.prefetchTableColumns([...tables.values()], {concurrency: this.concurrency})
69
+ tableCount += tables.size
70
+ }
71
+
72
+ void this.logger.debug(() => ["schema metadata prefetched", {
73
+ databases: tablesByDriver.size,
74
+ durationMs: Date.now() - startedAt,
75
+ tables: tableCount
76
+ }])
77
+ }
78
+
79
+ /**
80
+ * Resolves a model's table without initializing its record metadata.
81
+ * @param {typeof import("./record/index.js").default} modelClass - Model class to resolve a table for.
82
+ * @returns {Promise<import("./drivers/base-table.js").default | null>} Table, or null when it is not reachable.
83
+ */
84
+ async tableFor(modelClass) {
85
+ try {
86
+ const driver = modelClass.connection({enforceTenantDatabaseScope: false})
87
+
88
+ return await this.tableByName(driver, modelClass.tableName())
89
+ } catch (error) {
90
+ // A model whose database is unreachable - or whose table is optional and
91
+ // absent - stays deferred rather than failing startup; it initializes
92
+ // lazily on first use.
93
+ void this.logger.debug(() => [`Leaving ${modelClass.name} out of schema prefetch`, error])
94
+
95
+ return null
96
+ }
97
+ }
98
+
99
+ /**
100
+ * Looks a table up in the driver's table list without throwing when absent.
101
+ * @param {import("./drivers/base.js").default} driver - Driver owning the table.
102
+ * @param {string} tableName - Table name to resolve.
103
+ * @returns {Promise<import("./drivers/base-table.js").default | null>} Table, or null when it does not exist.
104
+ */
105
+ async tableByName(driver, tableName) {
106
+ return await driver.getTableByName(tableName, {throwError: false}) || null
107
+ }
108
+ }