uql-orm 0.93.0 → 0.94.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/dist/browser/uql-browser.min.js +1 -1
  2. package/dist/browser/uql-browser.min.js.map +3 -3
  3. package/dist/dialect/abstractDialect.d.ts +2 -0
  4. package/dist/dialect/abstractDialect.js +4 -0
  5. package/dist/dialect/abstractSqlDialect.d.ts +11 -6
  6. package/dist/dialect/abstractSqlDialect.js +37 -13
  7. package/dist/dialect/mysqlLikeSqlDialect.d.ts +1 -0
  8. package/dist/dialect/mysqlLikeSqlDialect.js +4 -1
  9. package/dist/dialect/pgLikeSqlDialect.d.ts +2 -0
  10. package/dist/dialect/pgLikeSqlDialect.js +4 -0
  11. package/dist/dialect/queryContext.d.ts +2 -0
  12. package/dist/dialect/queryContext.js +6 -0
  13. package/dist/maria/mariadbQuerier.d.ts +2 -1
  14. package/dist/maria/mariadbQuerier.js +2 -8
  15. package/dist/migrate/ddl/index.d.ts +1 -1
  16. package/dist/migrate/ddl/index.js +3 -3
  17. package/dist/migrate/ddl/mssqlTableDdl.d.ts +3 -1
  18. package/dist/migrate/ddl/mssqlTableDdl.js +4 -0
  19. package/dist/migrate/ddl/mysqlTableDdl.d.ts +1 -1
  20. package/dist/migrate/ddl/mysqlTableDdl.js +2 -2
  21. package/dist/migrate/ddl/pgTableDdl.d.ts +5 -1
  22. package/dist/migrate/ddl/pgTableDdl.js +12 -6
  23. package/dist/migrate/ddl/tableDdl.d.ts +9 -6
  24. package/dist/migrate/ddl/tableDdl.js +15 -6
  25. package/dist/migrate/ddl/tableRebuild.d.ts +2 -5
  26. package/dist/migrate/drift/driftDetector.js +7 -2
  27. package/dist/migrate/schemaChange.d.ts +2 -5
  28. package/dist/migrate/schemaGenerator.js +1 -1
  29. package/dist/mongo/mongoDialect.d.ts +1 -2
  30. package/dist/mongo/mongoDialect.js +18 -15
  31. package/dist/mongo/mongodbQuerier.js +3 -10
  32. package/dist/mssql/mssqlDialect.d.ts +3 -2
  33. package/dist/mssql/mssqlDialect.js +3 -2
  34. package/dist/mssql/mssqlQuerier.d.ts +2 -2
  35. package/dist/mssql/mssqlQuerier.js +1 -3
  36. package/dist/mysql/mysql2Querier.d.ts +1 -1
  37. package/dist/mysql/mysql2Querier.js +1 -9
  38. package/dist/pglite/index.d.ts +1 -0
  39. package/dist/pglite/index.js +1 -0
  40. package/dist/pglite/pgliteDialect.d.ts +6 -0
  41. package/dist/pglite/pgliteDialect.js +6 -0
  42. package/dist/pglite/pgliteQuerierPool.d.ts +2 -2
  43. package/dist/pglite/pgliteQuerierPool.js +2 -2
  44. package/dist/postgres/pgQuerier.d.ts +2 -4
  45. package/dist/postgres/pgQuerier.js +1 -9
  46. package/dist/querier/abstractQuerier.d.ts +21 -3
  47. package/dist/querier/abstractQuerier.js +86 -17
  48. package/dist/querier/abstractQuerierPool.d.ts +1 -1
  49. package/dist/querier/abstractQuerierPool.js +23 -23
  50. package/dist/querier/abstractSqlQuerier.d.ts +15 -10
  51. package/dist/querier/abstractSqlQuerier.js +54 -57
  52. package/dist/querier/cursorStream.d.ts +1 -1
  53. package/dist/schema/sqlExpression.d.ts +4 -1
  54. package/dist/schema/sqlExpression.js +20 -2
  55. package/dist/schema/types.d.ts +2 -5
  56. package/dist/sqlite/abstractSqliteQuerier.d.ts +1 -1
  57. package/dist/sqlite/sqliteDialect.d.ts +1 -0
  58. package/dist/sqlite/sqliteDialect.js +1 -0
  59. package/dist/type/entity.d.ts +9 -9
  60. package/dist/type/migration.d.ts +7 -8
  61. package/dist/type/querier.d.ts +1 -1
  62. package/dist/type/universalQuerier.d.ts +1 -1
  63. package/dist/util/dialect.util.d.ts +1 -1
  64. package/dist/util/object.util.d.ts +2 -0
  65. package/dist/util/object.util.js +8 -0
  66. package/dist/util/query.util.d.ts +3 -1
  67. package/dist/util/query.util.js +7 -0
  68. package/dist/util/sql.util.d.ts +2 -4
  69. package/dist/util/sql.util.js +10 -19
  70. package/package.json +10 -10
  71. package/skills/uql-orm/SKILL.md +22 -51
@@ -1,7 +1,7 @@
1
1
  import { assertSoleId, getMeta, idOf, namesKey, relationOf } from '../entity/index.js';
2
2
  import { namesRows } from '../dialect/operators.js';
3
3
  import { parseQueryLock } from '../type/index.js';
4
- import { cascadesOnDelete, childrenOf, clone, entityName, fillOnFields, filterFieldKeys, filterPersistableRelationKeys, forEachRequestedRelation, getKeys, hasKeys, getRelationRequestSummary, guardWrite, idOnlyQuery, keySet, isPagedQuery, isScalarId, LoggerWrapper, parentJoins, queryLoggerFor, parseRelationAtKey, parseRelationQueryValue, rowKey, runHooks, securityConditions, someKey, targetKeyColumns, whereAnyOf, whereEach, whereIds, whereWith, withoutSoftDeleteFilter, } from '../util/index.js';
4
+ import { cascadesOnDelete, childrenOf, chunk, clone, entityName, fillOnFields, filterFieldKeys, filterPersistableRelationKeys, forEachRequestedRelation, getKeys, hasKeys, getRelationRequestSummary, guardWrite, idOnlyQuery, keySet, isPagedQuery, isScalarId, LoggerWrapper, parentJoins, queryLoggerFor, parseRelationAtKey, parseRelationQueryValue, rowKey, runHooks, securityConditions, someKey, targetKeyColumns, whereEach, whereIds, whereKeysIn, whereWith, withoutSoftDeleteFilter, } from '../util/index.js';
5
5
  import { UqlOptimisticLockError, UqlUsageError } from '../util/uqlError.js';
6
6
  import { keysetRead } from './keyset.js';
7
7
  import { enrichError } from './queryError.js';
@@ -126,6 +126,7 @@ function isEntityFirst(args) {
126
126
  const [first] = args;
127
127
  return typeof first === 'function' && first.prototype !== undefined;
128
128
  }
129
+ const STREAM_HOLDS_QUERIER = 'a stream is reading on this querier: run the statement after its loop, or on another querier';
129
130
  /** Base class for all database queriers. */
130
131
  export class AbstractQuerier {
131
132
  extra;
@@ -135,6 +136,8 @@ export class AbstractQuerier {
135
136
  * and ensuring that the database connection is used safely across concurrent calls.
136
137
  */
137
138
  taskQueue = Promise.resolve();
139
+ /** The stream reading on the connection, from its first row asked for until its loop ends. */
140
+ #stream;
138
141
  /**
139
142
  * A querier is one unit of work, so releasing ends it. Checked where each backend reaches for its
140
143
  * connection, on every driver and not just the pooled ones.
@@ -205,7 +208,37 @@ export class AbstractQuerier {
205
208
  findManyStream(...args) {
206
209
  const [entity, q, opts] = entityArgs(args);
207
210
  this.validateReadQuery(entity, q);
208
- return this.internalFindManyStream(entity, q, opts);
211
+ const rows = this.internalFindManyStream(entity, q, opts);
212
+ return this.holdWhileReading(this.listensForLoad(entity, q.$populate) ? this.loadedEach(entity, rows, q.$populate) : rows);
213
+ }
214
+ /** `afterLoad` on each row before the loop sees it, as `findMany` runs it on each row it read. */
215
+ async *loadedEach(entity, rows, populate) {
216
+ for await (const row of rows) {
217
+ await this.emitLoaded(entity, [row], populate);
218
+ yield row;
219
+ }
220
+ }
221
+ /**
222
+ * `rows`, holding the querier from the first row asked for until the loop ends. Most drivers queue a
223
+ * statement behind an open read, which a loop awaiting it never finishes, so `serialize` refuses one
224
+ * meanwhile, on every engine alike, and `release` closes the stream first.
225
+ */
226
+ holdWhileReading(rows) {
227
+ const read = async function* (querier) {
228
+ if (querier.#stream) {
229
+ throw new UqlUsageError(STREAM_HOLDS_QUERIER);
230
+ }
231
+ querier.#stream = stream;
232
+ try {
233
+ await querier.taskQueue;
234
+ yield* rows;
235
+ }
236
+ finally {
237
+ querier.#stream = undefined;
238
+ }
239
+ };
240
+ const stream = read(this);
241
+ return stream;
209
242
  }
210
243
  async findManyAndCount(...args) {
211
244
  const [entity, q, opts] = entityArgs(args);
@@ -312,7 +345,10 @@ export class AbstractQuerier {
312
345
  if (!ids.length) {
313
346
  return 0;
314
347
  }
315
- const changes = await this.updateColumns(entity, { $where: whereIds(meta, ids) }, row, opts, ids.length);
348
+ let changes = 0;
349
+ for (const batch of chunk(ids, this.dialect.keyListCapacity(meta.ids.length))) {
350
+ changes += await this.updateColumns(entity, { $where: whereIds(meta, batch) }, row, opts, batch.length);
351
+ }
316
352
  for (const relKey of relKeys) {
317
353
  await this.saveRelation(entity, relKey, ids.map((id) => ({ id, value: row[relKey] })), true);
318
354
  }
@@ -453,11 +489,14 @@ export class AbstractQuerier {
453
489
  return 0;
454
490
  }
455
491
  await this.emitHook(entity, 'beforeDelete', doomed);
456
- // Children first: they hold the foreign key, which a schema without `ON DELETE CASCADE` enforces.
457
- if (cascades) {
458
- await this.deleteRelations(entity, ids, opts);
492
+ let changes = 0;
493
+ for (const batch of chunk(ids, this.dialect.keyListCapacity(meta.ids.length))) {
494
+ // Children first: they hold the foreign key, which a schema without `ON DELETE CASCADE` enforces.
495
+ if (cascades) {
496
+ await this.deleteRelations(entity, batch, opts);
497
+ }
498
+ changes += await this.internalDeleteMany(entity, { $where: whereIds(meta, batch) }, opts);
459
499
  }
460
- const changes = await this.internalDeleteMany(entity, { $where: whereIds(meta, ids) }, opts);
461
500
  await this.emitHook(entity, 'afterDelete', doomed);
462
501
  return changes;
463
502
  }
@@ -554,8 +593,9 @@ export class AbstractQuerier {
554
593
  const holder = relOpts.through ? relOpts.through() : relEntity;
555
594
  const parentColumn = soleParentColumn(relOpts);
556
595
  if (isUpdate) {
557
- const ids = writes.map(({ id }) => id);
558
- await this.deleteMany(holder, { $where: { [parentColumn]: ids } });
596
+ for (const batch of chunk(writes, this.dialect.keyListCapacity(1))) {
597
+ await this.deleteMany(holder, { $where: { [parentColumn]: batch.map(({ id }) => id) } });
598
+ }
559
599
  }
560
600
  // Each parent gets its own copies, so a row listed for two parents is written twice.
561
601
  const children = writes.flatMap(({ id, value }) => [value ?? []].flat().map((row) => ({ id, row })));
@@ -650,13 +690,14 @@ export class AbstractQuerier {
650
690
  const meta = getMeta(entity);
651
691
  const [idKey] = meta.ids;
652
692
  const keys = getKeys(conflictPaths);
653
- const q = {
654
- $select: keySet([idKey, ...keys]),
655
- $where: whereAnyOf(rows.map((row) => whereEach(keys, (key) => row[key]))),
656
- };
657
- const found = await this.internalFindMany(entity, q, { filters: withoutSoftDeleteFilter(undefined) });
693
+ const distinct = [...new Map(rows.map((row) => [rowKey(row, keys), row])).values()];
694
+ const found = [];
695
+ for (const batch of chunk(distinct, this.dialect.keyListCapacity(keys.length))) {
696
+ const q = { $select: keySet([idKey, ...keys]), $where: whereKeysIn(keys, batch) };
697
+ found.push(await this.internalFindMany(entity, q, { filters: withoutSoftDeleteFilter(undefined) }));
698
+ }
658
699
  const byConflict = new Map();
659
- for (const row of found) {
700
+ for (const row of found.flat()) {
660
701
  const key = rowKey(row, keys);
661
702
  byConflict.set(key, byConflict.has(key) ? undefined : row[idKey]);
662
703
  }
@@ -694,6 +735,9 @@ export class AbstractQuerier {
694
735
  }
695
736
  /** Runs `task` after everything already queued, one at a time. Not re-entrant: never nest `serialize` calls. */
696
737
  serialize(task) {
738
+ if (this.#stream) {
739
+ return Promise.reject(new UqlUsageError(STREAM_HOLDS_QUERIER));
740
+ }
697
741
  const res = this.taskQueue.then(task);
698
742
  this.taskQueue = res.catch(() => { });
699
743
  return res;
@@ -711,6 +755,26 @@ export class AbstractQuerier {
711
755
  this.logger.logQuery(query, values, Math.round(performance.now() - startTime));
712
756
  }
713
757
  }
758
+ /**
759
+ * `rows` as {@link timed} runs a statement: logged once they end, timed to the first row since the
760
+ * time after it is the loop's, and a failure tagged with `query`.
761
+ */
762
+ async *timedStream(query, values, rows) {
763
+ const startTime = performance.now();
764
+ let answeredAt;
765
+ try {
766
+ for await (const row of rows) {
767
+ answeredAt ??= performance.now();
768
+ yield row;
769
+ }
770
+ }
771
+ catch (err) {
772
+ throw enrichError(err, this.logger, query, values);
773
+ }
774
+ finally {
775
+ this.logger.logQuery(query, values, Math.round((answeredAt ?? performance.now()) - startTime));
776
+ }
777
+ }
714
778
  beginTransaction(opts) {
715
779
  return this.serialize(async () => {
716
780
  if (this.hasOpenTransaction) {
@@ -740,11 +804,16 @@ export class AbstractQuerier {
740
804
  });
741
805
  }
742
806
  /**
743
- * Rolls back an unfinished transaction, then hands the connection back, discarding it if the rollback
744
- * failed. Never throws first, since `await using` has no other way to release.
807
+ * Closes a stream left open and rolls back an unfinished transaction, then hands the connection back,
808
+ * discarding it if either failed: the pool would otherwise take one still reading. Never throws first,
809
+ * since `await using` has no other way to release.
745
810
  */
746
811
  async release() {
747
812
  let discard = false;
813
+ await this.#stream?.return(undefined).catch((err) => {
814
+ this.logger.logError('closing an open stream failed; discarding the connection', err);
815
+ discard = true;
816
+ });
748
817
  if (this.hasOpenTransaction) {
749
818
  this.logger.logWarn('rolling back a transaction left open at release');
750
819
  // The rollback doubles as a health check. One that succeeds proves the connection round-trips and
@@ -29,7 +29,7 @@ export declare abstract class AbstractQuerierPool<Q extends Querier, D extends A
29
29
  readonly findMany: UniversalQuerier['findMany'];
30
30
  /**
31
31
  * The connection outlives the call: it is held until the iterator is drained or closed by a `break` or
32
- * `throw`. Abandoning the iterator leaks it until GC, so consume it in a `for await`.
32
+ * `throw`. Abandoning the iterator leaks it, so consume it in a `for await`.
33
33
  */
34
34
  readonly findManyStream: UniversalQuerier['findManyStream'];
35
35
  private streamWithQuerier;
@@ -86,14 +86,14 @@ export class AbstractQuerierPool {
86
86
  runScoped(context, fn) {
87
87
  return context ? withContext(context, fn) : fn();
88
88
  }
89
- findOneById = (entity, id, q, opts) => this.withQuerier((querier) => querier.findOneById(entity, id, q, opts));
90
- findOne = (entity, q, opts) => this.withQuerier((querier) => querier.findOne(entity, q, opts));
91
- findMany = (entity, q, opts) => this.withQuerier((querier) => querier.findMany(entity, q, opts));
89
+ findOneById = (...args) => this.withQuerier((querier) => querier.findOneById(...args));
90
+ findOne = (...args) => this.withQuerier((querier) => querier.findOne(...args));
91
+ findMany = (...args) => this.withQuerier((querier) => querier.findMany(...args));
92
92
  /**
93
93
  * The connection outlives the call: it is held until the iterator is drained or closed by a `break` or
94
- * `throw`. Abandoning the iterator leaks it until GC, so consume it in a `for await`.
94
+ * `throw`. Abandoning the iterator leaks it, so consume it in a `for await`.
95
95
  */
96
- findManyStream = (entity, q, opts) => this.streamWithQuerier((querier) => querier.findManyStream(entity, q, opts));
96
+ findManyStream = (...args) => this.streamWithQuerier((querier) => querier.findManyStream(...args));
97
97
  async *streamWithQuerier(read) {
98
98
  const env_1 = { stack: [], error: void 0, hasError: false };
99
99
  try {
@@ -110,22 +110,22 @@ export class AbstractQuerierPool {
110
110
  await result_1;
111
111
  }
112
112
  }
113
- findManyAndCount = (entity, q, opts) => this.withQuerier((querier) => querier.findManyAndCount(entity, q, opts));
114
- findManyPage = (entity, q, opts) => this.withQuerier((querier) => querier.findManyPage(entity, q, opts));
115
- count = (entity, q, opts) => this.withQuerier((querier) => querier.count(entity, q, opts));
116
- exists = (entity, q, opts) => this.withQuerier((querier) => querier.exists(entity, q, opts));
117
- aggregate = (entity, q, opts) => this.withQuerier((querier) => querier.aggregate(entity, q, opts));
118
- estimatedCount = (entity) => this.withQuerier((querier) => querier.estimatedCount(entity));
119
- insertOne = (entity, payload) => this.withQuerier((querier) => querier.insertOne(entity, payload));
120
- insertMany = (entity, payload) => this.withQuerier((querier) => querier.insertMany(entity, payload));
121
- updateOneById = (entity, id, payload, opts) => this.withQuerier((querier) => querier.updateOneById(entity, id, payload, opts));
122
- updateMany = (entity, q, payload, opts) => this.withQuerier((querier) => querier.updateMany(entity, q, payload, opts));
123
- upsertOne = (entity, conflictPaths, payload) => this.withQuerier((querier) => querier.upsertOne(entity, conflictPaths, payload));
124
- upsertMany = (entity, conflictPaths, payload) => this.withQuerier((querier) => querier.upsertMany(entity, conflictPaths, payload));
125
- saveOne = (entity, payload) => this.withQuerier((querier) => querier.saveOne(entity, payload));
126
- saveMany = (entity, payload) => this.withQuerier((querier) => querier.saveMany(entity, payload));
127
- deleteOneById = (entity, id, opts) => this.withQuerier((querier) => querier.deleteOneById(entity, id, opts));
128
- deleteMany = (entity, q, opts) => this.withQuerier((querier) => querier.deleteMany(entity, q, opts));
129
- restoreOneById = (entity, id) => this.withQuerier((querier) => querier.restoreOneById(entity, id));
130
- restoreMany = (entity, q) => this.withQuerier((querier) => querier.restoreMany(entity, q));
113
+ findManyAndCount = (...args) => this.withQuerier((querier) => querier.findManyAndCount(...args));
114
+ findManyPage = (...args) => this.withQuerier((querier) => querier.findManyPage(...args));
115
+ count = (...args) => this.withQuerier((querier) => querier.count(...args));
116
+ exists = (...args) => this.withQuerier((querier) => querier.exists(...args));
117
+ aggregate = (...args) => this.withQuerier((querier) => querier.aggregate(...args));
118
+ estimatedCount = (...args) => this.withQuerier((querier) => querier.estimatedCount(...args));
119
+ insertOne = (...args) => this.withQuerier((querier) => querier.insertOne(...args));
120
+ insertMany = (...args) => this.withQuerier((querier) => querier.insertMany(...args));
121
+ updateOneById = (...args) => this.withQuerier((querier) => querier.updateOneById(...args));
122
+ updateMany = (...args) => this.withQuerier((querier) => querier.updateMany(...args));
123
+ upsertOne = (...args) => this.withQuerier((querier) => querier.upsertOne(...args));
124
+ upsertMany = (...args) => this.withQuerier((querier) => querier.upsertMany(...args));
125
+ saveOne = (...args) => this.withQuerier((querier) => querier.saveOne(...args));
126
+ saveMany = (...args) => this.withQuerier((querier) => querier.saveMany(...args));
127
+ deleteOneById = (...args) => this.withQuerier((querier) => querier.deleteOneById(...args));
128
+ deleteMany = (...args) => this.withQuerier((querier) => querier.deleteMany(...args));
129
+ restoreOneById = (...args) => this.withQuerier((querier) => querier.restoreOneById(...args));
130
+ restoreMany = (...args) => this.withQuerier((querier) => querier.restoreMany(...args));
131
131
  }
@@ -1,5 +1,5 @@
1
1
  import type { AbstractSqlDialect } from '../dialect/index.js';
2
- import type { EntityData, ExtraOptions, Query, QueryAggMap, QueryAggregate, QueryAggregateResult, QueryConflictPaths, QueryPage, QueryGroupMap, QueryOptions, QuerySearch, QueryUpdateResult, SqlQuerier, TransactionOptions, Type, UpdatePayload, UpsertOptions } from '../type/index.js';
2
+ import type { EntityData, ExtraOptions, Query, QueryAggMap, QueryAggregate, QueryAggregateResult, QueryConflictPaths, QueryPage, QueryGroupMap, QueryOptions, QuerySearch, QueryUpdateResult, RawRow, SqlQuerier, TransactionOptions, Type, UpdatePayload, UpsertOptions } from '../type/index.js';
3
3
  import type { BuildUpdateResultPayload } from '../util/sql.util.js';
4
4
  import { AbstractQuerier } from './abstractQuerier.js';
5
5
  export declare abstract class AbstractSqlQuerier extends AbstractQuerier implements SqlQuerier {
@@ -34,6 +34,8 @@ export declare abstract class AbstractSqlQuerier extends AbstractQuerier impleme
34
34
  protected lazyConnect(): Promise<void>;
35
35
  all<T>(query: string, values?: readonly unknown[]): Promise<T[]>;
36
36
  run(query: string, values?: readonly unknown[]): Promise<QueryUpdateResult>;
37
+ /** Refused before the driver fails it, or PGlite answers it and every read after it wrong. */
38
+ private assertBindBudget;
37
39
  /** The rows of a statement the dialect builds. */
38
40
  private query;
39
41
  /** Runs a statement the dialect builds. */
@@ -53,22 +55,25 @@ export declare abstract class AbstractSqlQuerier extends AbstractQuerier impleme
53
55
  */
54
56
  protected internalFindManyAndCount<E extends object>(entity: Type<E>, q: Query<E>, opts?: QueryOptions): Promise<[E[], number]>;
55
57
  private selectRows;
56
- private hydrateRows;
58
+ /**
59
+ * The one read not going through `all`, so it checks its budget and connects on its own; its values go as
60
+ * the context holds them, each normalized as it was bound. The tuning is guarded as `selectRows` says.
61
+ */
57
62
  protected internalFindManyStream<E extends object>(entity: Type<E>, q: Query<E>, opts?: QueryOptions): AsyncGenerator<Awaited<E>, void, unknown>;
58
63
  /**
59
64
  * A read's rows one at a time: paged through a server-side cursor where the engine has one, read whole
60
- * where it has none. A driver that streams on its own overrides this.
65
+ * where it has none. A driver that streams on its own overrides this; a Node `Readable` it hands back
66
+ * closes itself when the loop exits early.
61
67
  */
62
- protected internalStream<T>(query: string, values?: unknown[]): AsyncIterable<T>;
68
+ protected internalStream(query: string, values?: unknown[]): AsyncIterable<RawRow>;
63
69
  /**
64
- * Turn what a driver returned back into the types the entity declares, for every row and everything
65
- * populated under them. Which columns, and as what, is `hydratableFields`, resolved once for all the
66
- * rows; the per-cell decode is `decodeColumn`. Both live with the dialect, because a `sparsevec` is
67
- * only sparse on Postgres.
70
+ * How each row of one statement becomes the entity's, for a read, a stream and a to-many alike: its dotted
71
+ * columns nested as the first row names them, and its values decoded as `hydratableFields` says, resolved
72
+ * once. Both live with the dialect, because a `sparsevec` is only sparse on Postgres.
68
73
  */
69
- private hydrateAll;
74
+ private rowReader;
70
75
  /**
71
- * One row of {@link hydrateAll}. A related row arrives as its parent's statement read it: a to-one
76
+ * One row of {@link rowReader}. A related row arrives as its parent's statement read it: a to-one
72
77
  * joined and unflattened, there only when its key is, since an unmatched join still fills a computed
73
78
  * column or a to-many's empty array; a to-many as a JSON array, which a driver may hand over as text.
74
79
  * Each is an object of its own, so the walk reaches none twice.
@@ -2,7 +2,7 @@ import { AGGREGATE_VALUE_ALIAS, TOTAL_ALIAS } from '../dialect/aliases.js';
2
2
  import { decodeColumn } from '../dialect/hydrateColumn.js';
3
3
  import { getMeta, namesKey } from '../entity/index.js';
4
4
  import { COUNT_RESULT_KEY } from '../type/index.js';
5
- import { buildUpdateResult, clone, getInsertFieldKeys, insertShapeOf, isAutoIncrement, isRecord, obtainAttrsPaths, unflatObject, unflatObjects, } from '../util/index.js';
5
+ import { buildUpdateResult, chunk, clone, getInsertFieldKeys, insertShapeOf, isAutoIncrement, isRecord, obtainAttrsPaths, unflatObject, } from '../util/index.js';
6
6
  import { UqlUsageError } from '../util/uqlError.js';
7
7
  import { AbstractQuerier } from './abstractQuerier.js';
8
8
  import { streamViaCursor } from './cursorStream.js';
@@ -40,18 +40,13 @@ function groupByInsertShape(meta, payload) {
40
40
  return [...groups.values()];
41
41
  }
42
42
  /**
43
- * A group's row indexes split into statements within the dialect's bind budget, payload order kept.
44
- * `DEFAULT` cells bind no parameter, so fields-per-record is a safe upper bound. Every multi-row
45
- * write splits on this: D1 allows 100 binds, which a couple of dozen rows reach.
43
+ * A group's row indexes split into statements of at most `maxRows` rows within the bind budget, payload order
44
+ * kept, a row with no column to write in one of its own. `DEFAULT` cells bind no parameter, so fields-per-record
45
+ * is a safe upper bound. Every multi-row write splits on this: D1 allows 100 binds, which a couple of dozen reach.
46
46
  */
47
- function chunkByBindBudget(meta, payload, group, maxBindValues) {
47
+ function chunkWithinLimits(meta, payload, group, maxBindValues, maxRows = Infinity) {
48
48
  const fieldsPerRecord = getInsertFieldKeys(meta, group.map((index) => payload[index])).length;
49
- const size = Math.max(1, Math.floor(maxBindValues / (fieldsPerRecord || 1)));
50
- const chunks = [];
51
- for (let start = 0; start < group.length; start += size) {
52
- chunks.push(group.slice(start, start + size));
53
- }
54
- return chunks;
49
+ return chunk(group, fieldsPerRecord ? Math.max(1, Math.min(maxRows, Math.floor(maxBindValues / fieldsPerRecord))) : 1);
55
50
  }
56
51
  export class AbstractSqlQuerier extends AbstractQuerier {
57
52
  dialect;
@@ -95,17 +90,26 @@ export class AbstractSqlQuerier extends AbstractQuerier {
95
90
  }
96
91
  }
97
92
  async all(query, values) {
93
+ this.assertBindBudget(values);
98
94
  return this.serialize(async () => {
99
95
  await this.lazyConnect();
100
96
  return this.timed(query, values, () => this.internalAll(query, this.dialect.normalizeValues(values)));
101
97
  });
102
98
  }
103
99
  async run(query, values) {
100
+ this.assertBindBudget(values);
104
101
  return this.serialize(async () => {
105
102
  await this.lazyConnect();
106
103
  return this.timed(query, values, () => this.internalRun(query, this.dialect.normalizeValues(values)));
107
104
  });
108
105
  }
106
+ /** Refused before the driver fails it, or PGlite answers it and every read after it wrong. */
107
+ assertBindBudget(values) {
108
+ const { maxBindValues, dialectName } = this.dialect;
109
+ if (values && values.length > maxBindValues) {
110
+ throw new UqlUsageError(`a statement binding ${values.length} values is past the ${maxBindValues} ${dialectName} takes; split the list it binds`);
111
+ }
112
+ }
109
113
  /** The rows of a statement the dialect builds. */
110
114
  query(build) {
111
115
  const ctx = this.dialect.createContext();
@@ -138,7 +142,7 @@ export class AbstractSqlQuerier extends AbstractQuerier {
138
142
  }
139
143
  }
140
144
  async internalFindMany(entity, q, opts) {
141
- return this.hydrateRows(entity, await this.selectRows(entity, q, opts));
145
+ return (await this.selectRows(entity, q, opts)).map(this.rowReader(entity));
142
146
  }
143
147
  /** Every row `q` matches past its page, deduplicated where it reads `$distinct`. */
144
148
  countUnpaged(entity, q, opts) {
@@ -161,7 +165,7 @@ export class AbstractSqlQuerier extends AbstractQuerier {
161
165
  for (const row of rows) {
162
166
  delete row[TOTAL_ALIAS];
163
167
  }
164
- return [this.hydrateRows(entity, rows), total];
168
+ return [rows.map(this.rowReader(entity)), total];
165
169
  }
166
170
  async selectRows(entity, q, opts, totalAlias) {
167
171
  // Guarded rather than awaited unconditionally, here and in the stream below: an `await` on this
@@ -172,40 +176,27 @@ export class AbstractSqlQuerier extends AbstractQuerier {
172
176
  }
173
177
  return this.query((ctx) => this.dialect.find(ctx, entity, q, opts, totalAlias));
174
178
  }
175
- hydrateRows(entity, rows) {
176
- const founds = unflatObjects(rows);
177
- this.hydrateAll(entity, founds);
178
- return founds;
179
- }
179
+ /**
180
+ * The one read not going through `all`, so it checks its budget and connects on its own; its values go as
181
+ * the context holds them, each normalized as it was bound. The tuning is guarded as `selectRows` says.
182
+ */
180
183
  async *internalFindManyStream(entity, q, opts) {
181
- // Guarded for the reason `selectRows` above spells out.
184
+ const ctx = this.dialect.createContext();
185
+ this.dialect.find(ctx, entity, q, opts);
186
+ this.assertBindBudget(ctx.values);
182
187
  if (q.$candidates !== undefined) {
183
188
  await this.applyVectorTuning(entity, q);
184
189
  }
185
- const meta = getMeta(entity);
186
- // The one path not going through `all`/`run`, so it connects on its own.
187
190
  await this.lazyConnect();
188
- // No `normalizeValues` here, unlike `all`/`run`: those also take raw SQL, while every value a
189
- // context holds was normalized as it was bound.
190
- const ctx = this.dialect.createContext();
191
- this.dialect.find(ctx, entity, q, opts);
192
- const fields = this.dialect.hydratableFields(entity);
193
- let attrsPaths;
194
- try {
195
- for await (const row of this.internalStream(ctx.sql, ctx.values)) {
196
- attrsPaths ??= obtainAttrsPaths(row);
197
- const found = unflatObject(row, attrsPaths);
198
- this.hydrateFields(meta, fields, found);
199
- yield found;
200
- }
201
- }
202
- catch (err) {
203
- throw enrichError(err, this.logger, ctx.sql, ctx.values);
191
+ const read = this.rowReader(entity);
192
+ for await (const row of this.timedStream(ctx.sql, ctx.values, this.internalStream(ctx.sql, ctx.values))) {
193
+ yield read(row);
204
194
  }
205
195
  }
206
196
  /**
207
197
  * A read's rows one at a time: paged through a server-side cursor where the engine has one, read whole
208
- * where it has none. A driver that streams on its own overrides this.
198
+ * where it has none. A driver that streams on its own overrides this; a Node `Readable` it hands back
199
+ * closes itself when the loop exits early.
209
200
  */
210
201
  async *internalStream(query, values) {
211
202
  if (!this.dialect.features.serverSideCursors) {
@@ -215,20 +206,23 @@ export class AbstractSqlQuerier extends AbstractQuerier {
215
206
  yield* streamViaCursor((sql, params) => this.internalAll(sql, params), query, values, this.hasOpenTransaction);
216
207
  }
217
208
  /**
218
- * Turn what a driver returned back into the types the entity declares, for every row and everything
219
- * populated under them. Which columns, and as what, is `hydratableFields`, resolved once for all the
220
- * rows; the per-cell decode is `decodeColumn`. Both live with the dialect, because a `sparsevec` is
221
- * only sparse on Postgres.
209
+ * How each row of one statement becomes the entity's, for a read, a stream and a to-many alike: its dotted
210
+ * columns nested as the first row names them, and its values decoded as `hydratableFields` says, resolved
211
+ * once. Both live with the dialect, because a `sparsevec` is only sparse on Postgres.
222
212
  */
223
- hydrateAll(entity, dtos) {
213
+ rowReader(entity) {
224
214
  const meta = getMeta(entity);
225
215
  const fields = this.dialect.hydratableFields(entity);
226
- for (const dto of dtos) {
227
- this.hydrateFields(meta, fields, dto);
228
- }
216
+ let attrsPaths;
217
+ return (row) => {
218
+ attrsPaths ??= obtainAttrsPaths(row);
219
+ const found = unflatObject(row, attrsPaths);
220
+ this.hydrateFields(meta, fields, found);
221
+ return found;
222
+ };
229
223
  }
230
224
  /**
231
- * One row of {@link hydrateAll}. A related row arrives as its parent's statement read it: a to-one
225
+ * One row of {@link rowReader}. A related row arrives as its parent's statement read it: a to-one
232
226
  * joined and unflattened, there only when its key is, since an unmatched join still fills a computed
233
227
  * column or a to-many's empty array; a to-many as a JSON array, which a driver may hand over as text.
234
228
  * Each is an object of its own, so the walk reaches none twice.
@@ -261,9 +255,8 @@ export class AbstractSqlQuerier extends AbstractQuerier {
261
255
  const relEntity = rel.entity();
262
256
  if (typeof value === 'string' || Array.isArray(value)) {
263
257
  // A to-many's rows, as flat as a statement's own.
264
- const rows = unflatObjects(typeof value === 'string' ? JSON.parse(value) : value);
265
- row[key] = rows;
266
- this.hydrateAll(relEntity, rows);
258
+ const rows = typeof value === 'string' ? JSON.parse(value) : value;
259
+ row[key] = rows.map(this.rowReader(relEntity));
267
260
  }
268
261
  else if (isRecord(value)) {
269
262
  const relMeta = getMeta(relEntity);
@@ -325,9 +318,9 @@ export class AbstractSqlQuerier extends AbstractQuerier {
325
318
  }
326
319
  // Per group, not per batch: the two carry different columns - one names the key, one does not -
327
320
  // so a budget taken over their union would under-fill the statement that is missing one.
328
- for (const indexes of chunkByBindBudget(meta, rows, group, this.dialect.maxBindValues)) {
329
- const chunk = indexes.map((index) => rows[index]);
330
- const { ids = [] } = await this.exec((ctx) => this.dialect.insert(ctx, entity, chunk));
321
+ for (const indexes of chunkWithinLimits(meta, rows, group, this.dialect.maxBindValues, this.dialect.maxInsertRows)) {
322
+ const statementRows = indexes.map((index) => rows[index]);
323
+ const { ids = [] } = await this.exec((ctx) => this.dialect.insert(ctx, entity, statementRows));
331
324
  if (idsReliable) {
332
325
  for (let position = 0; position < indexes.length; position++) {
333
326
  rows[indexes[position]][idKey] ??= ids[position];
@@ -350,10 +343,14 @@ export class AbstractSqlQuerier extends AbstractQuerier {
350
343
  }
351
344
  payload = clone(payload);
352
345
  const meta = getMeta(entity);
353
- // One statement per shape, each split again to stay inside the bind budget. Grouping first is
354
- // what makes the budget arithmetic right: every row of a group carries the same columns, so the
355
- // `DO UPDATE SET` resolves to non-binding `EXCLUDED` references rather than inlined values.
356
- const statements = groupByInsertShape(meta, payload).flatMap((group) => chunkByBindBudget(meta, payload, group, this.dialect.maxBindValues));
346
+ // One statement per shape, each split again to stay inside the bind budget, less what its assignments bind
347
+ // once. Grouping first is what makes that one figure: every row of a group carries the same columns, so
348
+ // they assign the same. No row cap: SQL Server upserts through a MERGE, whose source takes any number.
349
+ const statements = groupByInsertShape(meta, payload).flatMap((group) => {
350
+ const rows = group.map((index) => payload[index]);
351
+ const assigned = this.dialect.upsertAssignmentBinds(entity, conflictPaths, rows, opts.update);
352
+ return chunkWithinLimits(meta, payload, group, this.dialect.maxBindValues - assigned);
353
+ });
357
354
  if (statements.length === 1) {
358
355
  return this.runUpsert(entity, conflictPaths, payload, opts);
359
356
  }
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * Runs one statement of the cursor protocol on the querier's own connection, which holds the cursor, and
3
- * not through `all()`, whose `serialize` is not re-entrant.
3
+ * not through `all()`, which refuses a statement while a stream holds the querier.
4
4
  */
5
5
  type CursorExecutor<T> = (query: string, values?: unknown[]) => Promise<T[]>;
6
6
  /**
@@ -24,5 +24,8 @@ export declare class SqlExpression {
24
24
  * for the column it fills; other `raw` becomes the SQL `compile` renders, parenthesized; a literal is kept.
25
25
  */
26
26
  export declare function schemaDefault(value: unknown, compile: (sql: QueryRaw) => string): unknown;
27
- /** A default as text for an exact comparison: objects (SQL included) as JSON, numbers as their digits. */
27
+ /**
28
+ * A default as text for an exact comparison: numbers as their digits, and objects (SQL included) and text
29
+ * holding a JSON document as JSON with its keys in order, since a `jsonb` column reprints what it stores.
30
+ */
28
31
  export declare function writtenDefault(value: unknown): string;
@@ -1,4 +1,5 @@
1
1
  import { QueryRaw, SQL_VALUE_NAMES } from '../type/index.js';
2
+ import { isRecord } from '../util/object.util.js';
2
3
  import { SQL_VALUES } from '../util/raw.js';
3
4
  /**
4
5
  * A column default that is SQL rather than a literal, whether an entity or a migration declares it or
@@ -36,7 +37,24 @@ export function schemaDefault(value, compile) {
36
37
  }
37
38
  return value instanceof QueryRaw ? SqlExpression.parenthesized(compile(value)) : value;
38
39
  }
39
- /** A default as text for an exact comparison: objects (SQL included) as JSON, numbers as their digits. */
40
+ /**
41
+ * A default as text for an exact comparison: numbers as their digits, and objects (SQL included) and text
42
+ * holding a JSON document as JSON with its keys in order, since a `jsonb` column reprints what it stores.
43
+ */
40
44
  export function writtenDefault(value) {
41
- return typeof value === 'object' ? JSON.stringify(value) : String(value);
45
+ const document = typeof value === 'string' ? jsonDocument(value) : value;
46
+ return typeof document === 'object' && document !== null ? JSON.stringify(document, keysInOrder) : String(value);
47
+ }
48
+ /** What `text` spells as JSON, or nothing where it is no JSON at all. */
49
+ function jsonDocument(text) {
50
+ try {
51
+ return JSON.parse(text);
52
+ }
53
+ catch {
54
+ return undefined;
55
+ }
56
+ }
57
+ /** A `JSON.stringify` replacer writing each object's keys in order. */
58
+ function keysInOrder(_key, value) {
59
+ return isRecord(value) ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => (a < b ? -1 : 1))) : value;
42
60
  }
@@ -1,4 +1,4 @@
1
- import type { Change, ForeignKeySchema, IndexSchema, PrimaryKeySchema, StoredDefinition } from '../type/migration.js';
1
+ import type { Alteration, Change, ForeignKeySchema, IndexSchema, PrimaryKeySchema, StoredDefinition } from '../type/migration.js';
2
2
  import type { IndexFacet } from './indexDifferences.js';
3
3
  /**
4
4
  * Type categories universal across SQL dialects.
@@ -175,10 +175,7 @@ export type NodeChange<T, Altered = unknown> = {
175
175
  } | {
176
176
  readonly from: T;
177
177
  readonly to?: undefined;
178
- } | ({
179
- readonly from: T;
180
- readonly to: T;
181
- } & Altered);
178
+ } | (Alteration<T> & Altered);
182
179
  /** A column to create, drop or alter; an alter lists in `changed` which parts of the column differ. */
183
180
  export type ColumnDiff = NodeChange<ColumnNode, {
184
181
  readonly changed: readonly ColumnFacet[];
@@ -19,7 +19,7 @@ export declare abstract class AbstractSqliteQuerier extends AbstractSqlQuerier {
19
19
  protected iterate(query: string, values: SqliteBindValue[]): Promise<Iterable<RawRow> | AsyncIterable<RawRow>>;
20
20
  internalAll<T>(query: string, values?: unknown[]): Promise<T[]>;
21
21
  internalRun(query: string, values?: unknown[]): Promise<import("../type/query.js").QueryUpdateResult>;
22
- internalStream<T>(query: string, values?: unknown[]): AsyncGenerator<Awaited<T>, void, unknown>;
22
+ internalStream(query: string, values?: unknown[]): AsyncGenerator<RawRow, void, unknown>;
23
23
  /**
24
24
  * SQLite drivers hold a single shared handle rather than a connection from a pool, so releasing
25
25
  * a querier returns nothing at all. Drivers owning a closable per-querier connection override this.
@@ -19,6 +19,7 @@ export declare class SqliteDialect extends AbstractSqlDialect {
19
19
  protected refusal(message: string): string;
20
20
  /** SQLite's own cap on a function call before 3.48, which libSQL and `bun:sqlite`'s build still have. */
21
21
  readonly maxFunctionArgs: number;
22
+ readonly maxBindValues: number;
22
23
  /**
23
24
  * The [sqlite-vec](https://github.com/asg017/sqlite-vec) functions, which need that extension
24
25
  * loaded on the connection (see `Sqlite3QuerierPool`'s `extensions` option). libSQL and Turso ship
@@ -71,6 +71,7 @@ export class SqliteDialect extends AbstractSqlDialect {
71
71
  }
72
72
  /** SQLite's own cap on a function call before 3.48, which libSQL and `bun:sqlite`'s build still have. */
73
73
  maxFunctionArgs = 127;
74
+ maxBindValues = 32766;
74
75
  /**
75
76
  * The [sqlite-vec](https://github.com/asg017/sqlite-vec) functions, which need that extension
76
77
  * loaded on the connection (see `Sqlite3QuerierPool`'s `extensions` option). libSQL and Turso ship