uql-orm 0.93.1 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/browser/uql-browser.min.js +1 -1
- package/dist/browser/uql-browser.min.js.map +4 -4
- package/dist/dialect/abstractDialect.d.ts +2 -0
- package/dist/dialect/abstractDialect.js +4 -0
- package/dist/dialect/abstractSqlDialect.d.ts +13 -8
- package/dist/dialect/abstractSqlDialect.js +29 -12
- package/dist/dialect/mergeSqlDialect.d.ts +2 -2
- package/dist/dialect/mergeSqlDialect.js +1 -1
- package/dist/dialect/mysqlLikeSqlDialect.d.ts +3 -2
- package/dist/dialect/mysqlLikeSqlDialect.js +5 -2
- package/dist/dialect/pgLikeSqlDialect.d.ts +2 -0
- package/dist/dialect/pgLikeSqlDialect.js +4 -0
- package/dist/dialect/queryContext.d.ts +2 -0
- package/dist/dialect/queryContext.js +6 -0
- package/dist/maria/mariadbQuerier.d.ts +2 -1
- package/dist/maria/mariadbQuerier.js +2 -8
- package/dist/migrate/cli-config.d.ts +9 -0
- package/dist/migrate/cli-config.js +24 -6
- package/dist/migrate/cli.js +9 -2
- package/dist/migrate/ddl/defaultSql.js +3 -1
- package/dist/migrate/drift/driftDetector.js +7 -2
- package/dist/mongo/mongoDialect.d.ts +1 -0
- package/dist/mongo/mongoDialect.js +0 -1
- package/dist/mongo/mongodbQuerier.d.ts +3 -3
- package/dist/mongo/mongodbQuerier.js +5 -13
- package/dist/mssql/mssqlDialect.d.ts +5 -4
- package/dist/mssql/mssqlDialect.js +5 -4
- package/dist/mssql/mssqlQuerier.d.ts +2 -2
- package/dist/mssql/mssqlQuerier.js +1 -3
- package/dist/mysql/mysql2Querier.d.ts +1 -1
- package/dist/mysql/mysql2Querier.js +1 -9
- package/dist/pglite/index.d.ts +1 -0
- package/dist/pglite/index.js +1 -0
- package/dist/pglite/pgliteDialect.d.ts +6 -0
- package/dist/pglite/pgliteDialect.js +6 -0
- package/dist/pglite/pgliteQuerierPool.d.ts +2 -2
- package/dist/pglite/pgliteQuerierPool.js +2 -2
- package/dist/postgres/pgQuerier.d.ts +2 -4
- package/dist/postgres/pgQuerier.js +1 -9
- package/dist/querier/abstractQuerier.d.ts +33 -15
- package/dist/querier/abstractQuerier.js +116 -33
- package/dist/querier/abstractQuerierPool.d.ts +1 -1
- package/dist/querier/abstractQuerierPool.js +1 -1
- package/dist/querier/abstractSqlQuerier.d.ts +17 -12
- package/dist/querier/abstractSqlQuerier.js +61 -65
- package/dist/querier/cursorStream.d.ts +1 -1
- package/dist/schema/sqlExpression.d.ts +4 -1
- package/dist/schema/sqlExpression.js +20 -2
- package/dist/sqlite/abstractSqliteQuerier.d.ts +1 -1
- package/dist/sqlite/sqliteDialect.d.ts +1 -0
- package/dist/sqlite/sqliteDialect.js +1 -0
- package/dist/type/dialect.d.ts +3 -3
- package/dist/type/entity.d.ts +4 -4
- package/dist/type/querier.d.ts +1 -1
- package/dist/type/query.d.ts +1 -12
- package/dist/type/universalQuerier.d.ts +10 -6
- package/dist/util/object.util.d.ts +2 -0
- package/dist/util/object.util.js +8 -0
- package/dist/util/query.util.d.ts +3 -1
- package/dist/util/query.util.js +7 -0
- package/dist/util/sql.util.d.ts +2 -4
- package/dist/util/sql.util.js +10 -19
- package/dist/util/triggerWrite.d.ts +3 -3
- package/dist/util/triggerWrite.js +2 -2
- package/package.json +1 -1
- package/skills/uql-orm/SKILL.md +8 -3
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { assertSoleId, getMeta, idOf, namesKey, relationOf } from '../entity/index.js';
|
|
2
2
|
import { namesRows } from '../dialect/operators.js';
|
|
3
3
|
import { parseQueryLock } from '../type/index.js';
|
|
4
|
-
import { cascadesOnDelete, childrenOf, clone, entityName, fillOnFields, filterFieldKeys, filterPersistableRelationKeys, forEachRequestedRelation, getKeys, hasKeys, getRelationRequestSummary, guardWrite, idOnlyQuery, keySet, isPagedQuery, isScalarId, LoggerWrapper, parentJoins, queryLoggerFor, parseRelationAtKey, parseRelationQueryValue, rowKey, runHooks, securityConditions, someKey, targetKeyColumns,
|
|
4
|
+
import { cascadesOnDelete, childrenOf, chunk, clone, entityName, fillOnFields, filterFieldKeys, filterPersistableRelationKeys, forEachRequestedRelation, getKeys, hasKeys, getRelationRequestSummary, guardWrite, idOnlyQuery, keySet, isPagedQuery, isScalarId, LoggerWrapper, parentJoins, queryLoggerFor, parseRelationAtKey, parseRelationQueryValue, rowKey, runHooks, securityConditions, someKey, targetKeyColumns, whereEach, whereIds, whereKeysIn, whereWith, withoutSoftDeleteFilter, } from '../util/index.js';
|
|
5
5
|
import { UqlOptimisticLockError, UqlUsageError } from '../util/uqlError.js';
|
|
6
6
|
import { keysetRead } from './keyset.js';
|
|
7
7
|
import { enrichError } from './queryError.js';
|
|
@@ -87,6 +87,15 @@ function assertLockableUpdate(meta, q, settles) {
|
|
|
87
87
|
function writtenIds(meta, rows) {
|
|
88
88
|
return rows.map((row) => (namesKey(meta, row) ? idOf(meta, row) : undefined));
|
|
89
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Whether an upsert has to read before it writes: `ON CONFLICT` updates a guarded row whatever tenant it belongs
|
|
92
|
+
* to, and writes no relation.
|
|
93
|
+
*/
|
|
94
|
+
function upsertsByReading(meta, rows, update) {
|
|
95
|
+
const relates = (payload) => filterPersistableRelationKeys(meta, payload, 'persist').length > 0;
|
|
96
|
+
return (securityConditions(meta).length > 0 ||
|
|
97
|
+
(relates(meta.relations) && (rows.some(relates) || (update !== undefined && relates(update)))));
|
|
98
|
+
}
|
|
90
99
|
/** What `ON CONFLICT` assigns a row it finds by default: the payload, less the columns it matched on. */
|
|
91
100
|
function conflictAssignments(row, conflictPaths) {
|
|
92
101
|
const assigned = { ...row };
|
|
@@ -126,6 +135,7 @@ function isEntityFirst(args) {
|
|
|
126
135
|
const [first] = args;
|
|
127
136
|
return typeof first === 'function' && first.prototype !== undefined;
|
|
128
137
|
}
|
|
138
|
+
const STREAM_HOLDS_QUERIER = 'a stream is reading on this querier: run the statement after its loop, or on another querier';
|
|
129
139
|
/** Base class for all database queriers. */
|
|
130
140
|
export class AbstractQuerier {
|
|
131
141
|
extra;
|
|
@@ -135,6 +145,8 @@ export class AbstractQuerier {
|
|
|
135
145
|
* and ensuring that the database connection is used safely across concurrent calls.
|
|
136
146
|
*/
|
|
137
147
|
taskQueue = Promise.resolve();
|
|
148
|
+
/** The stream reading on the connection, from its first row asked for until its loop ends. */
|
|
149
|
+
#stream;
|
|
138
150
|
/**
|
|
139
151
|
* A querier is one unit of work, so releasing ends it. Checked where each backend reaches for its
|
|
140
152
|
* connection, on every driver and not just the pooled ones.
|
|
@@ -205,7 +217,37 @@ export class AbstractQuerier {
|
|
|
205
217
|
findManyStream(...args) {
|
|
206
218
|
const [entity, q, opts] = entityArgs(args);
|
|
207
219
|
this.validateReadQuery(entity, q);
|
|
208
|
-
|
|
220
|
+
const rows = this.internalFindManyStream(entity, q, opts);
|
|
221
|
+
return this.holdWhileReading(this.listensForLoad(entity, q.$populate) ? this.loadedEach(entity, rows, q.$populate) : rows);
|
|
222
|
+
}
|
|
223
|
+
/** `afterLoad` on each row before the loop sees it, as `findMany` runs it on each row it read. */
|
|
224
|
+
async *loadedEach(entity, rows, populate) {
|
|
225
|
+
for await (const row of rows) {
|
|
226
|
+
await this.emitLoaded(entity, [row], populate);
|
|
227
|
+
yield row;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* `rows`, holding the querier from the first row asked for until the loop ends. Most drivers queue a
|
|
232
|
+
* statement behind an open read, which a loop awaiting it never finishes, so `serialize` refuses one
|
|
233
|
+
* meanwhile, on every engine alike, and `release` closes the stream first.
|
|
234
|
+
*/
|
|
235
|
+
holdWhileReading(rows) {
|
|
236
|
+
const read = async function* (querier) {
|
|
237
|
+
if (querier.#stream) {
|
|
238
|
+
throw new UqlUsageError(STREAM_HOLDS_QUERIER);
|
|
239
|
+
}
|
|
240
|
+
querier.#stream = stream;
|
|
241
|
+
try {
|
|
242
|
+
await querier.taskQueue;
|
|
243
|
+
yield* rows;
|
|
244
|
+
}
|
|
245
|
+
finally {
|
|
246
|
+
querier.#stream = undefined;
|
|
247
|
+
}
|
|
248
|
+
};
|
|
249
|
+
const stream = read(this);
|
|
250
|
+
return stream;
|
|
209
251
|
}
|
|
210
252
|
async findManyAndCount(...args) {
|
|
211
253
|
const [entity, q, opts] = entityArgs(args);
|
|
@@ -272,12 +314,13 @@ export class AbstractQuerier {
|
|
|
272
314
|
return writtenIds(meta, rows);
|
|
273
315
|
});
|
|
274
316
|
}
|
|
275
|
-
/** Fills and guards `rows`, then writes them: an insert, and the insert half of a
|
|
317
|
+
/** Fills and guards `rows`, then writes them and their relations: an insert, and the insert half of a read upsert. */
|
|
276
318
|
async insertRows(entity, rows) {
|
|
277
319
|
const meta = getMeta(entity);
|
|
278
320
|
fillOnFields(meta, rows, 'onInsert');
|
|
279
321
|
guardWrite(meta, rows, 'insert');
|
|
280
322
|
await this.internalInsertMany(entity, rows);
|
|
323
|
+
await this.insertRelations(entity, rows);
|
|
281
324
|
}
|
|
282
325
|
async updateOneById(entity, id, payload, opts) {
|
|
283
326
|
assertIdValue(entity, id);
|
|
@@ -312,7 +355,10 @@ export class AbstractQuerier {
|
|
|
312
355
|
if (!ids.length) {
|
|
313
356
|
return 0;
|
|
314
357
|
}
|
|
315
|
-
|
|
358
|
+
let changes = 0;
|
|
359
|
+
for (const batch of chunk(ids, this.dialect.keyListCapacity(meta.ids.length))) {
|
|
360
|
+
changes += await this.updateColumns(entity, { $where: whereIds(meta, batch) }, row, opts, batch.length);
|
|
361
|
+
}
|
|
316
362
|
for (const relKey of relKeys) {
|
|
317
363
|
await this.saveRelation(entity, relKey, ids.map((id) => ({ id, value: row[relKey] })), true);
|
|
318
364
|
}
|
|
@@ -376,36 +422,37 @@ export class AbstractQuerier {
|
|
|
376
422
|
return this.hooked(entity, 'Update', [payload], ([row]) => this.updateRows(entity, q, row, opts, undefined));
|
|
377
423
|
}
|
|
378
424
|
/** Fires `beforeUpsert`/`afterUpsert`: which branch a row takes is the database's to decide, so neither the insert's nor the update's pair fits. */
|
|
379
|
-
async upsertOne(entity, conflictPaths, payload,
|
|
425
|
+
async upsertOne(entity, conflictPaths, payload, update) {
|
|
380
426
|
const meta = getMeta(entity);
|
|
381
427
|
assertUnversioned(meta, "'upsertOne'");
|
|
382
428
|
return this.hooked(entity, 'Upsert', [payload], async (rows) => {
|
|
383
|
-
const { ids, changes, created } =
|
|
384
|
-
? await this.
|
|
385
|
-
: await this.internalUpsertOne(entity, conflictPaths, rows[0],
|
|
429
|
+
const { ids, changes, created } = upsertsByReading(meta, rows, update)
|
|
430
|
+
? await this.readThenUpsert(entity, conflictPaths, rows, update)
|
|
431
|
+
: await this.internalUpsertOne(entity, conflictPaths, rows[0], update);
|
|
386
432
|
adoptReportedIds(meta, rows, ids);
|
|
387
433
|
const [id] = writtenIds(meta, rows);
|
|
388
434
|
return { id, changes, created };
|
|
389
435
|
});
|
|
390
436
|
}
|
|
391
|
-
async upsertMany(entity, conflictPaths, payload,
|
|
437
|
+
async upsertMany(entity, conflictPaths, payload, update) {
|
|
392
438
|
const meta = getMeta(entity);
|
|
393
439
|
assertUnversioned(meta, "'upsertMany'");
|
|
394
440
|
return this.hooked(entity, 'Upsert', payload, async (rows) => {
|
|
395
|
-
const { ids, changes } =
|
|
396
|
-
? await this.
|
|
397
|
-
: await this.internalUpsertMany(entity, conflictPaths, rows,
|
|
441
|
+
const { ids, changes } = upsertsByReading(meta, rows, update)
|
|
442
|
+
? await this.readThenUpsert(entity, conflictPaths, rows, update)
|
|
443
|
+
: await this.internalUpsertMany(entity, conflictPaths, rows, update);
|
|
398
444
|
adoptReportedIds(meta, rows, ids);
|
|
399
445
|
return { ids: writtenIds(meta, rows), changes };
|
|
400
446
|
});
|
|
401
447
|
}
|
|
402
448
|
/**
|
|
403
|
-
* An upsert
|
|
404
|
-
*
|
|
405
|
-
*
|
|
449
|
+
* An upsert as a read, then writes: the rows the conflict names are read through the filters, those found update
|
|
450
|
+
* as `updateMany` does and the rest insert, so either branch writes its relations and a guarded row stays its
|
|
451
|
+
* tenant's. A row inserted concurrently fails on its key. See architecture/security-filter-writes.md.
|
|
406
452
|
*/
|
|
407
|
-
async
|
|
408
|
-
|
|
453
|
+
async readThenUpsert(entity, conflictPaths, rows, update) {
|
|
454
|
+
const meta = getMeta(entity);
|
|
455
|
+
guardWrite(meta, rows, 'insert');
|
|
409
456
|
if (!rows.length) {
|
|
410
457
|
return { changes: 0 };
|
|
411
458
|
}
|
|
@@ -416,7 +463,10 @@ export class AbstractQuerier {
|
|
|
416
463
|
for (const [index, row] of rows.entries()) {
|
|
417
464
|
// An empty `update` leaves a found row as it is, as `DO NOTHING` does.
|
|
418
465
|
if (ids[index] !== undefined && (update === undefined || hasKeys(update))) {
|
|
419
|
-
|
|
466
|
+
const q = { $where: whereEach(keys, (key) => row[key]) };
|
|
467
|
+
// A copy each: the update fills its `onUpdate` fields into what it is handed.
|
|
468
|
+
const assigned = update ? { ...update } : conflictAssignments(row, conflictPaths);
|
|
469
|
+
changes += await this.updateRows(entity, q, assigned, undefined, undefined);
|
|
420
470
|
}
|
|
421
471
|
}
|
|
422
472
|
const inserts = rows.filter((_, index) => ids[index] === undefined);
|
|
@@ -428,7 +478,7 @@ export class AbstractQuerier {
|
|
|
428
478
|
// A found row's key is adopted by the caller; an inserted one carries the key its insert wrote.
|
|
429
479
|
return { changes, created, ids };
|
|
430
480
|
};
|
|
431
|
-
// One row
|
|
481
|
+
// One row writes as an insert or an update of it does, with no transaction of its own.
|
|
432
482
|
return rows.length === 1 ? write() : this.transaction(write);
|
|
433
483
|
}
|
|
434
484
|
async deleteOneById(entity, id, opts) {
|
|
@@ -453,11 +503,14 @@ export class AbstractQuerier {
|
|
|
453
503
|
return 0;
|
|
454
504
|
}
|
|
455
505
|
await this.emitHook(entity, 'beforeDelete', doomed);
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
506
|
+
let changes = 0;
|
|
507
|
+
for (const batch of chunk(ids, this.dialect.keyListCapacity(meta.ids.length))) {
|
|
508
|
+
// Children first: they hold the foreign key, which a schema without `ON DELETE CASCADE` enforces.
|
|
509
|
+
if (cascades) {
|
|
510
|
+
await this.deleteRelations(entity, batch, opts);
|
|
511
|
+
}
|
|
512
|
+
changes += await this.internalDeleteMany(entity, { $where: whereIds(meta, batch) }, opts);
|
|
459
513
|
}
|
|
460
|
-
const changes = await this.internalDeleteMany(entity, { $where: whereIds(meta, ids) }, opts);
|
|
461
514
|
await this.emitHook(entity, 'afterDelete', doomed);
|
|
462
515
|
return changes;
|
|
463
516
|
}
|
|
@@ -554,8 +607,9 @@ export class AbstractQuerier {
|
|
|
554
607
|
const holder = relOpts.through ? relOpts.through() : relEntity;
|
|
555
608
|
const parentColumn = soleParentColumn(relOpts);
|
|
556
609
|
if (isUpdate) {
|
|
557
|
-
const
|
|
558
|
-
|
|
610
|
+
for (const batch of chunk(writes, this.dialect.keyListCapacity(1))) {
|
|
611
|
+
await this.deleteMany(holder, { $where: { [parentColumn]: batch.map(({ id }) => id) } });
|
|
612
|
+
}
|
|
559
613
|
}
|
|
560
614
|
// Each parent gets its own copies, so a row listed for two parents is written twice.
|
|
561
615
|
const children = writes.flatMap(({ id, value }) => [value ?? []].flat().map((row) => ({ id, row })));
|
|
@@ -650,13 +704,14 @@ export class AbstractQuerier {
|
|
|
650
704
|
const meta = getMeta(entity);
|
|
651
705
|
const [idKey] = meta.ids;
|
|
652
706
|
const keys = getKeys(conflictPaths);
|
|
653
|
-
const
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
707
|
+
const distinct = [...new Map(rows.map((row) => [rowKey(row, keys), row])).values()];
|
|
708
|
+
const found = [];
|
|
709
|
+
for (const batch of chunk(distinct, this.dialect.keyListCapacity(keys.length))) {
|
|
710
|
+
const q = { $select: keySet([idKey, ...keys]), $where: whereKeysIn(keys, batch) };
|
|
711
|
+
found.push(await this.internalFindMany(entity, q, { filters: withoutSoftDeleteFilter(undefined) }));
|
|
712
|
+
}
|
|
658
713
|
const byConflict = new Map();
|
|
659
|
-
for (const row of found) {
|
|
714
|
+
for (const row of found.flat()) {
|
|
660
715
|
const key = rowKey(row, keys);
|
|
661
716
|
byConflict.set(key, byConflict.has(key) ? undefined : row[idKey]);
|
|
662
717
|
}
|
|
@@ -694,6 +749,9 @@ export class AbstractQuerier {
|
|
|
694
749
|
}
|
|
695
750
|
/** Runs `task` after everything already queued, one at a time. Not re-entrant: never nest `serialize` calls. */
|
|
696
751
|
serialize(task) {
|
|
752
|
+
if (this.#stream) {
|
|
753
|
+
return Promise.reject(new UqlUsageError(STREAM_HOLDS_QUERIER));
|
|
754
|
+
}
|
|
697
755
|
const res = this.taskQueue.then(task);
|
|
698
756
|
this.taskQueue = res.catch(() => { });
|
|
699
757
|
return res;
|
|
@@ -711,6 +769,26 @@ export class AbstractQuerier {
|
|
|
711
769
|
this.logger.logQuery(query, values, Math.round(performance.now() - startTime));
|
|
712
770
|
}
|
|
713
771
|
}
|
|
772
|
+
/**
|
|
773
|
+
* `rows` as {@link timed} runs a statement: logged once they end, timed to the first row since the
|
|
774
|
+
* time after it is the loop's, and a failure tagged with `query`.
|
|
775
|
+
*/
|
|
776
|
+
async *timedStream(query, values, rows) {
|
|
777
|
+
const startTime = performance.now();
|
|
778
|
+
let answeredAt;
|
|
779
|
+
try {
|
|
780
|
+
for await (const row of rows) {
|
|
781
|
+
answeredAt ??= performance.now();
|
|
782
|
+
yield row;
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
catch (err) {
|
|
786
|
+
throw enrichError(err, this.logger, query, values);
|
|
787
|
+
}
|
|
788
|
+
finally {
|
|
789
|
+
this.logger.logQuery(query, values, Math.round((answeredAt ?? performance.now()) - startTime));
|
|
790
|
+
}
|
|
791
|
+
}
|
|
714
792
|
beginTransaction(opts) {
|
|
715
793
|
return this.serialize(async () => {
|
|
716
794
|
if (this.hasOpenTransaction) {
|
|
@@ -740,11 +818,16 @@ export class AbstractQuerier {
|
|
|
740
818
|
});
|
|
741
819
|
}
|
|
742
820
|
/**
|
|
743
|
-
*
|
|
744
|
-
*
|
|
821
|
+
* Closes a stream left open and rolls back an unfinished transaction, then hands the connection back,
|
|
822
|
+
* discarding it if either failed: the pool would otherwise take one still reading. Never throws first,
|
|
823
|
+
* since `await using` has no other way to release.
|
|
745
824
|
*/
|
|
746
825
|
async release() {
|
|
747
826
|
let discard = false;
|
|
827
|
+
await this.#stream?.return(undefined).catch((err) => {
|
|
828
|
+
this.logger.logError('closing an open stream failed; discarding the connection', err);
|
|
829
|
+
discard = true;
|
|
830
|
+
});
|
|
748
831
|
if (this.hasOpenTransaction) {
|
|
749
832
|
this.logger.logWarn('rolling back a transaction left open at release');
|
|
750
833
|
// The rollback doubles as a health check. One that succeeds proves the connection round-trips and
|
|
@@ -29,7 +29,7 @@ export declare abstract class AbstractQuerierPool<Q extends Querier, D extends A
|
|
|
29
29
|
readonly findMany: UniversalQuerier['findMany'];
|
|
30
30
|
/**
|
|
31
31
|
* The connection outlives the call: it is held until the iterator is drained or closed by a `break` or
|
|
32
|
-
* `throw`. Abandoning the iterator leaks it
|
|
32
|
+
* `throw`. Abandoning the iterator leaks it, so consume it in a `for await`.
|
|
33
33
|
*/
|
|
34
34
|
readonly findManyStream: UniversalQuerier['findManyStream'];
|
|
35
35
|
private streamWithQuerier;
|
|
@@ -91,7 +91,7 @@ export class AbstractQuerierPool {
|
|
|
91
91
|
findMany = (...args) => this.withQuerier((querier) => querier.findMany(...args));
|
|
92
92
|
/**
|
|
93
93
|
* The connection outlives the call: it is held until the iterator is drained or closed by a `break` or
|
|
94
|
-
* `throw`. Abandoning the iterator leaks it
|
|
94
|
+
* `throw`. Abandoning the iterator leaks it, so consume it in a `for await`.
|
|
95
95
|
*/
|
|
96
96
|
findManyStream = (...args) => this.streamWithQuerier((querier) => querier.findManyStream(...args));
|
|
97
97
|
async *streamWithQuerier(read) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AbstractSqlDialect } from '../dialect/index.js';
|
|
2
|
-
import type { EntityData, ExtraOptions, Query, QueryAggMap, QueryAggregate, QueryAggregateResult, QueryConflictPaths, QueryPage, QueryGroupMap, QueryOptions, QuerySearch, QueryUpdateResult, SqlQuerier, TransactionOptions, Type, UpdatePayload
|
|
2
|
+
import type { EntityData, ExtraOptions, Query, QueryAggMap, QueryAggregate, QueryAggregateResult, QueryConflictPaths, QueryPage, QueryGroupMap, QueryOptions, QuerySearch, QueryUpdateResult, RawRow, SqlQuerier, TransactionOptions, Type, UpdatePayload } from '../type/index.js';
|
|
3
3
|
import type { BuildUpdateResultPayload } from '../util/sql.util.js';
|
|
4
4
|
import { AbstractQuerier } from './abstractQuerier.js';
|
|
5
5
|
export declare abstract class AbstractSqlQuerier extends AbstractQuerier implements SqlQuerier {
|
|
@@ -34,6 +34,8 @@ export declare abstract class AbstractSqlQuerier extends AbstractQuerier impleme
|
|
|
34
34
|
protected lazyConnect(): Promise<void>;
|
|
35
35
|
all<T>(query: string, values?: readonly unknown[]): Promise<T[]>;
|
|
36
36
|
run(query: string, values?: readonly unknown[]): Promise<QueryUpdateResult>;
|
|
37
|
+
/** Refused before the driver fails it, or PGlite answers it and every read after it wrong. */
|
|
38
|
+
private assertBindBudget;
|
|
37
39
|
/** The rows of a statement the dialect builds. */
|
|
38
40
|
private query;
|
|
39
41
|
/** Runs a statement the dialect builds. */
|
|
@@ -53,22 +55,25 @@ export declare abstract class AbstractSqlQuerier extends AbstractQuerier impleme
|
|
|
53
55
|
*/
|
|
54
56
|
protected internalFindManyAndCount<E extends object>(entity: Type<E>, q: Query<E>, opts?: QueryOptions): Promise<[E[], number]>;
|
|
55
57
|
private selectRows;
|
|
56
|
-
|
|
58
|
+
/**
|
|
59
|
+
* The one read not going through `all`, so it checks its budget and connects on its own; its values go as
|
|
60
|
+
* the context holds them, each normalized as it was bound. The tuning is guarded as `selectRows` says.
|
|
61
|
+
*/
|
|
57
62
|
protected internalFindManyStream<E extends object>(entity: Type<E>, q: Query<E>, opts?: QueryOptions): AsyncGenerator<Awaited<E>, void, unknown>;
|
|
58
63
|
/**
|
|
59
64
|
* A read's rows one at a time: paged through a server-side cursor where the engine has one, read whole
|
|
60
|
-
* where it has none. A driver that streams on its own overrides this
|
|
65
|
+
* where it has none. A driver that streams on its own overrides this; a Node `Readable` it hands back
|
|
66
|
+
* closes itself when the loop exits early.
|
|
61
67
|
*/
|
|
62
|
-
protected internalStream
|
|
68
|
+
protected internalStream(query: string, values?: unknown[]): AsyncIterable<RawRow>;
|
|
63
69
|
/**
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
* only sparse on Postgres.
|
|
70
|
+
* How each row of one statement becomes the entity's, for a read, a stream and a to-many alike: its dotted
|
|
71
|
+
* columns nested as the first row names them, and its values decoded as `hydratableFields` says, resolved
|
|
72
|
+
* once. Both live with the dialect, because a `sparsevec` is only sparse on Postgres.
|
|
68
73
|
*/
|
|
69
|
-
private
|
|
74
|
+
private rowReader;
|
|
70
75
|
/**
|
|
71
|
-
* One row of {@link
|
|
76
|
+
* One row of {@link rowReader}. A related row arrives as its parent's statement read it: a to-one
|
|
72
77
|
* joined and unflattened, there only when its key is, since an unmatched join still fills a computed
|
|
73
78
|
* column or a to-many's empty array; a to-many as a JSON array, which a driver may hand over as text.
|
|
74
79
|
* Each is an object of its own, so the walk reaches none twice.
|
|
@@ -85,8 +90,8 @@ export declare abstract class AbstractSqlQuerier extends AbstractQuerier impleme
|
|
|
85
90
|
protected internalAggregate<E extends object, G extends QueryGroupMap<E>, A extends QueryAggMap<E>>(entity: Type<E>, q: QueryAggregate<E, G, A>, opts?: QueryOptions): Promise<QueryAggregateResult<E, G, A>[]>;
|
|
86
91
|
internalInsertMany<E extends object>(entity: Type<E>, rows: EntityData<E>[]): Promise<void>;
|
|
87
92
|
internalUpdateMany<E extends object>(entity: Type<E>, q: QuerySearch<E>, payload: UpdatePayload<E>, opts?: QueryOptions): Promise<number>;
|
|
88
|
-
protected internalUpsertOne<E extends object>(entity: Type<E>, conflictPaths: QueryConflictPaths<E>, payload: E,
|
|
89
|
-
protected internalUpsertMany<E extends object>(entity: Type<E>, conflictPaths: QueryConflictPaths<E>, payload: E[],
|
|
93
|
+
protected internalUpsertOne<E extends object>(entity: Type<E>, conflictPaths: QueryConflictPaths<E>, payload: E, update?: UpdatePayload<E>): Promise<QueryUpdateResult>;
|
|
94
|
+
protected internalUpsertMany<E extends object>(entity: Type<E>, conflictPaths: QueryConflictPaths<E>, payload: E[], update?: UpdatePayload<E>): Promise<QueryUpdateResult>;
|
|
90
95
|
private runUpsert;
|
|
91
96
|
protected internalDeleteMany<E extends object>(entity: Type<E>, q: QuerySearch<E>, opts?: QueryOptions): Promise<number>;
|
|
92
97
|
get hasOpenTransaction(): boolean;
|
|
@@ -2,7 +2,7 @@ import { AGGREGATE_VALUE_ALIAS, TOTAL_ALIAS } from '../dialect/aliases.js';
|
|
|
2
2
|
import { decodeColumn } from '../dialect/hydrateColumn.js';
|
|
3
3
|
import { getMeta, namesKey } from '../entity/index.js';
|
|
4
4
|
import { COUNT_RESULT_KEY } from '../type/index.js';
|
|
5
|
-
import { buildUpdateResult, clone, getInsertFieldKeys, insertShapeOf, isAutoIncrement, isRecord, obtainAttrsPaths, unflatObject,
|
|
5
|
+
import { buildUpdateResult, chunk, clone, getInsertFieldKeys, insertShapeOf, isAutoIncrement, isRecord, obtainAttrsPaths, unflatObject, } from '../util/index.js';
|
|
6
6
|
import { UqlUsageError } from '../util/uqlError.js';
|
|
7
7
|
import { AbstractQuerier } from './abstractQuerier.js';
|
|
8
8
|
import { streamViaCursor } from './cursorStream.js';
|
|
@@ -40,18 +40,13 @@ function groupByInsertShape(meta, payload) {
|
|
|
40
40
|
return [...groups.values()];
|
|
41
41
|
}
|
|
42
42
|
/**
|
|
43
|
-
* A group's row indexes split into statements within the
|
|
44
|
-
* `DEFAULT` cells bind no parameter, so fields-per-record
|
|
45
|
-
* write splits on this: D1 allows 100 binds, which a couple of dozen
|
|
43
|
+
* A group's row indexes split into statements of at most `maxRows` rows within the bind budget, payload order
|
|
44
|
+
* kept, a row with no column to write in one of its own. `DEFAULT` cells bind no parameter, so fields-per-record
|
|
45
|
+
* is a safe upper bound. Every multi-row write splits on this: D1 allows 100 binds, which a couple of dozen reach.
|
|
46
46
|
*/
|
|
47
|
-
function
|
|
47
|
+
function chunkWithinLimits(meta, payload, group, maxBindValues, maxRows = Infinity) {
|
|
48
48
|
const fieldsPerRecord = getInsertFieldKeys(meta, group.map((index) => payload[index])).length;
|
|
49
|
-
|
|
50
|
-
const chunks = [];
|
|
51
|
-
for (let start = 0; start < group.length; start += size) {
|
|
52
|
-
chunks.push(group.slice(start, start + size));
|
|
53
|
-
}
|
|
54
|
-
return chunks;
|
|
49
|
+
return chunk(group, fieldsPerRecord ? Math.max(1, Math.min(maxRows, Math.floor(maxBindValues / fieldsPerRecord))) : 1);
|
|
55
50
|
}
|
|
56
51
|
export class AbstractSqlQuerier extends AbstractQuerier {
|
|
57
52
|
dialect;
|
|
@@ -95,17 +90,26 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
95
90
|
}
|
|
96
91
|
}
|
|
97
92
|
async all(query, values) {
|
|
93
|
+
this.assertBindBudget(values);
|
|
98
94
|
return this.serialize(async () => {
|
|
99
95
|
await this.lazyConnect();
|
|
100
96
|
return this.timed(query, values, () => this.internalAll(query, this.dialect.normalizeValues(values)));
|
|
101
97
|
});
|
|
102
98
|
}
|
|
103
99
|
async run(query, values) {
|
|
100
|
+
this.assertBindBudget(values);
|
|
104
101
|
return this.serialize(async () => {
|
|
105
102
|
await this.lazyConnect();
|
|
106
103
|
return this.timed(query, values, () => this.internalRun(query, this.dialect.normalizeValues(values)));
|
|
107
104
|
});
|
|
108
105
|
}
|
|
106
|
+
/** Refused before the driver fails it, or PGlite answers it and every read after it wrong. */
|
|
107
|
+
assertBindBudget(values) {
|
|
108
|
+
const { maxBindValues, dialectName } = this.dialect;
|
|
109
|
+
if (values && values.length > maxBindValues) {
|
|
110
|
+
throw new UqlUsageError(`a statement binding ${values.length} values is past the ${maxBindValues} ${dialectName} takes; split the list it binds`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
109
113
|
/** The rows of a statement the dialect builds. */
|
|
110
114
|
query(build) {
|
|
111
115
|
const ctx = this.dialect.createContext();
|
|
@@ -138,7 +142,7 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
138
142
|
}
|
|
139
143
|
}
|
|
140
144
|
async internalFindMany(entity, q, opts) {
|
|
141
|
-
return
|
|
145
|
+
return (await this.selectRows(entity, q, opts)).map(this.rowReader(entity));
|
|
142
146
|
}
|
|
143
147
|
/** Every row `q` matches past its page, deduplicated where it reads `$distinct`. */
|
|
144
148
|
countUnpaged(entity, q, opts) {
|
|
@@ -161,7 +165,7 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
161
165
|
for (const row of rows) {
|
|
162
166
|
delete row[TOTAL_ALIAS];
|
|
163
167
|
}
|
|
164
|
-
return [this.
|
|
168
|
+
return [rows.map(this.rowReader(entity)), total];
|
|
165
169
|
}
|
|
166
170
|
async selectRows(entity, q, opts, totalAlias) {
|
|
167
171
|
// Guarded rather than awaited unconditionally, here and in the stream below: an `await` on this
|
|
@@ -172,40 +176,27 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
172
176
|
}
|
|
173
177
|
return this.query((ctx) => this.dialect.find(ctx, entity, q, opts, totalAlias));
|
|
174
178
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
}
|
|
179
|
+
/**
|
|
180
|
+
* The one read not going through `all`, so it checks its budget and connects on its own; its values go as
|
|
181
|
+
* the context holds them, each normalized as it was bound. The tuning is guarded as `selectRows` says.
|
|
182
|
+
*/
|
|
180
183
|
async *internalFindManyStream(entity, q, opts) {
|
|
181
|
-
|
|
184
|
+
const ctx = this.dialect.createContext();
|
|
185
|
+
this.dialect.find(ctx, entity, q, opts);
|
|
186
|
+
this.assertBindBudget(ctx.values);
|
|
182
187
|
if (q.$candidates !== undefined) {
|
|
183
188
|
await this.applyVectorTuning(entity, q);
|
|
184
189
|
}
|
|
185
|
-
const meta = getMeta(entity);
|
|
186
|
-
// The one path not going through `all`/`run`, so it connects on its own.
|
|
187
190
|
await this.lazyConnect();
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
this.dialect.find(ctx, entity, q, opts);
|
|
192
|
-
const fields = this.dialect.hydratableFields(entity);
|
|
193
|
-
let attrsPaths;
|
|
194
|
-
try {
|
|
195
|
-
for await (const row of this.internalStream(ctx.sql, ctx.values)) {
|
|
196
|
-
attrsPaths ??= obtainAttrsPaths(row);
|
|
197
|
-
const found = unflatObject(row, attrsPaths);
|
|
198
|
-
this.hydrateFields(meta, fields, found);
|
|
199
|
-
yield found;
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
catch (err) {
|
|
203
|
-
throw enrichError(err, this.logger, ctx.sql, ctx.values);
|
|
191
|
+
const read = this.rowReader(entity);
|
|
192
|
+
for await (const row of this.timedStream(ctx.sql, ctx.values, this.internalStream(ctx.sql, ctx.values))) {
|
|
193
|
+
yield read(row);
|
|
204
194
|
}
|
|
205
195
|
}
|
|
206
196
|
/**
|
|
207
197
|
* A read's rows one at a time: paged through a server-side cursor where the engine has one, read whole
|
|
208
|
-
* where it has none. A driver that streams on its own overrides this
|
|
198
|
+
* where it has none. A driver that streams on its own overrides this; a Node `Readable` it hands back
|
|
199
|
+
* closes itself when the loop exits early.
|
|
209
200
|
*/
|
|
210
201
|
async *internalStream(query, values) {
|
|
211
202
|
if (!this.dialect.features.serverSideCursors) {
|
|
@@ -215,20 +206,23 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
215
206
|
yield* streamViaCursor((sql, params) => this.internalAll(sql, params), query, values, this.hasOpenTransaction);
|
|
216
207
|
}
|
|
217
208
|
/**
|
|
218
|
-
*
|
|
219
|
-
*
|
|
220
|
-
*
|
|
221
|
-
* only sparse on Postgres.
|
|
209
|
+
* How each row of one statement becomes the entity's, for a read, a stream and a to-many alike: its dotted
|
|
210
|
+
* columns nested as the first row names them, and its values decoded as `hydratableFields` says, resolved
|
|
211
|
+
* once. Both live with the dialect, because a `sparsevec` is only sparse on Postgres.
|
|
222
212
|
*/
|
|
223
|
-
|
|
213
|
+
rowReader(entity) {
|
|
224
214
|
const meta = getMeta(entity);
|
|
225
215
|
const fields = this.dialect.hydratableFields(entity);
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
216
|
+
let attrsPaths;
|
|
217
|
+
return (row) => {
|
|
218
|
+
attrsPaths ??= obtainAttrsPaths(row);
|
|
219
|
+
const found = unflatObject(row, attrsPaths);
|
|
220
|
+
this.hydrateFields(meta, fields, found);
|
|
221
|
+
return found;
|
|
222
|
+
};
|
|
229
223
|
}
|
|
230
224
|
/**
|
|
231
|
-
* One row of {@link
|
|
225
|
+
* One row of {@link rowReader}. A related row arrives as its parent's statement read it: a to-one
|
|
232
226
|
* joined and unflattened, there only when its key is, since an unmatched join still fills a computed
|
|
233
227
|
* column or a to-many's empty array; a to-many as a JSON array, which a driver may hand over as text.
|
|
234
228
|
* Each is an object of its own, so the walk reaches none twice.
|
|
@@ -261,9 +255,8 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
261
255
|
const relEntity = rel.entity();
|
|
262
256
|
if (typeof value === 'string' || Array.isArray(value)) {
|
|
263
257
|
// A to-many's rows, as flat as a statement's own.
|
|
264
|
-
const rows =
|
|
265
|
-
row[key] = rows;
|
|
266
|
-
this.hydrateAll(relEntity, rows);
|
|
258
|
+
const rows = typeof value === 'string' ? JSON.parse(value) : value;
|
|
259
|
+
row[key] = rows.map(this.rowReader(relEntity));
|
|
267
260
|
}
|
|
268
261
|
else if (isRecord(value)) {
|
|
269
262
|
const relMeta = getMeta(relEntity);
|
|
@@ -325,9 +318,9 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
325
318
|
}
|
|
326
319
|
// Per group, not per batch: the two carry different columns - one names the key, one does not -
|
|
327
320
|
// so a budget taken over their union would under-fill the statement that is missing one.
|
|
328
|
-
for (const indexes of
|
|
329
|
-
const
|
|
330
|
-
const { ids = [] } = await this.exec((ctx) => this.dialect.insert(ctx, entity,
|
|
321
|
+
for (const indexes of chunkWithinLimits(meta, rows, group, this.dialect.maxBindValues, this.dialect.maxInsertRows)) {
|
|
322
|
+
const statementRows = indexes.map((index) => rows[index]);
|
|
323
|
+
const { ids = [] } = await this.exec((ctx) => this.dialect.insert(ctx, entity, statementRows));
|
|
331
324
|
if (idsReliable) {
|
|
332
325
|
for (let position = 0; position < indexes.length; position++) {
|
|
333
326
|
rows[indexes[position]][idKey] ??= ids[position];
|
|
@@ -335,27 +328,30 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
335
328
|
}
|
|
336
329
|
}
|
|
337
330
|
}
|
|
338
|
-
await this.insertRelations(entity, rows);
|
|
339
331
|
}
|
|
340
332
|
async internalUpdateMany(entity, q, payload, opts) {
|
|
341
333
|
const { changes = 0 } = await this.exec((ctx) => this.dialect.update(ctx, entity, q, payload, opts));
|
|
342
334
|
return changes;
|
|
343
335
|
}
|
|
344
|
-
async internalUpsertOne(entity, conflictPaths, payload,
|
|
345
|
-
return this.internalUpsertMany(entity, conflictPaths, [payload],
|
|
336
|
+
async internalUpsertOne(entity, conflictPaths, payload, update) {
|
|
337
|
+
return this.internalUpsertMany(entity, conflictPaths, [payload], update);
|
|
346
338
|
}
|
|
347
|
-
async internalUpsertMany(entity, conflictPaths, payload,
|
|
339
|
+
async internalUpsertMany(entity, conflictPaths, payload, update) {
|
|
348
340
|
if (!payload?.length) {
|
|
349
341
|
return { changes: 0 };
|
|
350
342
|
}
|
|
351
343
|
payload = clone(payload);
|
|
352
344
|
const meta = getMeta(entity);
|
|
353
|
-
// One statement per shape, each split again to stay inside the bind budget
|
|
354
|
-
// what makes
|
|
355
|
-
//
|
|
356
|
-
const statements = groupByInsertShape(meta, payload).flatMap((group) =>
|
|
345
|
+
// One statement per shape, each split again to stay inside the bind budget, less what its assignments bind
|
|
346
|
+
// once. Grouping first is what makes that one figure: every row of a group carries the same columns, so
|
|
347
|
+
// they assign the same. No row cap: SQL Server upserts through a MERGE, whose source takes any number.
|
|
348
|
+
const statements = groupByInsertShape(meta, payload).flatMap((group) => {
|
|
349
|
+
const rows = group.map((index) => payload[index]);
|
|
350
|
+
const assigned = this.dialect.upsertAssignmentBinds(entity, conflictPaths, rows, update);
|
|
351
|
+
return chunkWithinLimits(meta, payload, group, this.dialect.maxBindValues - assigned);
|
|
352
|
+
});
|
|
357
353
|
if (statements.length === 1) {
|
|
358
|
-
return this.runUpsert(entity, conflictPaths, payload,
|
|
354
|
+
return this.runUpsert(entity, conflictPaths, payload, update);
|
|
359
355
|
}
|
|
360
356
|
// An upsert's assignment list is the statement's, so rows of different shapes go in statements
|
|
361
357
|
// of their own, together in a transaction (`transaction` is re-entrant).
|
|
@@ -365,7 +361,7 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
365
361
|
// than it wrote places none.
|
|
366
362
|
const ids = new Array(payload.length);
|
|
367
363
|
for (const indexes of statements) {
|
|
368
|
-
const { changes: written = 0, ids: reported } = await this.runUpsert(entity, conflictPaths, indexes.map((index) => payload[index]),
|
|
364
|
+
const { changes: written = 0, ids: reported } = await this.runUpsert(entity, conflictPaths, indexes.map((index) => payload[index]), update);
|
|
369
365
|
changes += written;
|
|
370
366
|
if (reported?.length === indexes.length) {
|
|
371
367
|
for (let position = 0; position < indexes.length; position++) {
|
|
@@ -377,11 +373,11 @@ export class AbstractSqlQuerier extends AbstractQuerier {
|
|
|
377
373
|
return { changes, ids };
|
|
378
374
|
});
|
|
379
375
|
}
|
|
380
|
-
async runUpsert(entity, conflictPaths, payload,
|
|
376
|
+
async runUpsert(entity, conflictPaths, payload, update) {
|
|
381
377
|
const meta = getMeta(entity);
|
|
382
378
|
// Asked first: the statement fills an `onInsert` key into these rows whether it inserts them or not.
|
|
383
379
|
const unnamed = meta.ids.length === 1 && payload.some((row) => !namesKey(meta, row));
|
|
384
|
-
const result = await this.exec((ctx) => this.dialect.upsert(ctx, entity, conflictPaths, payload,
|
|
380
|
+
const result = await this.exec((ctx) => this.dialect.upsert(ctx, entity, conflictPaths, payload, update));
|
|
385
381
|
const ordered = payload.length === 1 ||
|
|
386
382
|
(this.dialect.insertIdSource === 'returning' && this.dialect.features.orderedUpsertReturning);
|
|
387
383
|
if (ordered && result.ids?.length === payload.length) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Runs one statement of the cursor protocol on the querier's own connection, which holds the cursor, and
|
|
3
|
-
* not through `all()`,
|
|
3
|
+
* not through `all()`, which refuses a statement while a stream holds the querier.
|
|
4
4
|
*/
|
|
5
5
|
type CursorExecutor<T> = (query: string, values?: unknown[]) => Promise<T[]>;
|
|
6
6
|
/**
|
|
@@ -24,5 +24,8 @@ export declare class SqlExpression {
|
|
|
24
24
|
* for the column it fills; other `raw` becomes the SQL `compile` renders, parenthesized; a literal is kept.
|
|
25
25
|
*/
|
|
26
26
|
export declare function schemaDefault(value: unknown, compile: (sql: QueryRaw) => string): unknown;
|
|
27
|
-
/**
|
|
27
|
+
/**
|
|
28
|
+
* A default as text for an exact comparison: numbers as their digits, and objects (SQL included) and text
|
|
29
|
+
* holding a JSON document as JSON with its keys in order, since a `jsonb` column reprints what it stores.
|
|
30
|
+
*/
|
|
28
31
|
export declare function writtenDefault(value: unknown): string;
|