turbine-orm 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -1
- package/dist/cjs/cli/index.js +83 -0
- package/dist/cjs/client.js +14 -0
- package/dist/cjs/dialect.js +24 -0
- package/dist/cjs/index-advisor.js +0 -0
- package/dist/cjs/mssql.js +3 -1
- package/dist/cjs/query/batched-loader.js +392 -0
- package/dist/cjs/query/builder.js +548 -60
- package/dist/cjs/query/utils.js +36 -0
- package/dist/cjs/serverless.js +35 -3
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +83 -0
- package/dist/client.d.ts +40 -1
- package/dist/client.js +14 -0
- package/dist/dialect.d.ts +11 -0
- package/dist/dialect.js +24 -0
- package/dist/index-advisor.d.ts +83 -0
- package/dist/index-advisor.js +0 -0
- package/dist/mssql.js +3 -1
- package/dist/query/batched-loader.d.ts +120 -0
- package/dist/query/batched-loader.js +386 -0
- package/dist/query/builder.d.ts +118 -1
- package/dist/query/builder.js +549 -61
- package/dist/query/index.d.ts +1 -1
- package/dist/query/types.d.ts +37 -2
- package/dist/query/utils.d.ts +16 -0
- package/dist/query/utils.js +35 -0
- package/dist/serverless.d.ts +32 -4
- package/dist/serverless.js +35 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -344,6 +344,30 @@ const db = turbine({
|
|
|
344
344
|
});
|
|
345
345
|
```
|
|
346
346
|
|
|
347
|
+
### Relation loading and wire encoding
|
|
348
|
+
|
|
349
|
+
A few client options tune how `with` relations are loaded and encoded. All are optional and default to today's behavior.
|
|
350
|
+
|
|
351
|
+
```typescript
|
|
352
|
+
const db = turbine({
|
|
353
|
+
connectionString: process.env.DATABASE_URL,
|
|
354
|
+
// How with-clause relations resolve: 'join' (default, one correlated-subquery
|
|
355
|
+
// statement) or 'batched' (base query + one flat follow-up per relation).
|
|
356
|
+
// Override per query on findMany/findFirst/findUnique. See Relations.
|
|
357
|
+
relationLoadStrategy: 'join',
|
|
358
|
+
// 'positional' (Postgres-only) drops repeated JSON keys from relation
|
|
359
|
+
// subqueries — ~39% fewer wire bytes on wide relations, byte-identical output.
|
|
360
|
+
// Default 'object'.
|
|
361
|
+
jsonEncoding: 'object',
|
|
362
|
+
// Parse `timestamp` (without time zone) as UTC — the Prisma/Rails/Django
|
|
363
|
+
// convention — so results don't shift with the server's local zone.
|
|
364
|
+
// Default true; set false for the legacy local-time interpretation.
|
|
365
|
+
utcTimestamps: true,
|
|
366
|
+
});
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
Run `npx turbine doctor` to catch relations whose child-side FK lacks a covering index — the correlated-subquery strategy probes the child once per parent row, so a missing FK index costs a full scan per parent.
|
|
370
|
+
|
|
347
371
|
### Middleware
|
|
348
372
|
|
|
349
373
|
Middleware wraps every query. It runs **after SQL generation**, so it can observe what's about to execute (`params.model`, `params.action`, `params.args`), measure timing, and transform the result returned by `next()` — but it cannot change the query itself.
|
|
@@ -579,6 +603,7 @@ Commands:
|
|
|
579
603
|
migrate status Show applied/pending migrations
|
|
580
604
|
seed Run seed file
|
|
581
605
|
status Show database schema summary
|
|
606
|
+
doctor Check relations for missing FK indexes (--fix emits migration)
|
|
582
607
|
studio Launch local read-only Studio web UI
|
|
583
608
|
observe Launch local metrics dashboard (requires TURBINE_OBSERVE_URL)
|
|
584
609
|
|
|
@@ -890,7 +915,7 @@ Turbine maps Postgres types to TypeScript:
|
|
|
890
915
|
| `int8` / `bigint` | `number` | Values > `Number.MAX_SAFE_INTEGER` (2^53 - 1) are returned as `string` at runtime to avoid precision loss. This affects < 0.01% of use cases (auto-increment IDs, counts, etc. are all safe). |
|
|
891
916
|
| `numeric`, `money` | `string` | Arbitrary precision — kept as string to avoid JS float issues |
|
|
892
917
|
| `text`, `varchar`, `uuid`, `citext` | `string` | |
|
|
893
|
-
| `timestamptz`, `timestamp`, `date` | `Date` | |
|
|
918
|
+
| `timestamptz`, `timestamp`, `date` | `Date` | `timestamp` (without time zone) is parsed as UTC by default (Prisma/Rails/Django convention), so the same row yields the same instant in every region. Opt out with `utcTimestamps: false`. |
|
|
894
919
|
| `boolean` | `boolean` | |
|
|
895
920
|
| `json`, `jsonb` | `unknown` | |
|
|
896
921
|
| `bytea` | `Buffer` | |
|
package/dist/cjs/cli/index.js
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* turbine migrate status — Show migration status
|
|
14
14
|
* turbine seed — Run seed file
|
|
15
15
|
* turbine status — Show schema summary
|
|
16
|
+
* turbine doctor — Check relations for missing FK indexes (--fix emits migration)
|
|
16
17
|
* turbine studio — Launch local read-only web UI
|
|
17
18
|
* turbine observe — Launch metrics dashboard (requires TURBINE_OBSERVE_URL)
|
|
18
19
|
*
|
|
@@ -59,6 +60,7 @@ const node_fs_1 = require("node:fs");
|
|
|
59
60
|
const node_path_1 = require("node:path");
|
|
60
61
|
const node_url_1 = require("node:url");
|
|
61
62
|
const generate_js_1 = require("../generate.js");
|
|
63
|
+
const index_advisor_js_1 = require("../index-advisor.js");
|
|
62
64
|
const introspect_js_1 = require("../introspect.js");
|
|
63
65
|
const schema_sql_js_1 = require("../schema-sql.js");
|
|
64
66
|
const config_js_1 = require("./config.js");
|
|
@@ -123,6 +125,9 @@ function parseArgs() {
|
|
|
123
125
|
case '--allow-empty':
|
|
124
126
|
result.allowEmpty = true;
|
|
125
127
|
break;
|
|
128
|
+
case '--fix':
|
|
129
|
+
result.fix = true;
|
|
130
|
+
break;
|
|
126
131
|
case '--force':
|
|
127
132
|
case '-f':
|
|
128
133
|
result.force = true;
|
|
@@ -1009,6 +1014,80 @@ async function cmdStatus(_args, config) {
|
|
|
1009
1014
|
}
|
|
1010
1015
|
}
|
|
1011
1016
|
// ---------------------------------------------------------------------------
|
|
1017
|
+
// Command: doctor — relation/index health check
|
|
1018
|
+
// ---------------------------------------------------------------------------
|
|
1019
|
+
async function cmdDoctor(args, config) {
|
|
1020
|
+
(0, ui_js_1.banner)();
|
|
1021
|
+
const url = requireUrl(config);
|
|
1022
|
+
(0, ui_js_1.label)('Database', (0, ui_js_1.redactUrl)(url));
|
|
1023
|
+
(0, ui_js_1.label)('Schema', config.schema);
|
|
1024
|
+
(0, ui_js_1.newline)();
|
|
1025
|
+
const spinner = new ui_js_1.Spinner('Introspecting database').start();
|
|
1026
|
+
const schema = await (0, introspect_js_1.introspect)({
|
|
1027
|
+
connectionString: url,
|
|
1028
|
+
schema: config.schema,
|
|
1029
|
+
include: config.include.length ? config.include : undefined,
|
|
1030
|
+
exclude: config.exclude.length ? config.exclude : undefined,
|
|
1031
|
+
});
|
|
1032
|
+
const missing = (0, index_advisor_js_1.findMissingRelationIndexes)(schema);
|
|
1033
|
+
if (missing.length === 0) {
|
|
1034
|
+
spinner.succeed('Every relation probe is backed by an index');
|
|
1035
|
+
(0, ui_js_1.newline)();
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
spinner.succeed(`Scanned ${(0, ui_js_1.bold)(String(Object.keys(schema.tables).length))} tables`);
|
|
1039
|
+
(0, ui_js_1.warn)(`Found ${(0, ui_js_1.bold)(String(missing.length))} unindexed relation probe(s)`);
|
|
1040
|
+
(0, ui_js_1.newline)();
|
|
1041
|
+
// Row counts put the findings in severity order: a missing index on a 300-row
|
|
1042
|
+
// table is noise; on a 300K-row table it is the whole page load.
|
|
1043
|
+
const rowCounts = new Map();
|
|
1044
|
+
{
|
|
1045
|
+
const { Pool } = (await Promise.resolve().then(() => __importStar(require('pg')))).default;
|
|
1046
|
+
const pool = new Pool({ connectionString: url, max: 1 });
|
|
1047
|
+
try {
|
|
1048
|
+
const tables = [...new Set(missing.map((m) => m.table))];
|
|
1049
|
+
const res = await pool.query(`SELECT c.relname, c.reltuples::bigint::text AS reltuples
|
|
1050
|
+
FROM pg_class c
|
|
1051
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
1052
|
+
WHERE n.nspname = $1 AND c.relname = ANY($2)`, [config.schema, tables]);
|
|
1053
|
+
for (const row of res.rows)
|
|
1054
|
+
rowCounts.set(row.relname, Math.max(0, Number(row.reltuples)));
|
|
1055
|
+
}
|
|
1056
|
+
finally {
|
|
1057
|
+
await pool.end();
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
missing.sort((a, b) => (rowCounts.get(b.table) ?? 0) - (rowCounts.get(a.table) ?? 0));
|
|
1061
|
+
console.log(` ${(0, ui_js_1.dim)('Turbine loads relations as correlated subqueries — the child table is probed')}`);
|
|
1062
|
+
console.log(` ${(0, ui_js_1.dim)('once per parent row, so an unindexed FK costs a full table scan PER PARENT.')}`);
|
|
1063
|
+
(0, ui_js_1.newline)();
|
|
1064
|
+
for (const m of missing) {
|
|
1065
|
+
const rows = rowCounts.get(m.table);
|
|
1066
|
+
const rowsLabel = rows !== undefined ? `~${rows.toLocaleString()} rows` : 'row count unknown';
|
|
1067
|
+
console.log(` ${(0, ui_js_1.yellow)(ui_js_1.symbols.warning)} ${(0, ui_js_1.bold)((0, ui_js_1.cyan)(m.table))} ${(0, ui_js_1.dim)(`(${m.columns.join(', ')})`)} ${(0, ui_js_1.gray)(rowsLabel)}`);
|
|
1068
|
+
for (const p of m.probes) {
|
|
1069
|
+
console.log(` ${(0, ui_js_1.dim)(ui_js_1.symbols.tee)} probed by ${p.from}.${(0, ui_js_1.blue)(p.relation)} ${(0, ui_js_1.dim)(`(${p.type})`)}`);
|
|
1070
|
+
}
|
|
1071
|
+
console.log(` ${(0, ui_js_1.dim)(ui_js_1.symbols.teeEnd)} ${(0, ui_js_1.green)(m.createSql)}`);
|
|
1072
|
+
(0, ui_js_1.newline)();
|
|
1073
|
+
}
|
|
1074
|
+
if (args.fix) {
|
|
1075
|
+
const up = missing.map((m) => m.createSql).join('\n');
|
|
1076
|
+
const down = missing.map((m) => m.dropSql).join('\n');
|
|
1077
|
+
const file = (0, migrate_js_1.createMigration)(config.migrationsDir, 'add_relation_fk_indexes', { up, down });
|
|
1078
|
+
(0, ui_js_1.success)(`Created migration: ${(0, ui_js_1.bold)(file.filename)}`);
|
|
1079
|
+
(0, ui_js_1.newline)();
|
|
1080
|
+
console.log(` ${(0, ui_js_1.dim)('Review it, then apply with:')} ${(0, ui_js_1.cyan)('npx turbine migrate up')}`);
|
|
1081
|
+
console.log(` ${(0, ui_js_1.dim)('Large, hot tables: consider running the statements manually with')} ${(0, ui_js_1.cyan)('CREATE INDEX CONCURRENTLY')}`);
|
|
1082
|
+
console.log(` ${(0, ui_js_1.dim)('(cannot run inside a transaction, so it is not emitted in the migration).')}`);
|
|
1083
|
+
(0, ui_js_1.newline)();
|
|
1084
|
+
}
|
|
1085
|
+
else {
|
|
1086
|
+
console.log(` ${(0, ui_js_1.dim)('Generate a fix migration with:')} ${(0, ui_js_1.cyan)('npx turbine doctor --fix')}`);
|
|
1087
|
+
(0, ui_js_1.newline)();
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
// ---------------------------------------------------------------------------
|
|
1012
1091
|
// Command: studio — local read-only web UI
|
|
1013
1092
|
// ---------------------------------------------------------------------------
|
|
1014
1093
|
async function cmdStudio(args, config) {
|
|
@@ -1287,6 +1366,7 @@ function showHelp() {
|
|
|
1287
1366
|
console.log(` ${(0, ui_js_1.dim)('status')} Show applied/pending migrations`);
|
|
1288
1367
|
console.log(` ${(0, ui_js_1.cyan)('seed')} Run seed file`);
|
|
1289
1368
|
console.log(` ${(0, ui_js_1.cyan)('status')} ${(0, ui_js_1.dim)('| info')} Show schema summary`);
|
|
1369
|
+
console.log(` ${(0, ui_js_1.cyan)('doctor')} Check relations for missing FK indexes ${(0, ui_js_1.dim)('(--fix emits migration)')}`);
|
|
1290
1370
|
console.log(` ${(0, ui_js_1.cyan)('studio')} Launch local read-only web UI`);
|
|
1291
1371
|
console.log(` ${(0, ui_js_1.cyan)('observe')} Launch metrics dashboard ${(0, ui_js_1.dim)('(requires TURBINE_OBSERVE_URL)')}`);
|
|
1292
1372
|
(0, ui_js_1.newline)();
|
|
@@ -1439,6 +1519,9 @@ async function main() {
|
|
|
1439
1519
|
case 'info':
|
|
1440
1520
|
await cmdStatus(args, config);
|
|
1441
1521
|
break;
|
|
1522
|
+
case 'doctor':
|
|
1523
|
+
await cmdDoctor(args, config);
|
|
1524
|
+
break;
|
|
1442
1525
|
case 'studio':
|
|
1443
1526
|
await cmdStudio(args, config);
|
|
1444
1527
|
break;
|
package/dist/cjs/client.js
CHANGED
|
@@ -205,6 +205,7 @@ class TurbineClient {
|
|
|
205
205
|
/** The schema metadata this client was built from */
|
|
206
206
|
schema;
|
|
207
207
|
static int8ParserRegistered = false;
|
|
208
|
+
static utcTimestampParserRegistered = false;
|
|
208
209
|
logging;
|
|
209
210
|
/** Active SQL dialect — owns transaction keywords, set_config, raw-SQL placeholders, capability flags. */
|
|
210
211
|
dialect;
|
|
@@ -252,6 +253,16 @@ class TurbineClient {
|
|
|
252
253
|
});
|
|
253
254
|
TurbineClient.int8ParserRegistered = true;
|
|
254
255
|
}
|
|
256
|
+
// Parse `timestamp` (OID 1114) as UTC instead of server-local time. The
|
|
257
|
+
// pg driver's default hands back a Date built in the process's local zone,
|
|
258
|
+
// so the same row yields a different instant per deployment region. The
|
|
259
|
+
// ORM convention (Prisma, Rails, Django) — and the only interpretation
|
|
260
|
+
// that round-trips what Postgres stores — is UTC. Same ownership rule as
|
|
261
|
+
// the int8 parser: never mutate parser state on external pools.
|
|
262
|
+
if (!config.pool && config.utcTimestamps !== false && !TurbineClient.utcTimestampParserRegistered) {
|
|
263
|
+
pg_1.default.types.setTypeParser(1114, (val) => new Date(`${val.replace(' ', 'T')}Z`));
|
|
264
|
+
TurbineClient.utcTimestampParserRegistered = true;
|
|
265
|
+
}
|
|
255
266
|
this.logging = config.logging ?? false;
|
|
256
267
|
this.dialect = config.dialect ?? dialect_js_1.postgresDialect;
|
|
257
268
|
this.schema = schema;
|
|
@@ -261,6 +272,9 @@ class TurbineClient {
|
|
|
261
272
|
this.queryOptions = {
|
|
262
273
|
defaultLimit: config.defaultLimit,
|
|
263
274
|
warnOnUnlimited: config.warnOnUnlimited,
|
|
275
|
+
utcTimestamps: config.utcTimestamps,
|
|
276
|
+
relationLoadStrategy: config.relationLoadStrategy,
|
|
277
|
+
jsonEncoding: config.jsonEncoding,
|
|
264
278
|
preparedStatements: envDisablePrepared ? false : (config.preparedStatements ?? !config.pool),
|
|
265
279
|
sqlCache: config.sqlCache ?? true,
|
|
266
280
|
dialect: config.dialect,
|
package/dist/cjs/dialect.js
CHANGED
|
@@ -68,8 +68,32 @@ exports.postgresDialect = {
|
|
|
68
68
|
},
|
|
69
69
|
buildJsonObject(pairs) {
|
|
70
70
|
const args = pairs.map(([key, expr]) => `'${this.escapeStringLiteral(key)}', ${expr}`);
|
|
71
|
+
// Postgres caps function calls at 100 arguments (= 50 key/value pairs).
|
|
72
|
+
// Wide tables (or wide select+relation trees) exceed that, so chunk into
|
|
73
|
+
// multiple jsonb_build_object calls merged with `||`, cast back to json.
|
|
74
|
+
if (pairs.length > 50) {
|
|
75
|
+
const chunks = [];
|
|
76
|
+
for (let i = 0; i < args.length; i += 50) {
|
|
77
|
+
chunks.push(`jsonb_build_object(${args.slice(i, i + 50).join(', ')})`);
|
|
78
|
+
}
|
|
79
|
+
return `(${chunks.join(' || ')})::json`;
|
|
80
|
+
}
|
|
71
81
|
return `json_build_object(${args.join(', ')})`;
|
|
72
82
|
},
|
|
83
|
+
buildJsonArray(exprs) {
|
|
84
|
+
// Mirror buildJsonObject's chunking at the SAME 50-element threshold: for
|
|
85
|
+
// wide rows, concatenate 50-element jsonb_build_array calls with `||` (which
|
|
86
|
+
// concatenates jsonb arrays) and cast back to json. `jsonb ||` preserves
|
|
87
|
+
// element order, so positions map back to keys unchanged after decode.
|
|
88
|
+
if (exprs.length > 50) {
|
|
89
|
+
const chunks = [];
|
|
90
|
+
for (let i = 0; i < exprs.length; i += 50) {
|
|
91
|
+
chunks.push(`jsonb_build_array(${exprs.slice(i, i + 50).join(', ')})`);
|
|
92
|
+
}
|
|
93
|
+
return `(${chunks.join(' || ')})::json`;
|
|
94
|
+
}
|
|
95
|
+
return `json_build_array(${exprs.join(', ')})`;
|
|
96
|
+
},
|
|
73
97
|
buildJsonArrayAgg(jsonObjectExpr, orderBy) {
|
|
74
98
|
const suffix = orderBy ? ` ${orderBy}` : '';
|
|
75
99
|
return `COALESCE(json_agg(${jsonObjectExpr}${suffix}), ${this.emptyJsonArrayLiteral})`;
|
|
Binary file
|
package/dist/cjs/mssql.js
CHANGED
|
@@ -807,7 +807,9 @@ function buildForJsonSubquery(dialect, ctx) {
|
|
|
807
807
|
return buildForJsonManyToMany(dialect, ctx, { colSelect, buildNested, buildPaging, hasLimit });
|
|
808
808
|
}
|
|
809
809
|
const isToOne = relDef.type === 'belongsTo' || relDef.type === 'hasOne';
|
|
810
|
-
|
|
810
|
+
// Correlation direction is about WHERE THE FK LIVES, not cardinality:
|
|
811
|
+
// belongsTo has it on the source; hasMany AND hasOne have it on the target.
|
|
812
|
+
const correlation = relDef.type === 'belongsTo'
|
|
811
813
|
? dialect.buildCorrelation(alias, relDef.referenceKey, qParent, relDef.foreignKey)
|
|
812
814
|
: dialect.buildCorrelation(alias, relDef.foreignKey, qParent, relDef.referenceKey);
|
|
813
815
|
// ----- to-one (belongsTo / hasOne): single object, no paging --------------
|
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* turbine-orm — Batched relation loader (the `relationLoadStrategy: 'batched'` path)
|
|
4
|
+
*
|
|
5
|
+
* ## Why this exists
|
|
6
|
+
*
|
|
7
|
+
* Turbine's default `with`-clause strategy resolves nested relations in ONE SQL
|
|
8
|
+
* statement using correlated `json_agg(json_build_object(...))` subqueries — one
|
|
9
|
+
* probe per parent row (see `buildRelationSubquery` in builder.ts). That is the
|
|
10
|
+
* right default: a single round-trip, and when the child FK columns are indexed
|
|
11
|
+
* each probe is an index seek. But it degrades in two situations:
|
|
12
|
+
*
|
|
13
|
+
* 1. **Missing FK index** — a correlated probe per parent row becomes
|
|
14
|
+
* N-parents × full-table-scan. A batched-loader ORM pays that missing index
|
|
15
|
+
* only ONCE (a single `WHERE fk = ANY($1)` seq-scan), which is why schemas
|
|
16
|
+
* migrated from those ORMs often lack the index the json_agg path needs.
|
|
17
|
+
* 2. **Huge unpaginated result sets** — the JSON wire format
|
|
18
|
+
* (`json_build_object` per row, re-serialized inside `json_agg`) is heavy to
|
|
19
|
+
* encode/decode compared with flat rows.
|
|
20
|
+
*
|
|
21
|
+
* This module implements the alternative, opt-in strategy: run the base query
|
|
22
|
+
* WITHOUT relation subqueries, collect the parent keys, then issue ONE flat
|
|
23
|
+
* follow-up query per relation (`SELECT ... FROM child WHERE fk = ANY($1)`),
|
|
24
|
+
* and stitch the children onto the parents in memory. D relation levels cost D
|
|
25
|
+
* extra round-trips instead of one, but each is a single indexed lookup over a
|
|
26
|
+
* key set, and rows come back flat.
|
|
27
|
+
*
|
|
28
|
+
* ## Design constraints (see CLAUDE.md)
|
|
29
|
+
*
|
|
30
|
+
* - **Same executor / connection path.** Every follow-up query runs through the
|
|
31
|
+
* caller's own executor ({@link RelationLoadContext.exec}) and child query
|
|
32
|
+
* interfaces built on the caller's pool. Inside a `$transaction` that pool is
|
|
33
|
+
* the pinned-connection `txPool`, so batched loads join the transaction — no
|
|
34
|
+
* separate pool checkout per query.
|
|
35
|
+
* - **Identical output shape.** The stitched result is byte-for-byte the same
|
|
36
|
+
* shape the join strategy produces: relation arrays for hasMany/manyToMany
|
|
37
|
+
* (`[]` when empty), single-or-null for hasOne/belongsTo, with the same
|
|
38
|
+
* camelCase keys and Date coercion — because the child rows are parsed by the
|
|
39
|
+
* very same `parseRow`/`buildFindMany` machinery via a child QueryInterface.
|
|
40
|
+
* - **Stitch keys never leak.** To stitch, the follow-up query must select the
|
|
41
|
+
* FK/PK it joins on even when the caller's `select`/`omit` excluded it; the
|
|
42
|
+
* loader adds those columns for the query and strips them from the returned
|
|
43
|
+
* entities afterwards ({@link includeKeysForBatching}).
|
|
44
|
+
*
|
|
45
|
+
* PowDB (powql.ts) has its own batched loaders for the same reasons — this is the
|
|
46
|
+
* clean Postgres/SQL implementation, deliberately NOT shared with PowQL.
|
|
47
|
+
*
|
|
48
|
+
* @module
|
|
49
|
+
*/
|
|
50
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
51
|
+
exports.includeKeysForBatching = includeKeysForBatching;
|
|
52
|
+
exports.stripFields = stripFields;
|
|
53
|
+
exports.neededParentKeyFields = neededParentKeyFields;
|
|
54
|
+
exports.loadRelationsBatched = loadRelationsBatched;
|
|
55
|
+
const errors_js_1 = require("../errors.js");
|
|
56
|
+
const schema_js_1 = require("../schema.js");
|
|
57
|
+
/**
|
|
58
|
+
* Max parent keys per follow-up query. On Postgres the whole key set travels as
|
|
59
|
+
* ONE array parameter (`= ANY($1)`), so this is not a bind-parameter limit — it
|
|
60
|
+
* only bounds planner/memory cost per statement. Keep it large: every extra
|
|
61
|
+
* chunk is an extra network round-trip, and round-trips are exactly what the
|
|
62
|
+
* batched strategy exists to minimize (a 9-chunk load was measured 2× slower
|
|
63
|
+
* than a single-statement one over a WAN link).
|
|
64
|
+
*/
|
|
65
|
+
const MAX_RELATION_KEYS = 32_000;
|
|
66
|
+
/** Nesting cap — parity with the join strategy's depth-10 guard. */
|
|
67
|
+
const MAX_DEPTH = 10;
|
|
68
|
+
/**
|
|
69
|
+
* Adjust a `select`/`omit` pair so that `fields` are guaranteed present in the
|
|
70
|
+
* query result, returning the adjusted projection plus the list of fields that
|
|
71
|
+
* were added ONLY for stitching and must be stripped from the final entities.
|
|
72
|
+
*
|
|
73
|
+
* Used both for the base query (parent keys) and each follow-up query (child
|
|
74
|
+
* keys) so a caller's `select: { title: true }` on a relation still stitches even
|
|
75
|
+
* though the FK was not requested — and the FK never appears in the output.
|
|
76
|
+
*/
|
|
77
|
+
function includeKeysForBatching(select, omit, fields) {
|
|
78
|
+
const unique = [...new Set(fields)];
|
|
79
|
+
if (select) {
|
|
80
|
+
const next = { ...select };
|
|
81
|
+
const strip = [];
|
|
82
|
+
for (const f of unique) {
|
|
83
|
+
if (!next[f]) {
|
|
84
|
+
next[f] = true;
|
|
85
|
+
strip.push(f); // not requested by the caller — added only to stitch
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return { select: next, omit, strip };
|
|
89
|
+
}
|
|
90
|
+
if (omit) {
|
|
91
|
+
const next = { ...omit };
|
|
92
|
+
const strip = [];
|
|
93
|
+
for (const f of unique) {
|
|
94
|
+
if (next[f]) {
|
|
95
|
+
delete next[f]; // un-omit so the key is present; the caller wanted it gone
|
|
96
|
+
strip.push(f);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return { select, omit: next, strip };
|
|
100
|
+
}
|
|
101
|
+
// Neither select nor omit — every column is already present; nothing to strip.
|
|
102
|
+
return { select, omit, strip: [] };
|
|
103
|
+
}
|
|
104
|
+
/** Delete stitch-only key fields from each row (no-op when `fields` is empty). */
|
|
105
|
+
function stripFields(rows, fields) {
|
|
106
|
+
if (fields.length === 0)
|
|
107
|
+
return;
|
|
108
|
+
for (const row of rows) {
|
|
109
|
+
for (const f of fields)
|
|
110
|
+
delete row[f];
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* The set of parent FIELD names a batched load of `withClause` needs present on
|
|
115
|
+
* each parent row in order to stitch (the local key of every requested relation).
|
|
116
|
+
* The caller adds these to the base query and strips the added ones afterwards.
|
|
117
|
+
*/
|
|
118
|
+
function neededParentKeyFields(parentMeta, withClause) {
|
|
119
|
+
const fields = new Set();
|
|
120
|
+
for (const [relName, spec] of Object.entries(withClause)) {
|
|
121
|
+
if (!spec)
|
|
122
|
+
continue;
|
|
123
|
+
const rel = parentMeta.relations[relName];
|
|
124
|
+
if (!rel)
|
|
125
|
+
continue; // unknown relation — the join path throws; let the loader surface it
|
|
126
|
+
for (const col of localKeyColumns(rel)) {
|
|
127
|
+
fields.add(parentMeta.reverseColumnMap[col] ?? col);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return [...fields];
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* The parent-side key column(s) used to correlate a relation:
|
|
134
|
+
* - hasMany / hasOne: the parent's `referenceKey` (child's FK points at it)
|
|
135
|
+
* - belongsTo: the parent's `foreignKey` (points at the child's PK)
|
|
136
|
+
* - manyToMany: the parent's `referenceKey` (junction's sourceKey → it)
|
|
137
|
+
*/
|
|
138
|
+
function localKeyColumns(rel) {
|
|
139
|
+
if (rel.type === 'belongsTo')
|
|
140
|
+
return (0, schema_js_1.normalizeKeyColumns)(rel.foreignKey);
|
|
141
|
+
return (0, schema_js_1.normalizeKeyColumns)(rel.referenceKey);
|
|
142
|
+
}
|
|
143
|
+
/** Stringified stitch key — robust to number/uuid/bigint type drift across a join. */
|
|
144
|
+
function keyOf(value) {
|
|
145
|
+
return String(value);
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Load every relation in `withClause` for `parents` and attach it onto each row
|
|
149
|
+
* in place. Mirrors the join strategy's output shape exactly. Recurses for nested
|
|
150
|
+
* `with` by re-running itself against the freshly-loaded child rows.
|
|
151
|
+
*/
|
|
152
|
+
async function loadRelationsBatched(ctx, parents, withClause, timeout, depth = 0, path = [ctx.parentMeta.name]) {
|
|
153
|
+
if (depth >= MAX_DEPTH)
|
|
154
|
+
throw new errors_js_1.CircularRelationError([...path, '…']);
|
|
155
|
+
if (parents.length === 0)
|
|
156
|
+
return;
|
|
157
|
+
// Sibling relations are independent (each writes only its own parent[relName]
|
|
158
|
+
// and reads only parent keys), so load them concurrently — on a pool that's
|
|
159
|
+
// real parallelism, inside a transaction pg queues them on the one connection.
|
|
160
|
+
const loads = [];
|
|
161
|
+
for (const [relName, spec] of Object.entries(withClause)) {
|
|
162
|
+
if (!spec)
|
|
163
|
+
continue;
|
|
164
|
+
const rel = ctx.parentMeta.relations[relName];
|
|
165
|
+
if (!rel) {
|
|
166
|
+
throw new errors_js_1.ValidationError(`[turbine] Unknown relation "${relName}" on table "${ctx.parentMeta.name}". ` +
|
|
167
|
+
`Available: ${Object.keys(ctx.parentMeta.relations).join(', ')}`);
|
|
168
|
+
}
|
|
169
|
+
const options = spec === true ? {} : spec;
|
|
170
|
+
loads.push(rel.type === 'manyToMany'
|
|
171
|
+
? loadManyToMany(ctx, parents, rel, relName, options, timeout, depth, path)
|
|
172
|
+
: loadToOneOrMany(ctx, parents, rel, relName, options, timeout, depth, path));
|
|
173
|
+
}
|
|
174
|
+
await Promise.all(loads);
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* hasMany / hasOne / belongsTo: one follow-up `SELECT ... WHERE childKey = ANY($1)`
|
|
178
|
+
* (chunked), grouped by the correlation key and attached (array vs single-or-null).
|
|
179
|
+
*/
|
|
180
|
+
async function loadToOneOrMany(ctx, parents, rel, relName, options, timeout, depth, path) {
|
|
181
|
+
const fk = (0, schema_js_1.normalizeKeyColumns)(rel.foreignKey);
|
|
182
|
+
const rk = (0, schema_js_1.normalizeKeyColumns)(rel.referenceKey);
|
|
183
|
+
if (fk.length > 1 || rk.length > 1) {
|
|
184
|
+
throw new errors_js_1.UnsupportedFeatureError('composite-key batched relation loading', 'relationLoadStrategy: "batched"', `relation "${relName}" — use the default 'join' strategy for composite-key relations`);
|
|
185
|
+
}
|
|
186
|
+
const targetMeta = requireTable(ctx.schema, rel.to, relName);
|
|
187
|
+
// Local key lives on the parent; the correlating key lives on the child.
|
|
188
|
+
// hasMany/hasOne: parent.referenceKey ← child.foreignKey
|
|
189
|
+
// belongsTo: parent.foreignKey → child.referenceKey
|
|
190
|
+
const parentKeyCol = rel.type === 'belongsTo' ? fk[0] : rk[0];
|
|
191
|
+
const childKeyCol = rel.type === 'belongsTo' ? rk[0] : fk[0];
|
|
192
|
+
const parentKeyField = ctx.parentMeta.reverseColumnMap[parentKeyCol] ?? parentKeyCol;
|
|
193
|
+
const childKeyField = targetMeta.reverseColumnMap[childKeyCol] ?? childKeyCol;
|
|
194
|
+
const keys = uniqueKeys(parents, parentKeyField);
|
|
195
|
+
const single = rel.type === 'belongsTo' || rel.type === 'hasOne';
|
|
196
|
+
if (keys.length === 0) {
|
|
197
|
+
for (const parent of parents)
|
|
198
|
+
parent[relName] = single ? null : [];
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
// The follow-up must project the child correlation key even if the caller's
|
|
202
|
+
// select/omit excluded it; strip it back off afterwards so the shape matches join.
|
|
203
|
+
const proj = includeKeysForBatching(options.select, options.omit, [childKeyField]);
|
|
204
|
+
const child = ctx.makeChild(rel.to);
|
|
205
|
+
const chunks = [];
|
|
206
|
+
for (let i = 0; i < keys.length; i += MAX_RELATION_KEYS)
|
|
207
|
+
chunks.push(keys.slice(i, i + MAX_RELATION_KEYS));
|
|
208
|
+
// Chunks run concurrently, results concatenated in chunk order. Per-relation
|
|
209
|
+
// `limit` is NOT pushed down here: `LIMIT` on a `fk = ANY($1)` query over the
|
|
210
|
+
// whole batch would cap TOTAL children, not children-per-parent. It is applied
|
|
211
|
+
// client-side per group after stitching (below).
|
|
212
|
+
const chunkResults = await Promise.all(chunks.map(async (chunk) => {
|
|
213
|
+
const deferred = child.buildFindMany({
|
|
214
|
+
where: mergeChildWhere(options.where, childKeyField, chunk),
|
|
215
|
+
select: proj.select,
|
|
216
|
+
omit: proj.omit,
|
|
217
|
+
orderBy: options.orderBy,
|
|
218
|
+
});
|
|
219
|
+
const result = await ctx.exec(deferred.sql, deferred.params, deferred.preparedName);
|
|
220
|
+
return deferred.transform(result);
|
|
221
|
+
}));
|
|
222
|
+
const allChildren = chunkResults.flat();
|
|
223
|
+
// Recurse for nested `with` BEFORE stripping keys (children carry their own keys).
|
|
224
|
+
if (options.with && allChildren.length > 0) {
|
|
225
|
+
await loadRelationsBatched({ ...ctx, parentMeta: targetMeta }, allChildren, options.with, timeout, depth + 1, [...path, relName]);
|
|
226
|
+
}
|
|
227
|
+
const byKey = groupBy(allChildren, childKeyField);
|
|
228
|
+
const limit = options.limit;
|
|
229
|
+
for (const parent of parents) {
|
|
230
|
+
const bucket = byKey.get(keyOf(parent[parentKeyField])) ?? [];
|
|
231
|
+
if (single) {
|
|
232
|
+
parent[relName] = bucket[0] ?? null;
|
|
233
|
+
}
|
|
234
|
+
else {
|
|
235
|
+
parent[relName] = limit !== undefined ? bucket.slice(0, limit) : bucket;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
stripFields(allChildren, proj.strip);
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* manyToMany: a three-hop batched loader (no join pushdown):
|
|
242
|
+
* (1) read junction rows for all parents (`sourceKey = ANY($1)` chunks),
|
|
243
|
+
* (2) read the target rows for the collected targetKeys,
|
|
244
|
+
* (3) stitch parent → junction targetKeys → target rows in memory.
|
|
245
|
+
* Composite junction/target keys fall back to the join strategy (throw E017).
|
|
246
|
+
*/
|
|
247
|
+
async function loadManyToMany(ctx, parents, rel, relName, options, timeout, depth, path) {
|
|
248
|
+
const through = rel.through;
|
|
249
|
+
if (!through) {
|
|
250
|
+
throw new errors_js_1.ValidationError(`[turbine] manyToMany relation "${relName}" is missing its junction (\`through\`).`);
|
|
251
|
+
}
|
|
252
|
+
const sourceJ = (0, schema_js_1.normalizeKeyColumns)(through.sourceKey);
|
|
253
|
+
const targetJ = (0, schema_js_1.normalizeKeyColumns)(through.targetKey);
|
|
254
|
+
const sourceRef = (0, schema_js_1.normalizeKeyColumns)(rel.referenceKey);
|
|
255
|
+
const targetMeta = requireTable(ctx.schema, rel.to, relName);
|
|
256
|
+
if (sourceJ.length > 1 || targetJ.length > 1 || sourceRef.length > 1 || targetMeta.primaryKey.length !== 1) {
|
|
257
|
+
throw new errors_js_1.UnsupportedFeatureError('composite-key batched manyToMany loading', 'relationLoadStrategy: "batched"', `relation "${relName}" — use the default 'join' strategy for composite-key m2m relations`);
|
|
258
|
+
}
|
|
259
|
+
const sourceJCol = sourceJ[0];
|
|
260
|
+
const targetJCol = targetJ[0];
|
|
261
|
+
const sourceRefCol = sourceRef[0];
|
|
262
|
+
const targetPkCol = targetMeta.primaryKey[0];
|
|
263
|
+
const parentRefField = ctx.parentMeta.reverseColumnMap[sourceRefCol] ?? sourceRefCol;
|
|
264
|
+
const targetPkField = targetMeta.reverseColumnMap[targetPkCol] ?? targetPkCol;
|
|
265
|
+
const parentKeys = uniqueKeys(parents, parentRefField);
|
|
266
|
+
if (parentKeys.length === 0) {
|
|
267
|
+
for (const parent of parents)
|
|
268
|
+
parent[relName] = [];
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
// (1) Junction rows: sourceKeyVal → [targetKeyVal]. Raw SQL through the caller's
|
|
272
|
+
// executor (the junction table has no relations we need, so no child reader).
|
|
273
|
+
const targetsBySource = new Map();
|
|
274
|
+
const targetValSet = new Set();
|
|
275
|
+
const jTable = ctx.quote(through.table);
|
|
276
|
+
const jSource = ctx.quote(sourceJCol);
|
|
277
|
+
const jTarget = ctx.quote(targetJCol);
|
|
278
|
+
const jChunks = [];
|
|
279
|
+
for (let i = 0; i < parentKeys.length; i += MAX_RELATION_KEYS) {
|
|
280
|
+
jChunks.push(parentKeys.slice(i, i + MAX_RELATION_KEYS));
|
|
281
|
+
}
|
|
282
|
+
const jResults = await Promise.all(jChunks.map((chunk) => {
|
|
283
|
+
const params = [ctx.inClauseParam(chunk)];
|
|
284
|
+
const predicate = ctx.buildInClause(`${jTable}.${jSource}`, ctx.paramPlaceholder(1), false);
|
|
285
|
+
const sql = `SELECT ${jTable}.${jSource} AS "s", ${jTable}.${jTarget} AS "t" FROM ${jTable} WHERE ${predicate}`;
|
|
286
|
+
return ctx.exec(sql, params);
|
|
287
|
+
}));
|
|
288
|
+
for (const { rows } of jResults) {
|
|
289
|
+
for (const row of rows) {
|
|
290
|
+
const sv = keyOf(row.s);
|
|
291
|
+
const tv = row.t;
|
|
292
|
+
if (tv == null)
|
|
293
|
+
continue;
|
|
294
|
+
const bucket = targetsBySource.get(sv);
|
|
295
|
+
if (bucket)
|
|
296
|
+
bucket.push(tv);
|
|
297
|
+
else
|
|
298
|
+
targetsBySource.set(sv, [tv]);
|
|
299
|
+
targetValSet.add(tv);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
// (2) Target rows by PK, honouring the relation's own where/select/omit/orderBy.
|
|
303
|
+
const proj = includeKeysForBatching(options.select, options.omit, [targetPkField]);
|
|
304
|
+
const child = ctx.makeChild(rel.to);
|
|
305
|
+
const targetVals = [...targetValSet];
|
|
306
|
+
const tChunks = [];
|
|
307
|
+
for (let i = 0; i < targetVals.length; i += MAX_RELATION_KEYS) {
|
|
308
|
+
tChunks.push(targetVals.slice(i, i + MAX_RELATION_KEYS));
|
|
309
|
+
}
|
|
310
|
+
const tResults = await Promise.all(tChunks.map(async (chunk) => {
|
|
311
|
+
const deferred = child.buildFindMany({
|
|
312
|
+
where: mergeChildWhere(options.where, targetPkField, chunk),
|
|
313
|
+
select: proj.select,
|
|
314
|
+
omit: proj.omit,
|
|
315
|
+
orderBy: options.orderBy,
|
|
316
|
+
});
|
|
317
|
+
const result = await ctx.exec(deferred.sql, deferred.params, deferred.preparedName);
|
|
318
|
+
return deferred.transform(result);
|
|
319
|
+
}));
|
|
320
|
+
const targetsInOrder = tResults.flat();
|
|
321
|
+
// Nested `with` on the target rows (before stripping their PK).
|
|
322
|
+
if (options.with && targetsInOrder.length > 0) {
|
|
323
|
+
await loadRelationsBatched({ ...ctx, parentMeta: targetMeta }, targetsInOrder, options.with, timeout, depth + 1, [...path, relName]);
|
|
324
|
+
}
|
|
325
|
+
const targetByPk = new Map();
|
|
326
|
+
for (const t of targetsInOrder)
|
|
327
|
+
targetByPk.set(keyOf(t[targetPkField]), t);
|
|
328
|
+
// (3) Stitch. Iterate `targetsInOrder` (already ordered by the relation's
|
|
329
|
+
// orderBy) and pick the ones each parent links to, so per-parent order honours
|
|
330
|
+
// orderBy; then apply the per-relation `limit` client-side.
|
|
331
|
+
const limit = options.limit;
|
|
332
|
+
for (const parent of parents) {
|
|
333
|
+
const linked = new Set((targetsBySource.get(keyOf(parent[parentRefField])) ?? []).map(keyOf));
|
|
334
|
+
if (linked.size === 0) {
|
|
335
|
+
parent[relName] = [];
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
const out = [];
|
|
339
|
+
for (const t of targetsInOrder) {
|
|
340
|
+
if (linked.has(keyOf(t[targetPkField]))) {
|
|
341
|
+
out.push(t);
|
|
342
|
+
if (limit !== undefined && out.length >= limit)
|
|
343
|
+
break;
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
parent[relName] = out;
|
|
347
|
+
}
|
|
348
|
+
stripFields(targetsInOrder, proj.strip);
|
|
349
|
+
}
|
|
350
|
+
// ---------------------------------------------------------------------------
|
|
351
|
+
// Small helpers
|
|
352
|
+
// ---------------------------------------------------------------------------
|
|
353
|
+
/** Merge the batched correlation predicate (`key IN chunk`) into the relation's own where. */
|
|
354
|
+
function mergeChildWhere(where, keyField, chunk) {
|
|
355
|
+
return { ...(where ?? {}), [keyField]: { in: chunk } };
|
|
356
|
+
}
|
|
357
|
+
/** Distinct, non-null values of `field` across `rows`. */
|
|
358
|
+
function uniqueKeys(rows, field) {
|
|
359
|
+
const seen = new Set();
|
|
360
|
+
const out = [];
|
|
361
|
+
for (const row of rows) {
|
|
362
|
+
const v = row[field];
|
|
363
|
+
if (v == null)
|
|
364
|
+
continue;
|
|
365
|
+
const k = keyOf(v);
|
|
366
|
+
if (seen.has(k))
|
|
367
|
+
continue;
|
|
368
|
+
seen.add(k);
|
|
369
|
+
out.push(v);
|
|
370
|
+
}
|
|
371
|
+
return out;
|
|
372
|
+
}
|
|
373
|
+
/** Group rows by the stringified value of `field`, preserving input order. */
|
|
374
|
+
function groupBy(rows, field) {
|
|
375
|
+
const map = new Map();
|
|
376
|
+
for (const row of rows) {
|
|
377
|
+
const k = keyOf(row[field]);
|
|
378
|
+
const bucket = map.get(k);
|
|
379
|
+
if (bucket)
|
|
380
|
+
bucket.push(row);
|
|
381
|
+
else
|
|
382
|
+
map.set(k, [row]);
|
|
383
|
+
}
|
|
384
|
+
return map;
|
|
385
|
+
}
|
|
386
|
+
/** Resolve a table's metadata or throw a clear relation error. */
|
|
387
|
+
function requireTable(schema, table, relName) {
|
|
388
|
+
const meta = schema.tables[table];
|
|
389
|
+
if (!meta)
|
|
390
|
+
throw new errors_js_1.ValidationError(`[turbine] Relation "${relName}" targets unknown table "${table}".`);
|
|
391
|
+
return meta;
|
|
392
|
+
}
|