turbine-orm 0.25.0 → 0.27.0

package/dist/cjs/query/batched-loader.js

@@ -0,0 +1,392 @@
+"use strict";
+/**
+ * turbine-orm — Batched relation loader (the `relationLoadStrategy: 'batched'` path)
+ *
+ * ## Why this exists
+ *
+ * Turbine's default `with`-clause strategy resolves nested relations in ONE SQL
+ * statement using correlated `json_agg(json_build_object(...))` subqueries — one
+ * probe per parent row (see `buildRelationSubquery` in builder.ts). That is the
+ * right default: a single round-trip, and when the child FK columns are indexed
+ * each probe is an index seek. But it degrades in two situations:
+ *
+ *   1. **Missing FK index** — a correlated probe per parent row becomes
+ *      N-parents × full-table-scan. A batched-loader ORM pays that missing index
+ *      only ONCE (a single `WHERE fk = ANY($1)` seq-scan), which is why schemas
+ *      migrated from those ORMs often lack the index the json_agg path needs.
+ *   2. **Huge unpaginated result sets** — the JSON wire format
+ *      (`json_build_object` per row, re-serialized inside `json_agg`) is heavy to
+ *      encode/decode compared with flat rows.
+ *
+ * This module implements the alternative, opt-in strategy: run the base query
+ * WITHOUT relation subqueries, collect the parent keys, then issue ONE flat
+ * follow-up query per relation (`SELECT ... FROM child WHERE fk = ANY($1)`),
+ * and stitch the children onto the parents in memory. D relation levels cost D
+ * extra round-trips instead of one, but each is a single indexed lookup over a
+ * key set, and rows come back flat.
+ *
+ * ## Design constraints (see CLAUDE.md)
+ *
+ *   - **Same executor / connection path.** Every follow-up query runs through the
+ *     caller's own executor ({@link RelationLoadContext.exec}) and child query
+ *     interfaces built on the caller's pool. Inside a `$transaction` that pool is
+ *     the pinned-connection `txPool`, so batched loads join the transaction — no
+ *     separate pool checkout per query.
+ *   - **Identical output shape.** The stitched result is byte-for-byte the same
+ *     shape the join strategy produces: relation arrays for hasMany/manyToMany
+ *     (`[]` when empty), single-or-null for hasOne/belongsTo, with the same
+ *     camelCase keys and Date coercion — because the child rows are parsed by the
+ *     very same `parseRow`/`buildFindMany` machinery via a child QueryInterface.
+ *   - **Stitch keys never leak.** To stitch, the follow-up query must select the
+ *     FK/PK it joins on even when the caller's `select`/`omit` excluded it; the
+ *     loader adds those columns for the query and strips them from the returned
+ *     entities afterwards ({@link includeKeysForBatching}).
+ *
+ * PowDB (powql.ts) has its own batched loaders for the same reasons — this is the
+ * clean Postgres/SQL implementation, deliberately NOT shared with PowQL.
+ *
+ * @module
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.includeKeysForBatching = includeKeysForBatching;
+exports.stripFields = stripFields;
+exports.neededParentKeyFields = neededParentKeyFields;
+exports.loadRelationsBatched = loadRelationsBatched;
+const errors_js_1 = require("../errors.js");
+const schema_js_1 = require("../schema.js");
+/**
+ * Max parent keys per follow-up query. On Postgres the whole key set travels as
+ * ONE array parameter (`= ANY($1)`), so this is not a bind-parameter limit — it
+ * only bounds planner/memory cost per statement. Keep it large: every extra
+ * chunk is an extra network round-trip, and round-trips are exactly what the
+ * batched strategy exists to minimize (a 9-chunk load was measured 2× slower
+ * than a single-statement one over a WAN link).
+ */
+const MAX_RELATION_KEYS = 32_000;
+/** Nesting cap — parity with the join strategy's depth-10 guard. */
+const MAX_DEPTH = 10;
+/**
+ * Adjust a `select`/`omit` pair so that `fields` are guaranteed present in the
+ * query result, returning the adjusted projection plus the list of fields that
+ * were added ONLY for stitching and must be stripped from the final entities.
+ *
+ * Used both for the base query (parent keys) and each follow-up query (child
+ * keys) so a caller's `select: { title: true }` on a relation still stitches even
+ * though the FK was not requested — and the FK never appears in the output.
+ */
+function includeKeysForBatching(select, omit, fields) {
+    const unique = [...new Set(fields)];
+    if (select) {
+        const next = { ...select };
+        const strip = [];
+        for (const f of unique) {
+            if (!next[f]) {
+                next[f] = true;
+                strip.push(f); // not requested by the caller — added only to stitch
+            }
+        }
+        return { select: next, omit, strip };
+    }
+    if (omit) {
+        const next = { ...omit };
+        const strip = [];
+        for (const f of unique) {
+            if (next[f]) {
+                delete next[f]; // un-omit so the key is present; the caller wanted it gone
+                strip.push(f);
+            }
+        }
+        return { select, omit: next, strip };
+    }
+    // Neither select nor omit — every column is already present; nothing to strip.
+    return { select, omit, strip: [] };
+}
+/** Delete stitch-only key fields from each row (no-op when `fields` is empty). */
+function stripFields(rows, fields) {
+    if (fields.length === 0)
+        return;
+    for (const row of rows) {
+        for (const f of fields)
+            delete row[f];
+    }
+}
+/**
+ * The set of parent FIELD names a batched load of `withClause` needs present on
+ * each parent row in order to stitch (the local key of every requested relation).
+ * The caller adds these to the base query and strips the added ones afterwards.
+ */
+function neededParentKeyFields(parentMeta, withClause) {
+    const fields = new Set();
+    for (const [relName, spec] of Object.entries(withClause)) {
+        if (!spec)
+            continue;
+        const rel = parentMeta.relations[relName];
+        if (!rel)
+            continue; // unknown relation — the join path throws; let the loader surface it
+        for (const col of localKeyColumns(rel)) {
+            fields.add(parentMeta.reverseColumnMap[col] ?? col);
+        }
+    }
+    return [...fields];
+}
+/**
+ * The parent-side key column(s) used to correlate a relation:
+ *   - hasMany / hasOne:  the parent's `referenceKey` (child's FK points at it)
+ *   - belongsTo:         the parent's `foreignKey`   (points at the child's PK)
+ *   - manyToMany:        the parent's `referenceKey` (junction's sourceKey → it)
+ */
+function localKeyColumns(rel) {
+    if (rel.type === 'belongsTo')
+        return (0, schema_js_1.normalizeKeyColumns)(rel.foreignKey);
+    return (0, schema_js_1.normalizeKeyColumns)(rel.referenceKey);
+}
+/** Stringified stitch key — robust to number/uuid/bigint type drift across a join. */
+function keyOf(value) {
+    return String(value);
+}
+/**
+ * Load every relation in `withClause` for `parents` and attach it onto each row
+ * in place. Mirrors the join strategy's output shape exactly. Recurses for nested
+ * `with` by re-running itself against the freshly-loaded child rows.
+ */
+async function loadRelationsBatched(ctx, parents, withClause, timeout, depth = 0, path = [ctx.parentMeta.name]) {
+    if (depth >= MAX_DEPTH)
+        throw new errors_js_1.CircularRelationError([...path, '…']);
+    if (parents.length === 0)
+        return;
+    // Sibling relations are independent (each writes only its own parent[relName]
+    // and reads only parent keys), so load them concurrently — on a pool that's
+    // real parallelism, inside a transaction pg queues them on the one connection.
+    const loads = [];
+    for (const [relName, spec] of Object.entries(withClause)) {
+        if (!spec)
+            continue;
+        const rel = ctx.parentMeta.relations[relName];
+        if (!rel) {
+            throw new errors_js_1.ValidationError(`[turbine] Unknown relation "${relName}" on table "${ctx.parentMeta.name}". ` +
+                `Available: ${Object.keys(ctx.parentMeta.relations).join(', ')}`);
+        }
+        const options = spec === true ? {} : spec;
+        loads.push(rel.type === 'manyToMany'
+            ? loadManyToMany(ctx, parents, rel, relName, options, timeout, depth, path)
+            : loadToOneOrMany(ctx, parents, rel, relName, options, timeout, depth, path));
+    }
+    await Promise.all(loads);
+}
+/**
+ * hasMany / hasOne / belongsTo: one follow-up `SELECT ... WHERE childKey = ANY($1)`
+ * (chunked), grouped by the correlation key and attached (array vs single-or-null).
+ */
+async function loadToOneOrMany(ctx, parents, rel, relName, options, timeout, depth, path) {
+    const fk = (0, schema_js_1.normalizeKeyColumns)(rel.foreignKey);
+    const rk = (0, schema_js_1.normalizeKeyColumns)(rel.referenceKey);
+    if (fk.length > 1 || rk.length > 1) {
+        throw new errors_js_1.UnsupportedFeatureError('composite-key batched relation loading', 'relationLoadStrategy: "batched"', `relation "${relName}" — use the default 'join' strategy for composite-key relations`);
+    }
+    const targetMeta = requireTable(ctx.schema, rel.to, relName);
+    // Local key lives on the parent; the correlating key lives on the child.
+    //   hasMany/hasOne: parent.referenceKey  ←  child.foreignKey
+    //   belongsTo:      parent.foreignKey     →  child.referenceKey
+    const parentKeyCol = rel.type === 'belongsTo' ? fk[0] : rk[0];
+    const childKeyCol = rel.type === 'belongsTo' ? rk[0] : fk[0];
+    const parentKeyField = ctx.parentMeta.reverseColumnMap[parentKeyCol] ?? parentKeyCol;
+    const childKeyField = targetMeta.reverseColumnMap[childKeyCol] ?? childKeyCol;
+    const keys = uniqueKeys(parents, parentKeyField);
+    const single = rel.type === 'belongsTo' || rel.type === 'hasOne';
+    if (keys.length === 0) {
+        for (const parent of parents)
+            parent[relName] = single ? null : [];
+        return;
+    }
+    // The follow-up must project the child correlation key even if the caller's
+    // select/omit excluded it; strip it back off afterwards so the shape matches join.
+    const proj = includeKeysForBatching(options.select, options.omit, [childKeyField]);
+    const child = ctx.makeChild(rel.to);
+    const chunks = [];
+    for (let i = 0; i < keys.length; i += MAX_RELATION_KEYS)
+        chunks.push(keys.slice(i, i + MAX_RELATION_KEYS));
+    // Chunks run concurrently, results concatenated in chunk order. Per-relation
+    // `limit` is NOT pushed down here: `LIMIT` on a `fk = ANY($1)` query over the
+    // whole batch would cap TOTAL children, not children-per-parent. It is applied
+    // client-side per group after stitching (below).
+    const chunkResults = await Promise.all(chunks.map(async (chunk) => {
+        const deferred = child.buildFindMany({
+            where: mergeChildWhere(options.where, childKeyField, chunk),
+            select: proj.select,
+            omit: proj.omit,
+            orderBy: options.orderBy,
+        });
+        const result = await ctx.exec(deferred.sql, deferred.params, deferred.preparedName);
+        return deferred.transform(result);
+    }));
+    const allChildren = chunkResults.flat();
+    // Recurse for nested `with` BEFORE stripping keys (children carry their own keys).
+    if (options.with && allChildren.length > 0) {
+        await loadRelationsBatched({ ...ctx, parentMeta: targetMeta }, allChildren, options.with, timeout, depth + 1, [...path, relName]);
+    }
+    const byKey = groupBy(allChildren, childKeyField);
+    const limit = options.limit;
+    for (const parent of parents) {
+        const bucket = byKey.get(keyOf(parent[parentKeyField])) ?? [];
+        if (single) {
+            parent[relName] = bucket[0] ?? null;
+        }
+        else {
+            parent[relName] = limit !== undefined ? bucket.slice(0, limit) : bucket;
+        }
+    }
+    stripFields(allChildren, proj.strip);
+}
+/**
+ * manyToMany: a three-hop batched loader (no join pushdown):
+ *   (1) read junction rows for all parents (`sourceKey = ANY($1)` chunks),
+ *   (2) read the target rows for the collected targetKeys,
+ *   (3) stitch parent → junction targetKeys → target rows in memory.
+ * Composite junction/target keys fall back to the join strategy (throw E017).
+ */
+async function loadManyToMany(ctx, parents, rel, relName, options, timeout, depth, path) {
+    const through = rel.through;
+    if (!through) {
+        throw new errors_js_1.ValidationError(`[turbine] manyToMany relation "${relName}" is missing its junction (\`through\`).`);
+    }
+    const sourceJ = (0, schema_js_1.normalizeKeyColumns)(through.sourceKey);
+    const targetJ = (0, schema_js_1.normalizeKeyColumns)(through.targetKey);
+    const sourceRef = (0, schema_js_1.normalizeKeyColumns)(rel.referenceKey);
+    const targetMeta = requireTable(ctx.schema, rel.to, relName);
+    if (sourceJ.length > 1 || targetJ.length > 1 || sourceRef.length > 1 || targetMeta.primaryKey.length !== 1) {
+        throw new errors_js_1.UnsupportedFeatureError('composite-key batched manyToMany loading', 'relationLoadStrategy: "batched"', `relation "${relName}" — use the default 'join' strategy for composite-key m2m relations`);
+    }
+    const sourceJCol = sourceJ[0];
+    const targetJCol = targetJ[0];
+    const sourceRefCol = sourceRef[0];
+    const targetPkCol = targetMeta.primaryKey[0];
+    const parentRefField = ctx.parentMeta.reverseColumnMap[sourceRefCol] ?? sourceRefCol;
+    const targetPkField = targetMeta.reverseColumnMap[targetPkCol] ?? targetPkCol;
+    const parentKeys = uniqueKeys(parents, parentRefField);
+    if (parentKeys.length === 0) {
+        for (const parent of parents)
+            parent[relName] = [];
+        return;
+    }
+    // (1) Junction rows: sourceKeyVal → [targetKeyVal]. Raw SQL through the caller's
+    // executor (the junction table has no relations we need, so no child reader).
+    const targetsBySource = new Map();
+    const targetValSet = new Set();
+    const jTable = ctx.quote(through.table);
+    const jSource = ctx.quote(sourceJCol);
+    const jTarget = ctx.quote(targetJCol);
+    const jChunks = [];
+    for (let i = 0; i < parentKeys.length; i += MAX_RELATION_KEYS) {
+        jChunks.push(parentKeys.slice(i, i + MAX_RELATION_KEYS));
+    }
+    const jResults = await Promise.all(jChunks.map((chunk) => {
+        const params = [ctx.inClauseParam(chunk)];
+        const predicate = ctx.buildInClause(`${jTable}.${jSource}`, ctx.paramPlaceholder(1), false);
+        const sql = `SELECT ${jTable}.${jSource} AS "s", ${jTable}.${jTarget} AS "t" FROM ${jTable} WHERE ${predicate}`;
+        return ctx.exec(sql, params);
+    }));
+    for (const { rows } of jResults) {
+        for (const row of rows) {
+            const sv = keyOf(row.s);
+            const tv = row.t;
+            if (tv == null)
+                continue;
+            const bucket = targetsBySource.get(sv);
+            if (bucket)
+                bucket.push(tv);
+            else
+                targetsBySource.set(sv, [tv]);
+            targetValSet.add(tv);
+        }
+    }
+    // (2) Target rows by PK, honouring the relation's own where/select/omit/orderBy.
+    const proj = includeKeysForBatching(options.select, options.omit, [targetPkField]);
+    const child = ctx.makeChild(rel.to);
+    const targetVals = [...targetValSet];
+    const tChunks = [];
+    for (let i = 0; i < targetVals.length; i += MAX_RELATION_KEYS) {
+        tChunks.push(targetVals.slice(i, i + MAX_RELATION_KEYS));
+    }
+    const tResults = await Promise.all(tChunks.map(async (chunk) => {
+        const deferred = child.buildFindMany({
+            where: mergeChildWhere(options.where, targetPkField, chunk),
+            select: proj.select,
+            omit: proj.omit,
+            orderBy: options.orderBy,
+        });
+        const result = await ctx.exec(deferred.sql, deferred.params, deferred.preparedName);
+        return deferred.transform(result);
+    }));
+    const targetsInOrder = tResults.flat();
+    // Nested `with` on the target rows (before stripping their PK).
+    if (options.with && targetsInOrder.length > 0) {
+        await loadRelationsBatched({ ...ctx, parentMeta: targetMeta }, targetsInOrder, options.with, timeout, depth + 1, [...path, relName]);
+    }
+    const targetByPk = new Map();
+    for (const t of targetsInOrder)
+        targetByPk.set(keyOf(t[targetPkField]), t);
+    // (3) Stitch. Iterate `targetsInOrder` (already ordered by the relation's
+    // orderBy) and pick the ones each parent links to, so per-parent order honours
+    // orderBy; then apply the per-relation `limit` client-side.
+    const limit = options.limit;
+    for (const parent of parents) {
+        const linked = new Set((targetsBySource.get(keyOf(parent[parentRefField])) ?? []).map(keyOf));
+        if (linked.size === 0) {
+            parent[relName] = [];
+            continue;
+        }
+        const out = [];
+        for (const t of targetsInOrder) {
+            if (linked.has(keyOf(t[targetPkField]))) {
+                out.push(t);
+                if (limit !== undefined && out.length >= limit)
+                    break;
+            }
+        }
+        parent[relName] = out;
+    }
+    stripFields(targetsInOrder, proj.strip);
+}
+// ---------------------------------------------------------------------------
+// Small helpers
+// ---------------------------------------------------------------------------
+/** Merge the batched correlation predicate (`key IN chunk`) into the relation's own where. */
+function mergeChildWhere(where, keyField, chunk) {
+    return { ...(where ?? {}), [keyField]: { in: chunk } };
+}
+/** Distinct, non-null values of `field` across `rows`. */
+function uniqueKeys(rows, field) {
+    const seen = new Set();
+    const out = [];
+    for (const row of rows) {
+        const v = row[field];
+        if (v == null)
+            continue;
+        const k = keyOf(v);
+        if (seen.has(k))
+            continue;
+        seen.add(k);
+        out.push(v);
+    }
+    return out;
+}
+/** Group rows by the stringified value of `field`, preserving input order. */
+function groupBy(rows, field) {
+    const map = new Map();
+    for (const row of rows) {
+        const k = keyOf(row[field]);
+        const bucket = map.get(k);
+        if (bucket)
+            bucket.push(row);
+        else
+            map.set(k, [row]);
+    }
+    return map;
+}
+/** Resolve a table's metadata or throw a clear relation error. */
+function requireTable(schema, table, relName) {
+    const meta = schema.tables[table];
+    if (!meta)
+        throw new errors_js_1.ValidationError(`[turbine] Relation "${relName}" targets unknown table "${table}".`);
+    return meta;
+}