proto.io 0.0.166 → 0.0.168

package/dist/adapters/storage/progres.mjs

@@ -287,7 +287,7 @@ class QueryCompiler {
         const _stages = _.mapValues(context.populates, (populate) => this.dialect.encodePopulate(this, populate));
         const stages = _.fromPairs(_.flatMap(_.values(_stages), (p) => _.toPairs(p)));
         const fetchName = `_fetch_$${query.className.toLowerCase()}`;
-        const parent = { className: query.className, name: fetchName };
+        const parent = { className: query.className, name: fetchName, populates: context.populates };
         const baseFilter = this._encodeFilter(parent, query.filter);
         const populates = this._selectPopulateMap(context, query.className, fetchName);
         const joins = _.compact(_.map(populates, ({ join }) => join));
@@ -795,36 +795,8 @@ class SqlStorage {
 //  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 //  THE SOFTWARE.
 //
-const foreignFieldType = (schema, className, path) => {
-    let fields = schema[className].fields;
-    let last;
-    let result = false;
-    for (const key of _.toPath(path)) {
-        const dataType = fields[key];
-        if (_.isNil(dataType))
-            return;
-        if (isPrimitive(dataType) || isVector(dataType))
-            return;
-        if (isShape(dataType)) {
-            fields = dataType.shape;
-            continue;
-        }
-        if (_.isNil(schema[dataType.target]))
-            return;
-        if (dataType.type === 'relation')
-            result = true;
-        fields = schema[dataType.target].fields;
-        last = dataType;
-    }
-    if (!last)
-        return;
-    return {
-        target: last.target,
-        type: result ? 'relation' : last.type,
-    };
-};
 const _fetchElement = (parent, colname, subpath, dataType) => {
-    const element = sql `${{ identifier: parent.name }}.${{ identifier: parent.name.startsWith('_expr_$') ? '$' : colname }}`;
+    const element = sql `${{ identifier: parent.name }}.${{ identifier: parent.name.startsWith('_doller_expr_$') ? '$' : colname }}`;
     if (!parent.className) {
         if (colname !== '$') {
             return {
@@ -854,7 +826,7 @@ const _fetchElement = (parent, colname, subpath, dataType) => {
             };
         }
     }
-    if (parent.name.startsWith('_expr_$') && colname !== '$') {
+    if (parent.name.startsWith('_doller_expr_$') && colname !== '$') {
         return {
             element: sql `jsonb_extract_path(${element}, ${{ quote: colname.startsWith('$') ? `$${colname}` : colname }})`,
             json: true,
@@ -886,36 +858,37 @@ const resolvePaths = (compiler, className, paths) => {
     }
     return { dataType, colname, subpath };
 };
+const _resolvePopulate = (path, populates) => {
+    let [colname, ...subpath] = path;
+    while (populates && !_.isEmpty(subpath)) {
+        const populate = populates[colname];
+        if (populate)
+            return _resolvePopulate(subpath, populate.populates);
+        const [key, ...remain] = subpath;
+        colname = `${colname}.${key}`;
+        subpath = remain;
+    }
+    return populates?.[colname];
+};
 const fetchElement = (compiler, parent, field) => {
     if (parent.className) {
         const { dataType, colname, subpath } = resolvePaths(compiler, parent.className, _.toPath(field));
         const { element, json } = _fetchElement(parent, colname, subpath, dataType);
-        if (!_.isEmpty(subpath)) {
-            const foreignField = foreignFieldType(compiler.schema, parent.className, field);
-            if (foreignField) {
-                return {
-                    element,
-                    dataType: foreignField,
-                    relation: {
-                        target: foreignField.target,
-                        sql: (callback) => sql `SELECT 
-              ${callback(sql `UNNEST`)} 
-            FROM UNNEST(${{ identifier: parent.name }}.${{ identifier: colname }})`,
-                    },
-                };
-            }
-        }
         if (isPointer(dataType))
             return { element: sql `${{ identifier: parent.name }}.${{ identifier: `${colname}._id` }}`, dataType };
+        const populate = isRelation(dataType) && _resolvePopulate(_.toPath(colname), parent.populates);
+        if (!populate)
+            return { element, dataType: json ? null : dataType };
         return {
             element,
             dataType: json ? null : dataType,
-            relation: isRelation(dataType) ? {
+            relation: {
                 target: dataType.target,
-                sql: (callback) => sql `SELECT
+                populate,
+                mapElem: (callback) => sql `SELECT
           ${callback(sql `${json ? sql `VALUE` : sql `UNNEST`}`)}
         FROM ${json ? sql `jsonb_array_elements(${element})` : sql `UNNEST(${element})`}`,
-            } : null,
+            },
         };
     }
     const [colname, ...subpath] = _.toPath(field);
@@ -1874,6 +1847,231 @@ const encodeQueryExpression = (compiler, parent, expr) => {
     return encodeBooleanExpression(compiler, parent, expr);
 };
 
+//
+//  populate.ts
+//
+//  The MIT License
+//  Copyright (c) 2021 - 2024 O2ter Limited. All rights reserved.
+//
+//  Permission is hereby granted, free of charge, to any person obtaining a copy
+//  of this software and associated documentation files (the "Software"), to deal
+//  in the Software without restriction, including without limitation the rights
+//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+//  copies of the Software, and to permit persons to whom the Software is
+//  furnished to do so, subject to the following conditions:
+//
+//  The above copyright notice and this permission notice shall be included in
+//  all copies or substantial portions of the Software.
+//
+//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+//  THE SOFTWARE.
+//
+const resolveSubpaths = (compiler, populate) => {
+    const subpaths = [];
+    for (const [name, type] of _.toPairs(populate.includes)) {
+        if (isPointer(type)) {
+            subpaths.push(..._.map(resolveSubpaths(compiler, populate.populates[name]), ({ path, type }) => ({
+                path: `${name}.${path}`,
+                type,
+            })));
+        }
+        else {
+            subpaths.push({
+                path: name,
+                type,
+            });
+        }
+    }
+    return subpaths;
+};
+const _isPointer = (schema, className, path) => {
+    let fields = schema[className].fields;
+    let last;
+    for (const key of _.toPath(path)) {
+        const dataType = fields[key];
+        if (_.isNil(dataType))
+            throw Error(`Invalid path: ${path}`);
+        if (isPrimitive(dataType) || isVector(dataType))
+            throw Error(`Invalid path: ${path}`);
+        if (isShape(dataType)) {
+            fields = dataType.shape;
+            continue;
+        }
+        if (dataType.type !== 'pointer')
+            return false;
+        if (_.isNil(schema[dataType.target]))
+            throw Error(`Invalid path: ${path}`);
+        fields = schema[dataType.target].fields;
+        last = dataType;
+    }
+    return last?.type === 'pointer';
+};
+const _selectRelationPopulate = (compiler, parent, populate, field, encode) => {
+    const _local = (field) => sql `${{ identifier: parent.name }}.${{ identifier: field }}`;
+    const _foreign = (field) => sql `${{ identifier: populate.name }}.${{ identifier: field }}`;
+    const subpaths = resolveSubpaths(compiler, populate);
+    let cond;
+    if (_.isNil(populate.foreignField)) {
+        cond = sql `${sql `(${{ quote: populate.className + '$' }} || ${_foreign('_id')})`} = ANY(${_local(field)})`;
+    }
+    else if (_isPointer(compiler.schema, populate.className, populate.foreignField)) {
+        cond = sql `${sql `(${{ quote: parent.className + '$' }} || ${_local('_id')})`} = ${_foreign(populate.colname)}`;
+    }
+    else {
+        cond = sql `${sql `(${{ quote: parent.className + '$' }} || ${_local('_id')})`} = ANY(${_foreign(populate.colname)})`;
+    }
+    return sql `
+    SELECT ${_.compact(_.flatMap(subpaths, ({ path, type }) => [
+        encode && _encodePopulateInclude(populate.name, path, type),
+        !encode && sql `${{ identifier: populate.name }}.${{ identifier: path }}`,
+        !encode && isRelation(type) && sql `${{ identifier: populate.name }}.${{ identifier: `$${path}` }}`,
+    ]))}
+    FROM ${{ identifier: populate.name }} WHERE ${cond}
+    ${!_.isEmpty(populate.sort) ? sql `ORDER BY ${compiler._encodeSort(populate.sort, { className: populate.className, name: populate.name })}` : sql ``}
+    ${populate.limit ? sql `LIMIT ${{ literal: `${populate.limit}` }}` : sql ``}
+    ${populate.skip ? sql `OFFSET ${{ literal: `${populate.skip}` }}` : sql ``}
+    ${compiler.selectLock ? compiler.isUpdate ? sql `FOR UPDATE NOWAIT` : sql `FOR SHARE NOWAIT` : sql ``}
+  `;
+};
+const selectPopulate = (compiler, parent, populate, field) => {
+    if (populate.type === 'relation') {
+        return {
+            columns: [
+                sql `
+          ARRAY(
+            SELECT to_jsonb(${{ identifier: populate.name }}) FROM (
+              ${_selectRelationPopulate(compiler, parent, populate, field, true)}
+            ) ${{ identifier: populate.name }}
+          ) AS ${{ identifier: field }}
+        `,
+                sql `${{ identifier: parent.name }}.${{ identifier: field }} AS ${{ identifier: `$${field}` }}`
+            ],
+        };
+    }
+    const _local = (field) => sql `${{ identifier: parent.name }}.${{ identifier: field }}`;
+    const _foreign = (field) => sql `${{ identifier: populate.name }}.${{ identifier: field }}`;
+    const subpaths = resolveSubpaths(compiler, populate);
+    return {
+        columns: _.compact(_.flatMap(subpaths, ({ path, type }) => [
+            sql `${{ identifier: populate.name }}.${{ identifier: path }} AS ${{ identifier: `${field}.${path}` }}`,
+            isRelation(type) && sql `${{ identifier: populate.name }}.${{ identifier: `$${path}` }} AS ${{ identifier: `$${field}.${path}` }}`,
+        ])),
+        join: sql `
+      LEFT JOIN ${{ identifier: populate.name }}
+      ON ${sql `(${{ quote: populate.className + '$' }} || ${_foreign('_id')})`} = ${_local(field)}
+    `,
+    };
+};
+const encodeRemix = (parent, remix) => sql `${remix?.className === parent.className ? sql `
+  (SELECT * FROM ${{ identifier: remix.name }} UNION SELECT * FROM ${{ identifier: parent.className }})
+` : { identifier: parent.className }}`;
+const encodeForeignField = (compiler, parent, foreignField, remix) => {
+    const { paths: [colname, ...subpath], dataType } = resolveColumn(compiler.schema, parent.className, foreignField);
+    const tempName = `_populate_$${compiler.nextIdx()}`;
+    const _local = (field) => sql `${{ identifier: parent.name }}.${{ identifier: field }}`;
+    const _foreign = (field) => sql `${{ identifier: tempName }}.${{ identifier: field }}`;
+    if (_.isEmpty(subpath) && isRelation(dataType) && dataType.foreignField) {
+        const { joins, field, rows, array } = encodeForeignField(compiler, { className: dataType.target, name: tempName }, dataType.foreignField, remix);
+        return {
+            joins: [],
+            field: sql `(
+        SELECT ${sql `(${{ quote: dataType.target + '$' }} || ${_foreign('_id')})`}
+        FROM ${encodeRemix({ className: dataType.target }, remix)} AS ${{ identifier: tempName }}
+        ${!_.isEmpty(joins) ? { literal: joins, separator: '\n' } : sql ``}
+        WHERE ${sql `(${{ quote: parent.className + '$' }} || ${_local('_id')})`} = ${array || rows ? sql `ANY(${field})` : field}
+      )`,
+            array: false,
+            rows: true,
+        };
+    }
+    if (_.isEmpty(subpath)) {
+        return {
+            joins: [],
+            field: sql `${{ identifier: parent.name }}.${{ identifier: foreignField }}`,
+            array: isRelation(dataType),
+            rows: false,
+        };
+    }
+    if (!isPointer(dataType) && !isRelation(dataType))
+        throw Error(`Invalid path: ${foreignField}`);
+    const { joins, field, rows, array } = encodeForeignField(compiler, { className: dataType.target, name: tempName }, subpath.join('.'), remix);
+    const cond = [];
+    if (compiler.extraFilter) {
+        const filter = compiler.extraFilter(dataType.target);
+        cond.push(compiler._encodeFilter({ className: dataType.target, name: tempName }, filter));
+    }
+    if (isPointer(dataType)) {
+        cond.push(sql `${sql `(${{ quote: dataType.target + '$' }} || ${_foreign('_id')})`} = ${_local(colname)}`);
+        return {
+            joins: [sql `
+        LEFT JOIN ${encodeRemix({ className: dataType.target }, remix)} AS ${{ identifier: tempName }}
+        ON ${{ literal: _.map(_.compact(cond), x => sql `(${x})`), separator: ' AND ' }}
+      `, ...joins],
+            field,
+            array,
+            rows,
+        };
+    }
+    if (_.isNil(dataType.foreignField)) {
+        cond.push(sql `${sql `(${{ quote: dataType.target + '$' }} || ${_foreign('_id')})`} = ANY(${_local(colname)})`);
+    }
+    else if (_isPointer(compiler.schema, dataType.target, dataType.foreignField)) {
+        cond.push(sql `${sql `(${{ quote: parent.className + '$' }} || ${_local('_id')})`} = ${_foreign(dataType.foreignField)}`);
+    }
+    else {
+        cond.push(sql `${sql `(${{ quote: parent.className + '$' }} || ${_local('_id')})`} = ANY(${_foreign(dataType.foreignField)})`);
+    }
+    return {
+        joins: [],
+        field: sql `(
+      SELECT ${array ? sql `UNNEST(${field})` : field}
+      FROM ${encodeRemix({ className: dataType.target }, remix)} AS ${{ identifier: tempName }}
+      ${!_.isEmpty(joins) ? { literal: joins, separator: '\n' } : sql ``}
+      WHERE ${{ literal: _.map(_.compact(cond), x => sql `(${x})`), separator: ' AND ' }}
+    )`,
+        array: false,
+        rows: true,
+    };
+};
+const encodePopulate = (compiler, parent, remix) => {
+    const _filter = _.compact([
+        parent.filter && compiler._encodeFilter(parent, parent.filter),
+        compiler.extraFilter && compiler._encodeFilter(parent, compiler.extraFilter(parent.className)),
+    ]);
+    const _populates = _.map(parent.populates, (populate, field) => selectPopulate(compiler, parent, populate, field));
+    const _joins = _.compact(_.map(_populates, ({ join }) => join));
+    const _includes = _.pickBy(parent.includes, v => isPrimitive(v));
+    const { joins: _joins2 = [], field: _foreignField = undefined, rows = false, } = parent.foreignField ? encodeForeignField(compiler, {
+        className: parent.className,
+        name: parent.name,
+    }, parent.foreignField, remix) : {};
+    return _.reduce(parent.populates, (acc, populate) => ({
+        ...encodePopulate(compiler, populate, remix),
+        ...acc,
+    }), {
+        [parent.name]: sql `
+      SELECT * FROM (
+        SELECT
+        ${{
+            literal: [
+                ..._.map(_.keys(_includes), colname => sql `${{ identifier: parent.name }}.${{ identifier: colname }}`),
+                ..._.flatMap(_populates, ({ columns: column }) => column),
+                ..._foreignField ? [sql `${rows ? sql `ARRAY(${_foreignField})` : _foreignField} AS ${{ identifier: parent.colname }}`] : [],
+            ], separator: ',\n'
+        }}
+        FROM ${encodeRemix(parent, remix)} AS ${{ identifier: parent.name }}
+        ${!_.isEmpty(_joins) || !_.isEmpty(_joins2) ? { literal: [..._joins, ..._joins2], separator: '\n' } : sql ``}
+      ) AS ${{ identifier: parent.name }}
+      ${!_.isEmpty(_filter) ? sql `WHERE ${{ literal: _.map(_.compact(_filter), x => sql `(${x})`), separator: ' AND ' }}` : sql ``}
+    `,
+    });
+};
+
 //
 //  selectors.ts
 //
@@ -2060,12 +2258,12 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
                 if (dataType === 'array' || (!_.isString(dataType) && dataType?.type === 'array')) {
                     return sql `${element} <@ ${{ value: _encodeValue(expr.value) }}`;
                 }
-                else if (relation) {
+                if (relation) {
                     if (!_.every(expr.value, x => x instanceof TObject && x.objectId))
                         break;
-                    return sql `ARRAY(${relation.sql((v) => sql `${v} ->> '_id'`)}) <@ ARRAY[${_.map(expr.value, (x) => sql `${{ value: x.objectId }}`)}]`;
+                    return sql `ARRAY(${relation.mapElem((v) => sql `${v} ->> '_id'`)}) <@ ARRAY[${_.map(expr.value, (x) => sql `${{ value: x.objectId }}`)}]`;
                 }
-                else if (!dataType) {
+                if (!dataType) {
                     return sql `jsonb_typeof(${element}) ${nullSafeEqual()} 'array' AND ${element} <@ ${_encodeJsonValue(_encodeValue(expr.value))}`;
                 }
             }
@@ -2078,12 +2276,12 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
                 if (dataType === 'array' || (!_.isString(dataType) && dataType?.type === 'array')) {
                     return sql `${element} @> ${{ value: _encodeValue(expr.value) }}`;
                 }
-                else if (relation) {
+                if (relation) {
                     if (!_.every(expr.value, x => x instanceof TObject && x.objectId))
                         break;
-                    return sql `ARRAY(${relation.sql((v) => sql `${v} ->> '_id'`)}) @> ARRAY[${_.map(expr.value, (x) => sql `${{ value: x.objectId }}`)}]`;
+                    return sql `ARRAY(${relation.mapElem((v) => sql `${v} ->> '_id'`)}) @> ARRAY[${_.map(expr.value, (x) => sql `${{ value: x.objectId }}`)}]`;
                 }
-                else if (!dataType) {
+                if (!dataType) {
                     return sql `jsonb_typeof(${element}) ${nullSafeEqual()} 'array' AND ${element} @> ${_encodeJsonValue(_encodeValue(expr.value))}`;
                 }
             }
@@ -2096,12 +2294,12 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
                 if (dataType === 'array' || (!_.isString(dataType) && dataType?.type === 'array')) {
                     return sql `NOT ${element} && ${{ value: _encodeValue(expr.value) }}`;
                 }
-                else if (relation) {
+                if (relation) {
                     if (!_.every(expr.value, x => x instanceof TObject && x.objectId))
                         break;
-                    return sql `NOT ARRAY(${relation.sql((v) => sql `${v} ->> '_id'`)}) && ARRAY[${_.map(expr.value, (x) => sql `${{ value: x.objectId }}`)}]`;
+                    return sql `NOT ARRAY(${relation.mapElem((v) => sql `${v} ->> '_id'`)}) && ARRAY[${_.map(expr.value, (x) => sql `${{ value: x.objectId }}`)}]`;
                 }
-                else if (!dataType) {
+                if (!dataType) {
                     return sql `jsonb_typeof(${element}) ${nullSafeEqual()} 'array' AND NOT ${element} && ${_encodeJsonValue(_encodeValue(expr.value))}`;
                 }
             }
@@ -2114,12 +2312,12 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
                 if (dataType === 'array' || (!_.isString(dataType) && dataType?.type === 'array')) {
                     return sql `${element} && ${{ value: _encodeValue(expr.value) }}`;
                 }
-                else if (relation) {
+                if (relation) {
                     if (!_.every(expr.value, x => x instanceof TObject && x.objectId))
                         break;
-                    return sql `ARRAY(${relation.sql((v) => sql `${v} ->> '_id'`)}) && ARRAY[${_.map(expr.value, (x) => sql `${{ value: x.objectId }}`)}]`;
+                    return sql `ARRAY(${relation.mapElem((v) => sql `${v} ->> '_id'`)}) && ARRAY[${_.map(expr.value, (x) => sql `${{ value: x.objectId }}`)}]`;
                 }
-                else if (!dataType) {
+                if (!dataType) {
                     return sql `jsonb_typeof(${element}) ${nullSafeEqual()} 'array' AND ${element} && ${_encodeJsonValue(_encodeValue(expr.value))}`;
                 }
             }
@@ -2135,7 +2333,7 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
                     if (_.isString(expr.value)) {
                         return sql `${element} LIKE ${{ value: `%${expr.value.replace(/([\\_%])/g, '\\$1')}%` }}`;
                     }
-                    else if (_.isRegExp(expr.value)) {
+                    if (_.isRegExp(expr.value)) {
                         if (expr.value.ignoreCase)
                             return sql `${element} ~* ${{ value: expr.value.source }}`;
                         return sql `${element} ~ ${{ value: expr.value.source }}`;
@@ -2145,7 +2343,7 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
                     if (_.isString(expr.value)) {
                         return sql `jsonb_typeof(${element}) ${nullSafeEqual()} 'string' AND (${element} #>> '{}') LIKE ${{ value: `%${expr.value.replace(/([\\_%])/g, '\\$1')}%` }}`;
                     }
-                    else if (_.isRegExp(expr.value)) {
+                    if (_.isRegExp(expr.value)) {
                         if (expr.value.ignoreCase)
                             return sql `${element} ~* ${{ value: expr.value.source }}`;
                         return sql `jsonb_typeof(${element}) ${nullSafeEqual()} 'string' AND (${element} #>> '{}') ~ ${{ value: expr.value.source }}`;
@@ -2159,7 +2357,7 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
                 if (dataType === 'string' || (!_.isString(dataType) && dataType?.type === 'string')) {
                     return sql `${element} LIKE ${{ value: `${expr.value.replace(/([\\_%])/g, '\\$1')}%` }}`;
                 }
-                else if (!dataType) {
+                if (!dataType) {
                     return sql `jsonb_typeof(${element}) ${nullSafeEqual()} 'string' AND (${element} #>> '{}') LIKE ${{ value: `${expr.value.replace(/([\\_%])/g, '\\$1')}%` }}`;
                 }
             }
@@ -2170,7 +2368,7 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
                 if (dataType === 'string' || (!_.isString(dataType) && dataType?.type === 'string')) {
                     return sql `${element} LIKE ${{ value: `%${expr.value.replace(/([\\_%])/g, '\\$1')}` }}`;
                 }
-                else if (!dataType) {
+                if (!dataType) {
                     return sql `jsonb_typeof(${element}) ${nullSafeEqual()} 'string' AND (${element} #>> '{}') LIKE ${{ value: `%${expr.value.replace(/([\\_%])/g, '\\$1')}` }}`;
                 }
             }
@@ -2181,10 +2379,10 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
                 if (dataType === 'string' || (!_.isString(dataType) && dataType?.type === 'string')) {
                     return sql `COALESCE(length(${element}), 0) = ${{ value: expr.value }}`;
                 }
-                else if (dataType === 'array' || (!_.isString(dataType) && (dataType?.type === 'array' || dataType?.type === 'vector' || dataType?.type === 'relation'))) {
+                if (dataType === 'array' || (!_.isString(dataType) && (dataType?.type === 'array' || dataType?.type === 'vector' || dataType?.type === 'relation'))) {
                     return sql `COALESCE(array_length(${element}, 1), 0) = ${{ value: expr.value }}`;
                 }
-                else if (!dataType) {
+                if (!dataType) {
                     return sql `(
             CASE jsonb_typeof(${element})
               WHEN 'array' THEN jsonb_array_length(${element}) = ${{ value: expr.value }}
@@ -2201,10 +2399,10 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
                 if (dataType === 'string' || (!_.isString(dataType) && dataType?.type === 'string')) {
                     return sql `COALESCE(length(${element}), 0) ${{ literal: expr.value ? '=' : '<>' }} 0`;
                 }
-                else if (dataType === 'array' || (!_.isString(dataType) && (dataType?.type === 'array' || dataType?.type === 'vector' || dataType?.type === 'relation'))) {
+                if (dataType === 'array' || (!_.isString(dataType) && (dataType?.type === 'array' || dataType?.type === 'vector' || dataType?.type === 'relation'))) {
                     return sql `COALESCE(array_length(${element}, 1), 0) ${{ literal: expr.value ? '=' : '<>' }} 0`;
                 }
-                else if (!dataType) {
+                if (!dataType) {
                     return sql `(
             CASE jsonb_typeof(${element})
               WHEN 'array' THEN jsonb_array_length(${element}) ${{ literal: expr.value ? '=' : '<>' }} 0
@@ -2218,23 +2416,34 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
             {
                 if (!(expr.value instanceof QuerySelector))
                     break;
-                const tempName = `_expr_$${compiler.nextIdx()}`;
-                const filter = compiler._encodeFilter({ name: tempName, className: relation?.target }, expr.value);
-                if (!filter)
-                    break;
-                if (relation) {
+                if (relation?.populate && parent.className) {
+                    const tempName = `_populate_expr_$${compiler.nextIdx()}`;
+                    const filter = compiler._encodeFilter({
+                        name: tempName,
+                        className: relation.target,
+                        populates: relation.populate.populates,
+                    }, expr.value);
+                    if (!filter)
+                        break;
                     return sql `NOT EXISTS(
-            SELECT * FROM (${relation.sql((v) => sql `${v} AS "$"`)}) AS ${{ identifier: tempName }}
+            SELECT * FROM (${_selectRelationPopulate(compiler, {
+                        className: parent.className,
+                        name: parent.name,
+                    }, relation.populate, `$${field}`, false)}) AS ${{ identifier: tempName }}
             WHERE NOT (${filter})
           )`;
                 }
-                else if (dataType === 'array' || (!_.isString(dataType) && (dataType?.type === 'array' || dataType?.type === 'vector'))) {
+                const tempName = `_doller_expr_$${compiler.nextIdx()}`;
+                const filter = compiler._encodeFilter({ name: tempName, className: relation?.target }, expr.value);
+                if (!filter)
+                    break;
+                if (dataType === 'array' || (!_.isString(dataType) && (dataType?.type === 'array' || dataType?.type === 'vector'))) {
                     return sql `NOT EXISTS(
             SELECT * FROM (SELECT UNNEST AS "$" FROM UNNEST(${element})) AS ${{ identifier: tempName }}
             WHERE NOT (${filter})
           )`;
                 }
-                else if (!dataType) {
+                if (!dataType) {
                     return sql `jsonb_typeof(${element}) ${nullSafeEqual()} 'array' AND NOT EXISTS(
             SELECT * FROM (SELECT value AS "$" FROM jsonb_array_elements(${element})) AS ${{ identifier: tempName }}
             WHERE NOT (${filter})
@@ -2245,23 +2454,34 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
             {
                 if (!(expr.value instanceof QuerySelector))
                     break;
-                const tempName = `_expr_$${compiler.nextIdx()}`;
-                const filter = compiler._encodeFilter({ name: tempName, className: relation?.target }, expr.value);
-                if (!filter)
-                    break;
-                if (relation) {
+                if (relation?.populate && parent.className) {
+                    const tempName = `_populate_expr_$${compiler.nextIdx()}`;
+                    const filter = compiler._encodeFilter({
+                        name: tempName,
+                        className: relation.target,
+                        populates: relation.populate.populates,
+                    }, expr.value);
+                    if (!filter)
+                        break;
                     return sql `EXISTS(
-            SELECT * FROM (${relation.sql((v) => sql `${v} AS "$"`)}) AS ${{ identifier: tempName }}
+            SELECT * FROM (${_selectRelationPopulate(compiler, {
+                        className: parent.className,
+                        name: parent.name,
+                    }, relation.populate, `$${field}`, false)}) AS ${{ identifier: tempName }}
             WHERE ${filter}
           )`;
                 }
-                else if (dataType === 'array' || (!_.isString(dataType) && (dataType?.type === 'array' || dataType?.type === 'vector'))) {
+                const tempName = `_doller_expr_$${compiler.nextIdx()}`;
+                const filter = compiler._encodeFilter({ name: tempName, className: relation?.target }, expr.value);
+                if (!filter)
+                    break;
+                if (dataType === 'array' || (!_.isString(dataType) && (dataType?.type === 'array' || dataType?.type === 'vector'))) {
                     return sql `EXISTS(
             SELECT * FROM (SELECT UNNEST AS "$" FROM UNNEST(${element})) AS ${{ identifier: tempName }}
             WHERE ${filter}
           )`;
                 }
-                else if (!dataType) {
+                if (!dataType) {
                     return sql `jsonb_typeof(${element}) ${nullSafeEqual()} 'array' AND EXISTS(
             SELECT * FROM (SELECT value AS "$" FROM jsonb_array_elements(${element})) AS ${{ identifier: tempName }}
             WHERE ${filter}
@@ -2272,215 +2492,6 @@ const encodeFieldExpression = (compiler, parent, field, expr) => {
     throw Error('Invalid expression');
 };
 
-//
-//  populate.ts
-//
-//  The MIT License
-//  Copyright (c) 2021 - 2024 O2ter Limited. All rights reserved.
-//
-//  Permission is hereby granted, free of charge, to any person obtaining a copy
-//  of this software and associated documentation files (the "Software"), to deal
-//  in the Software without restriction, including without limitation the rights
-//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-//  copies of the Software, and to permit persons to whom the Software is
-//  furnished to do so, subject to the following conditions:
-//
-//  The above copyright notice and this permission notice shall be included in
-//  all copies or substantial portions of the Software.
-//
-//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-//  THE SOFTWARE.
-//
-const resolveSubpaths = (compiler, populate) => {
-    const subpaths = [];
-    for (const [name, type] of _.toPairs(populate.includes)) {
-        if (isPointer(type)) {
-            subpaths.push(..._.map(resolveSubpaths(compiler, populate.populates[name]), ({ path, type }) => ({
-                path: `${name}.${path}`,
-                type,
-            })));
-        }
-        else {
-            subpaths.push({
-                path: name,
-                type,
-            });
-        }
-    }
-    return subpaths;
-};
-const _isPointer = (schema, className, path) => {
-    let fields = schema[className].fields;
-    let last;
-    for (const key of _.toPath(path)) {
-        const dataType = fields[key];
-        if (_.isNil(dataType))
-            throw Error(`Invalid path: ${path}`);
-        if (isPrimitive(dataType) || isVector(dataType))
-            throw Error(`Invalid path: ${path}`);
-        if (isShape(dataType)) {
-            fields = dataType.shape;
-            continue;
-        }
-        if (dataType.type !== 'pointer')
-            return false;
-        if (_.isNil(schema[dataType.target]))
-            throw Error(`Invalid path: ${path}`);
-        fields = schema[dataType.target].fields;
-        last = dataType;
-    }
-    return last?.type === 'pointer';
-};
-const selectPopulate = (compiler, parent, populate, field) => {
-    const _local = (field) => sql `${{ identifier: parent.name }}.${{ identifier: field }}`;
-    const _foreign = (field) => sql `${{ identifier: populate.name }}.${{ identifier: field }}`;
-    const subpaths = resolveSubpaths(compiler, populate);
-    const cond = [];
-    if (compiler.extraFilter) {
-        const filter = compiler.extraFilter(populate.className);
-        cond.push(compiler._encodeFilter(populate, filter));
-    }
-    if (populate.type === 'pointer') {
-        cond.push(sql `${sql `(${{ quote: populate.className + '$' }} || ${_foreign('_id')})`} = ${_local(field)}`);
-        return {
-            columns: _.map(subpaths, ({ path }) => sql `${{ identifier: populate.name }}.${{ identifier: path }} AS ${{ identifier: `${field}.${path}` }}`),
-            join: sql `
-        LEFT JOIN ${{ identifier: populate.name }}
-        ON ${{ literal: _.map(_.compact(cond), x => sql `(${x})`), separator: ' AND ' }}
-      `,
-        };
-    }
-    if (_.isNil(populate.foreignField)) {
-        cond.push(sql `${sql `(${{ quote: populate.className + '$' }} || ${_foreign('_id')})`} = ANY(${_local(field)})`);
-    }
-    else if (_isPointer(compiler.schema, populate.className, populate.foreignField)) {
-        cond.push(sql `${sql `(${{ quote: parent.className + '$' }} || ${_local('_id')})`} = ${_foreign(populate.colname)}`);
-    }
-    else {
-        cond.push(sql `${sql `(${{ quote: parent.className + '$' }} || ${_local('_id')})`} = ANY(${_foreign(populate.colname)})`);
-    }
-    return {
-        columns: [sql `
-      ARRAY(
-        SELECT to_jsonb(${{ identifier: populate.name }}) FROM (
-          SELECT ${_.map(subpaths, ({ path, type }) => _encodePopulateInclude(populate.name, path, type))}
-          FROM ${{ identifier: populate.name }} WHERE ${{ literal: _.map(_.compact(cond), x => sql `(${x})`), separator: ' AND ' }}
-          ${!_.isEmpty(populate.sort) ? sql `ORDER BY ${compiler._encodeSort(populate.sort, { className: populate.className, name: populate.name })}` : sql ``}
-          ${populate.limit ? sql `LIMIT ${{ literal: `${populate.limit}` }}` : sql ``}
-          ${populate.skip ? sql `OFFSET ${{ literal: `${populate.skip}` }}` : sql ``}
-          ${compiler.selectLock ? compiler.isUpdate ? sql `FOR UPDATE NOWAIT` : sql `FOR SHARE NOWAIT` : sql ``}
-        ) ${{ identifier: populate.name }}
-      ) AS ${{ identifier: field }}
-    `],
-    };
-};
-const encodeRemix = (parent, remix) => sql `${remix?.className === parent.className ? sql `
-  (SELECT * FROM ${{ identifier: remix.name }} UNION SELECT * FROM ${{ identifier: parent.className }})
-` : { identifier: parent.className }}`;
-const encodeForeignField = (compiler, parent, foreignField, remix) => {
-    const { paths: [colname, ...subpath], dataType } = resolveColumn(compiler.schema, parent.className, foreignField);
-    const tempName = `_populate_$${compiler.nextIdx()}`;
-    const _local = (field) => sql `${{ identifier: parent.name }}.${{ identifier: field }}`;
-    const _foreign = (field) => sql `${{ identifier: tempName }}.${{ identifier: field }}`;
-    if (_.isEmpty(subpath) && isRelation(dataType) && dataType.foreignField) {
-        const { joins, field, rows, array } = encodeForeignField(compiler, { className: dataType.target, name: tempName }, dataType.foreignField, remix);
-        return {
-            joins: [],
-            field: sql `(
-        SELECT ${sql `(${{ quote: dataType.target + '$' }} || ${_foreign('_id')})`}
-        FROM ${encodeRemix({ className: dataType.target }, remix)} AS ${{ identifier: tempName }}
-        ${!_.isEmpty(joins) ? { literal: joins, separator: '\n' } : sql ``}
-        WHERE ${sql `(${{ quote: parent.className + '$' }} || ${_local('_id')})`} = ${array || rows ? sql `ANY(${field})` : field}
-      )`,
-            array: false,
-            rows: true,
-        };
-    }
-    if (_.isEmpty(subpath)) {
-        return {
-            joins: [],
-            field: sql `${{ identifier: parent.name }}.${{ identifier: foreignField }}`,
-            array: isRelation(dataType),
-            rows: false,
-        };
-    }
-    if (!isPointer(dataType) && !isRelation(dataType))
-        throw Error(`Invalid path: ${foreignField}`);
-    const { joins, field, rows, array } = encodeForeignField(compiler, { className: dataType.target, name: tempName }, subpath.join('.'), remix);
-    const cond = [];
-    if (compiler.extraFilter) {
-        const filter = compiler.extraFilter(dataType.target);
-        cond.push(compiler._encodeFilter({ className: dataType.target, name: tempName }, filter));
-    }
-    if (isPointer(dataType)) {
-        cond.push(sql `${sql `(${{ quote: dataType.target + '$' }} || ${_foreign('_id')})`} = ${_local(colname)}`);
-        return {
-            joins: [sql `
-        LEFT JOIN ${encodeRemix({ className: dataType.target }, remix)} AS ${{ identifier: tempName }}
-        ON ${{ literal: _.map(_.compact(cond), x => sql `(${x})`), separator: ' AND ' }}
-      `, ...joins],
-            field,
-            array,
-            rows,
-        };
-    }
-    if (_.isNil(dataType.foreignField)) {
-        cond.push(sql `${sql `(${{ quote: dataType.target + '$' }} || ${_foreign('_id')})`} = ANY(${_local(colname)})`);
-    }
-    else if (_isPointer(compiler.schema, dataType.target, dataType.foreignField)) {
-        cond.push(sql `${sql `(${{ quote: parent.className + '$' }} || ${_local('_id')})`} = ${_foreign(dataType.foreignField)}`);
-    }
-    else {
-        cond.push(sql `${sql `(${{ quote: parent.className + '$' }} || ${_local('_id')})`} = ANY(${_foreign(dataType.foreignField)})`);
-    }
-    return {
-        joins: [],
-        field: sql `(
-      SELECT ${array ? sql `UNNEST(${field})` : field}
-      FROM ${encodeRemix({ className: dataType.target }, remix)} AS ${{ identifier: tempName }}
-      ${!_.isEmpty(joins) ? { literal: joins, separator: '\n' } : sql ``}
-      WHERE ${{ literal: _.map(_.compact(cond), x => sql `(${x})`), separator: ' AND ' }}
-    )`,
-        array: false,
-        rows: true,
-    };
-};
-const encodePopulate = (compiler, parent, remix) => {
-    const _filter = parent.filter && compiler._encodeFilter(parent, parent.filter);
-    const _populates = _.map(parent.populates, (populate, field) => selectPopulate(compiler, parent, populate, field));
-    const _joins = _.compact(_.map(_populates, ({ join }) => join));
-    const _includes = _.pickBy(parent.includes, v => isPrimitive(v));
-    const { joins: _joins2 = [], field: _foreignField = undefined, rows = false, } = parent.foreignField ? encodeForeignField(compiler, {
-        className: parent.className,
-        name: parent.name,
-    }, parent.foreignField, remix) : {};
-    return _.reduce(parent.populates, (acc, populate) => ({
-        ...encodePopulate(compiler, populate, remix),
-        ...acc,
-    }), {
-        [parent.name]: sql `
-      SELECT * FROM (
-        SELECT
-        ${{
-            literal: [
-                ..._.map(_.keys(_includes), colname => sql `${{ identifier: parent.name }}.${{ identifier: colname }}`),
-                ..._.flatMap(_populates, ({ columns: column }) => column),
-                ..._foreignField ? [sql `${rows ? sql `ARRAY(${_foreignField})` : _foreignField} AS ${{ identifier: parent.colname }}`] : [],
-            ], separator: ',\n'
-        }}
-        FROM ${encodeRemix(parent, remix)} AS ${{ identifier: parent.name }}
-        ${!_.isEmpty(_joins) || !_.isEmpty(_joins2) ? { literal: [..._joins, ..._joins2], separator: '\n' } : sql ``}
-      ) AS ${{ identifier: parent.name }}
-      ${_filter ? sql `WHERE ${_filter}` : sql ``}
-    `,
-    });
-};
-
 //
 //  relation.ts
 //