remult 3.4.0-next.7 → 3.4.0

package/esm/internals.js

@@ -4,6 +4,7 @@ export { getRelationInfo, getRelationFieldInfo, } from './src/remult3/relationIn
 export { decorateColumnSettings, controllerRefImpl, getControllerRef, fieldOptionsEnricher, } from './src/remult3/RepositoryImplementation.js';
 export { getEntitySettings } from './src/remult3/getEntityRef.js';
 export { __updateEntityBasedOnWhere } from './src/filter/filter-interfaces.js';
+export { omitFilterField, walkFilter } from './src/filter/walk-filter.js';
 export const actionInfo = remultStatic.actionInfo;
 export { isOfType } from './src/isOfType.js';
 export { flags } from './src/remult3/remult3.js';

package/esm/remult-knex/index.js

@@ -1,12 +1,12 @@
-import { CustomSqlFilterBuilder, dbNamesOfWithForceSqlExpression, } from '../src/filter/filter-consumer-bridge-to-sql-request.js';
+import { CustomSqlFilterBuilder, dbNamesOfWithForceSqlExpression, escapeLike, LIKE_ESCAPE_SQL, } from '../src/filter/filter-consumer-bridge-to-sql-request.js';
 import { dbNamesOf, isDbReadonly, shouldCreateEntity, shouldNotCreateField, } from '../src/filter/filter-consumer-bridge-to-sql-request.js';
 import { customDatabaseFilterToken, Filter, } from '../src/filter/filter-interfaces.js';
 import { remult as remultContext } from '../src/remult-proxy.js';
 import { getRepository, isAutoIncrement, } from '../src/remult3/RepositoryImplementation.js';
 import { Sort } from '../src/sort.js';
-import { ValueConverters } from '../src/valueConverters.js';
+import { fromDbColumn, toDbColumn, ValueConverters, } from '../src/valueConverters.js';
 import { getRepositoryInternals } from '../src/remult3/repository-internals.js';
-import { groupByImpl, getRowAfterUpdate, } from '../src/data-providers/sql-database.js';
+import { groupByImpl, getRowAfterUpdate, jsonColumnStorageOf, } from '../src/data-providers/sql-database.js';
 export class KnexDataProvider {
     knex;
     constructor(knex) {
@@ -46,29 +46,14 @@ export class KnexDataProvider {
         return r.knex;
     }
     wrapIdentifier = (x) => this.knex.ref(x) + '';
-    getEntityDataProvider(entity) {
+    get jsonColumnStorage() {
         if (!supportsJsonLoadingAndSaving(this.knex))
-            for (const f of entity.fields.toArray()) {
-                if (f.valueConverter.fieldTypeInDb === 'json') {
-                    //@ts-ignore
-                    f.valueConverter = {
-                        ...f.valueConverter,
-                        toDb: ValueConverters.JsonString.toDb,
-                        fromDb: ValueConverters.JsonString.fromDb,
-                    };
-                }
-            }
-        else if (this.knex.client.config.client === 'mysql2') {
-            for (const f of entity.fields.toArray()) {
-                if (f.valueConverter.fieldTypeInDb === 'json') {
-                    //@ts-ignore
-                    f.valueConverter = {
-                        ...f.valueConverter,
-                        toDb: ValueConverters.JsonString.toDb,
-                    };
-                }
-            }
-        }
+            return 'text';
+        if (this.knex.client.config.client === 'mysql2')
+            return 'textOnWrite';
+        return 'native';
+    }
+    getEntityDataProvider(entity) {
         return new KnexEntityDataProvider(entity, this.knex, this.wrapIdentifier, this);
     }
     async transaction(action) {
@@ -93,7 +78,7 @@ export class KnexDataProvider {
         if (!wrapIdentifier)
             wrapIdentifier = (x) => x;
         const repo = getRepository(entity);
-        var b = new FilterConsumerBridgeToKnexRequest(await dbNamesOfWithForceSqlExpression(repo.metadata, wrapIdentifier), wrapIdentifier ?? wrapIdentifier);
+        var b = new FilterConsumerBridgeToKnexRequest(await dbNamesOfWithForceSqlExpression(repo.metadata, wrapIdentifier), wrapIdentifier ?? wrapIdentifier, jsonColumnStorageOf(repo));
         b._addWhere = false;
         await (await getRepositoryInternals(repo)._translateWhereToFilter(condition)).__applyToConsumer(b);
         let r = await b.resolveWhere();
@@ -119,7 +104,7 @@ class KnexEntityDataProvider {
     async groupBy(options) {
         return groupByImpl(options, await dbNamesOf(this.entity, this.dp.wrapIdentifier), this.dp.createCommand(), false, (limit, offset) => this.knex.client.config.client?.includes('mssql')
             ? ' offset ' + offset + ' rows fetch next ' + limit + ' rows only'
-            : ' limit ' + limit + ' offset ' + offset);
+            : ' limit ' + limit + ' offset ' + offset, this.dp.jsonColumnStorage);
     }
     getEntityFrom(e) {
         if (this.entity.options.sqlExpression)
@@ -128,7 +113,7 @@ class KnexEntityDataProvider {
     }
     async count(where) {
         const e = await this.init();
-        const br = new FilterConsumerBridgeToKnexRequest(e, this.rawSqlWrapIdentifier);
+        const br = new FilterConsumerBridgeToKnexRequest(e, this.rawSqlWrapIdentifier, this.dp.jsonColumnStorage);
         where.__applyToConsumer(br);
         let r = await br.resolveWhere();
         const result = await this.getEntityFrom(e)
@@ -162,7 +147,7 @@ class KnexEntityDataProvider {
         }
         let query = this.getEntityFrom(e).select(cols);
         if (options?.where) {
-            const br = new FilterConsumerBridgeToKnexRequest(e, this.rawSqlWrapIdentifier);
+            const br = new FilterConsumerBridgeToKnexRequest(e, this.rawSqlWrapIdentifier, this.dp.jsonColumnStorage);
             options.where.__applyToConsumer(br);
             let r = await br.resolveWhere();
             query.where((b) => r.forEach((y) => y(b)));
@@ -188,7 +173,7 @@ class KnexEntityDataProvider {
             for (let m in y) {
                 let field = colKeys[i++];
                 try {
-                    result[field.key] = field.valueConverter.fromDb(y[m]);
+                    result[field.key] = fromDbColumn(field, y[m], this.dp.jsonColumnStorage);
                 }
                 catch (err) {
                     throw new Error('Failed to load from db:' + field.key + '\r\n' + err);
@@ -212,14 +197,14 @@ class KnexEntityDataProvider {
     }
     async update(id, data, options) {
         const e = await this.init();
-        let f = new FilterConsumerBridgeToKnexRequest(e, this.rawSqlWrapIdentifier);
+        let f = new FilterConsumerBridgeToKnexRequest(e, this.rawSqlWrapIdentifier, this.dp.jsonColumnStorage);
         Filter.fromEntityFilter(this.entity, this.entity.idMetadata.getIdFilter(id)).__applyToConsumer(f);
         let updateObject = {};
         for (const x of this.entity.fields) {
             if (isDbReadonly(x, e)) {
             }
             else if (data[x.key] !== undefined) {
-                let v = translateValueAndHandleArrayAndHandleArray(x, data[x.key]);
+                let v = translateValueAndHandleArrayAndHandleArray(x, data[x.key], this.dp.jsonColumnStorage);
                 if (v !== undefined) {
                     let key = await e.$dbNameOf(x);
                     updateObject[key] = v;
@@ -238,7 +223,7 @@ class KnexEntityDataProvider {
         if (ids.length === 0)
             return;
         const e = await this.init();
-        let f = new FilterConsumerBridgeToKnexRequest(e, this.rawSqlWrapIdentifier);
+        let f = new FilterConsumerBridgeToKnexRequest(e, this.rawSqlWrapIdentifier, this.dp.jsonColumnStorage);
         Filter.fromEntityFilter(this.entity, this.entity.idMetadata.getIdFilter(...ids)).__applyToConsumer(f);
         let where = await f.resolveWhere();
         await this.getEntityFrom(e)
@@ -257,7 +242,7 @@ class KnexEntityDataProvider {
             return result;
         }
         const e = await this.init();
-        const batches = splitByMaxParameters(data, (row) => countKnexInsertBindParameters(this.entity, e, row), knexMaxParametersInOneSqlStatement(this.knex));
+        const batches = splitByMaxParameters(data, (row) => countKnexInsertBindParameters(this.entity, e, row, this.dp.jsonColumnStorage), knexMaxParametersInOneSqlStatement(this.knex));
         const result = [];
         for (const batch of batches)
             result.push(...(await this.insertBatch(batch, e, options)));
@@ -268,15 +253,14 @@ class KnexEntityDataProvider {
         for (const x of this.entity.fields) {
             if (isDbReadonly(x, e))
                 continue;
-            if (batch.some((row) => translateValueAndHandleArrayAndHandleArray(x, row[x.key]) !=
-                undefined))
+            if (batch.some((row) => translateValueAndHandleArrayAndHandleArray(x, row[x.key], this.dp.jsonColumnStorage) != undefined))
                 cols.push(x);
         }
         const insertObjects = [];
         for (const row of batch) {
             const insertObject = {};
             for (const x of cols) {
-                const v = translateValueAndHandleArrayAndHandleArray(x, row[x.key]);
+                const v = translateValueAndHandleArrayAndHandleArray(x, row[x.key], this.dp.jsonColumnStorage);
                 const key = await e.$dbNameOf(x);
                 if (v != undefined)
                     insertObject[key] = v;
@@ -291,9 +275,7 @@ class KnexEntityDataProvider {
         const wantRows = options?.select !== 'none';
         if (isAutoIncrement(this.entity.idMetadata.field) && wantRows) {
             const result = await insert.returning(this.entity.idMetadata.field.dbName);
-            const ids = result.map((r) => r && typeof r === 'object'
-                ? r[this.entity.idMetadata.field.dbName]
-                : r);
+            const ids = result.map((r) => r && typeof r === 'object' ? r[this.entity.idMetadata.field.dbName] : r);
             return this.loadRowsByIds(ids);
         }
         await insert;
@@ -321,7 +303,7 @@ class KnexEntityDataProvider {
             if (isDbReadonly(x, e)) {
             }
             else {
-                let v = translateValueAndHandleArrayAndHandleArray(x, data[x.key]);
+                let v = translateValueAndHandleArrayAndHandleArray(x, data[x.key], this.dp.jsonColumnStorage);
                 if (v != undefined) {
                     let key = await e.$dbNameOf(x);
                     insertObject[key] = v;
@@ -357,6 +339,7 @@ class KnexEntityDataProvider {
 class FilterConsumerBridgeToKnexRequest {
     innerNameProvider;
     rawSqlWrapIdentifier;
+    jsonColumnStorage;
     _addWhere = true;
     promises = [];
     result = [];
@@ -370,9 +353,10 @@ class FilterConsumerBridgeToKnexRequest {
         }
         return this.result;
     }
-    constructor(innerNameProvider, rawSqlWrapIdentifier) {
+    constructor(innerNameProvider, rawSqlWrapIdentifier, jsonColumnStorage) {
         this.innerNameProvider = innerNameProvider;
         this.rawSqlWrapIdentifier = rawSqlWrapIdentifier;
+        this.jsonColumnStorage = jsonColumnStorage;
     }
     custom(key, customItem) {
         throw new Error('Custom filter should be translated before it gets here');
@@ -381,7 +365,7 @@ class FilterConsumerBridgeToKnexRequest {
         this.promises.push((async () => {
             const result = [];
             for (const element of orElements) {
-                let f = new FilterConsumerBridgeToKnexRequest(this.innerNameProvider, this.rawSqlWrapIdentifier);
+                let f = new FilterConsumerBridgeToKnexRequest(this.innerNameProvider, this.rawSqlWrapIdentifier, this.jsonColumnStorage);
                 f._addWhere = false;
                 element.__applyToConsumer(f);
                 let where = await f.resolveWhere();
@@ -402,7 +386,7 @@ class FilterConsumerBridgeToKnexRequest {
     }
     not(element) {
         this.promises.push((async () => {
-            let f = new FilterConsumerBridgeToKnexRequest(this.innerNameProvider, this.rawSqlWrapIdentifier);
+            let f = new FilterConsumerBridgeToKnexRequest(this.innerNameProvider, this.rawSqlWrapIdentifier, this.jsonColumnStorage);
             f._addWhere = false;
             element.__applyToConsumer(f);
             let where = await f.resolveWhere();
@@ -424,7 +408,7 @@ class FilterConsumerBridgeToKnexRequest {
         this.result.push((b) => b.whereNotNull(this.innerNameProvider.$dbNameOf(col)));
     }
     isIn(col, val) {
-        this.result.push((knex) => knex.whereIn(this.innerNameProvider.$dbNameOf(col), val.map((x) => translateValueAndHandleArrayAndHandleArray(col, x))));
+        this.result.push((knex) => knex.whereIn(this.innerNameProvider.$dbNameOf(col), val.map((x) => translateValueAndHandleArrayAndHandleArray(col, x, this.jsonColumnStorage))));
     }
     isEqualTo(col, val) {
         this.add(col, val, '=');
@@ -445,39 +429,27 @@ class FilterConsumerBridgeToKnexRequest {
         this.add(col, val, '<');
     }
     containsCaseInsensitive(col, val) {
-        this.result.push((b) => b.whereRaw('lower (' +
-            b.client.ref(this.innerNameProvider.$dbNameOf(col)) +
-            ") like lower ('%" +
-            val.replace(/'/g, "''") +
-            "%')"));
-        this.promises.push((async () => { })());
+        this.like(col, '%' + escapeLike(val) + '%');
     }
     notContainsCaseInsensitive(col, val) {
-        this.result.push((b) => b.whereRaw('not lower (' +
-            b.client.ref(this.innerNameProvider.$dbNameOf(col)) +
-            ") like lower ('%" +
-            val.replace(/'/g, "''") +
-            "%')"));
-        this.promises.push((async () => { })());
+        this.like(col, '%' + escapeLike(val) + '%', 'not ');
     }
     startsWithCaseInsensitive(col, val) {
-        this.result.push((b) => b.whereRaw('lower (' +
-            b.client.ref(this.innerNameProvider.$dbNameOf(col)) +
-            ") like lower ('" +
-            val.replace(/'/g, "''") +
-            "%')"));
-        this.promises.push((async () => { })());
+        this.like(col, escapeLike(val) + '%');
     }
     endsWithCaseInsensitive(col, val) {
-        this.result.push((b) => b.whereRaw('lower (' +
+        this.like(col, '%' + escapeLike(val));
+    }
+    like(col, pattern, not = '') {
+        this.result.push((b) => b.whereRaw(not +
+            'lower (' +
             b.client.ref(this.innerNameProvider.$dbNameOf(col)) +
-            ") like lower ('%" +
-            val.replace(/'/g, "''") +
-            "')"));
+            ') like lower (?)' +
+            LIKE_ESCAPE_SQL, [pattern]));
         this.promises.push((async () => { })());
     }
     add(col, val, operator) {
-        this.result.push((b) => b.where(this.innerNameProvider.$dbNameOf(col), operator, translateValueAndHandleArrayAndHandleArray(col, val)));
+        this.result.push((b) => b.where(this.innerNameProvider.$dbNameOf(col), operator, translateValueAndHandleArrayAndHandleArray(col, val, this.jsonColumnStorage)));
     }
     databaseCustom(databaseCustom) {
         this.promises.push((async () => {
@@ -486,7 +458,7 @@ class FilterConsumerBridgeToKnexRequest {
             }
             if (databaseCustom?.buildSql) {
                 let r = new KnexCommandHelper();
-                const item = new CustomSqlFilterBuilder(r, this.rawSqlWrapIdentifier);
+                const item = new CustomSqlFilterBuilder(r, this.rawSqlWrapIdentifier, this.jsonColumnStorage);
                 let sql = await databaseCustom.buildSql(item);
                 if (typeof sql !== 'string')
                     sql = item.sql;
@@ -691,18 +663,19 @@ function splitByMaxParameters(items, parametersInItem, maxParameters) {
         batches.push(batch);
     return batches;
 }
-function countKnexInsertBindParameters(entity, e, row) {
+function countKnexInsertBindParameters(entity, e, row, storage) {
     let n = 0;
     for (const x of entity.fields) {
         if (isDbReadonly(x, e))
             continue;
-        if (translateValueAndHandleArrayAndHandleArray(x, row[x.key]) != undefined)
+        if (translateValueAndHandleArrayAndHandleArray(x, row[x.key], storage) !=
+            undefined)
             n++;
     }
     return n;
 }
-function translateValueAndHandleArrayAndHandleArray(field, val) {
-    let result = field.valueConverter.toDb(val);
+function translateValueAndHandleArrayAndHandleArray(field, val, storage) {
+    let result = toDbColumn(field, val, storage);
     if (Array.isArray(result))
         return JSON.stringify(result);
     return result;

package/esm/src/data-api.js

@@ -2,6 +2,7 @@ import { doTransaction } from './context.js';
 import { EntityError } from './data-interfaces.js';
 import { findOptionsToJson, liveQueryAction, } from './data-providers/rest-data-provider.js';
 import { Filter, buildFilterFromRequestParameters, customUrlToken, } from './filter/filter-interfaces.js';
+import { omitFilterField, walkFilter } from './filter/walk-filter.js';
 import { getRelationFieldInfo } from './remult3/relationInfoMember.js';
 import { GroupByCountMember, GroupByForApiKey, } from './remult3/remult3.js';
 import { ForbiddenError } from './server-action.js';
@@ -27,10 +28,7 @@ export class DataApi {
             return this.getArray(res, req, undefined);
         }
         catch (err) {
-            if (err.isForbiddenError)
-                res.forbidden(err.message);
-            else
-                res.error(err, this.repository.metadata);
+            this.handleError(res, err);
         }
     }
     async httpPost(res, req, body, serializeContext) {
@@ -76,10 +74,7 @@ export class DataApi {
             }
         }
         catch (err) {
-            if (err.isForbiddenError)
-                res.forbidden(err.message);
-            else
-                res.error(err, this.repository.metadata);
+            this.handleError(res, err);
         }
     }
     async upsertMany(response, request, body) {
@@ -130,10 +125,7 @@ export class DataApi {
             });
         }
         catch (err) {
-            if (err.isForbiddenError)
-                response.forbidden();
-            else
-                response.error(err, this.repository.metadata);
+            this.handleError(response, err);
         }
     }
     static defaultGetLimit = 0;
@@ -155,7 +147,7 @@ export class DataApi {
             });
         }
         catch (err) {
-            response.error(err, this.repository.metadata);
+            this.handleError(response, err);
         }
     }
     async deleteMany(response, request, body) {
@@ -175,7 +167,7 @@ export class DataApi {
             });
         }
         catch (err) {
-            response.error(err, this.repository.metadata);
+            this.handleError(response, err);
         }
     }
     async prepareWhereForManyOperation(body, request) {
@@ -196,7 +188,7 @@ export class DataApi {
             for (const element of body?.orderBy) {
                 const direction = element.isDescending ? 'desc' : 'asc';
                 if (element.operation !== 'count' &&
-                    !this.repository.fields.find(element.field)?.includedInApi())
+                    !this.fieldIsInApi(element.field))
                     continue;
                 switch (element.operation) {
                     case undefined:
@@ -214,9 +206,9 @@ export class DataApi {
                 }
             }
         }
-        const group = body?.groupBy?.filter((x) => this.repository.fields.find(x).includedInApi());
-        const min = body?.min?.filter((x) => this.repository.fields.find(x).includedInApi());
-        const max = body?.max?.filter((x) => this.repository.fields.find(x).includedInApi());
+        const group = this.fieldsInApi(body?.groupBy);
+        const min = this.fieldsInApi(body?.min);
+        const max = this.fieldsInApi(body?.max);
         let result = await this.repository.groupBy({
             where: findOptions.where,
             limit: findOptions.limit,
@@ -224,11 +216,11 @@ export class DataApi {
             //@ts-expect-error internal key
             [GroupByForApiKey]: true,
             group,
-            sum: body?.sum?.filter((x) => this.repository.fields.find(x).includedInApi()),
-            avg: body?.avg?.filter((x) => this.repository.fields.find(x).includedInApi()),
+            sum: this.fieldsInApi(body?.sum),
+            avg: this.fieldsInApi(body?.avg),
             min,
             max,
-            distinctCount: body?.distinctCount?.filter((x) => this.repository.fields.find(x).includedInApi()),
+            distinctCount: this.fieldsInApi(body?.distinctCount),
             orderBy: orderBy,
         });
         if (group)
@@ -269,7 +261,7 @@ export class DataApi {
             let sort = request.get('_sort');
             if (sort != undefined) {
                 let dir = request.get('_order');
-                findOptions.orderBy = determineSort(sort, dir);
+                findOptions.orderBy = this.omitFieldsHiddenFromApi(determineSort(sort, dir));
             }
             let limit = +request.get('_limit');
             if (!limit && DataApi.defaultGetLimit)
@@ -341,10 +333,7 @@ export class DataApi {
             response.success(r);
         }
         catch (err) {
-            if (err.isForbiddenError)
-                response.forbidden(err.message);
-            else
-                response.error(err, this.repository.metadata);
+            this.handleError(response, err);
         }
     }
     async liveQuery(response, request, body, serializeContext, queryChannel) {
@@ -368,10 +357,7 @@ export class DataApi {
             response.success(r.r);
         }
         catch (err) {
-            if (err.isForbiddenError)
-                response.forbidden();
-            else
-                response.error(err, this.repository.metadata);
+            this.handleError(response, err);
         }
     }
     async buildWhere(request, body) {
@@ -396,6 +382,7 @@ export class DataApi {
                     return result;
                 },
             });
+            f = this.omitFieldsHiddenFromApi(f);
             if (this.repository.metadata.options.apiPreprocessFilter) {
                 f = await this.repository.metadata.options.apiPreprocessFilter(f, {
                     metadata: this.repository.metadata,
@@ -408,6 +395,30 @@ export class DataApi {
         }
         return { $and: where };
     }
+    // A hidden field is not on the api, same as a field that is not on the entity.
+    fieldIsInApi(key) {
+        return this.repository.fields.find(key)?.includedInApi() === true;
+    }
+    fieldsInApi(keys) {
+        return keys?.filter((key) => this.fieldIsInApi(key));
+    }
+    // Drop a field the api hides. Same as a field that is not on the entity:
+    // refusing it would confirm the name exists.
+    omitFieldsHiddenFromApi(filterOrSort) {
+        const fields = this.repository.metadata.fields;
+        return walkFilter(filterOrSort, function (key, value) {
+            if (!key.startsWith(customUrlToken) &&
+                fields.find(key)?.includedInApi() === false)
+                return omitFilterField;
+            return value;
+        });
+    }
+    handleError(response, err) {
+        if (err.isForbiddenError)
+            response.forbidden(err.message);
+        else
+            response.error(err, this.repository.metadata);
+    }
     async doOnId(response, id, what) {
         try {
             var where = [
@@ -467,7 +478,7 @@ export class DataApi {
             });
         }
         catch (err) {
-            response.error(err, this.repository.metadata);
+            this.handleError(response, err);
         }
     }
     async actualUpdate(row, body, req, doNotSelect) {