remult 0.25.8-exp.14 → 0.25.8-exp.16

package/esm/migrations/compare-migration-snapshots.js

@@ -1,15 +1,18 @@
 import { repo } from '../index.js';
 import { dbNamesOf, shouldCreateEntity, shouldNotCreateField, } from '../src/filter/filter-consumer-bridge-to-sql-request.js';
-export async function compareMigrationSnapshot({ entities, snapshot, migrationBuilder: migrationBuilder, }) {
+export async function compareMigrationSnapshot({ entities, snapshot, migrationBuilder, reporter, }) {
     snapshot = JSON.parse(JSON.stringify(snapshot));
+    let processedEntities = new Set();
     for (const entity of entities) {
         const meta = repo(entity).metadata;
         const e = await dbNamesOf(meta, (x) => x);
+        processedEntities.add(e.$entityName);
         if (shouldCreateEntity(meta, e)) {
             let entitySnapshot = snapshot.entities[e.$entityName];
             let createColumns = true;
             if (!entitySnapshot) {
                 createColumns = false;
+                reporter('create table ' + e.$entityName);
                 await migrationBuilder.createTable(meta);
                 entitySnapshot = snapshot.entities[e.$entityName] = {
                     key: meta.key,
@@ -17,16 +20,35 @@ export async function compareMigrationSnapshot({ entities, snapshot, migrationBu
                     columns: {},
                 };
             }
+            const processedColumns = new Set();
             for (const field of meta.fields) {
                 if (!shouldNotCreateField(field, e)) {
-                    const column = entitySnapshot.columns[e.$dbNameOf(field)];
+                    const columnDbName = e.$dbNameOf(field);
+                    processedColumns.add(columnDbName);
+                    const column = entitySnapshot.columns[columnDbName];
                     if (!column) {
-                        if (createColumns)
+                        if (createColumns) {
+                            reporter('add column ' + e.$entityName + '.' + columnDbName);
                             await migrationBuilder.addColumn(meta, field);
-                        entitySnapshot.columns[e.$dbNameOf(field)] = { key: field.key };
+                        }
+                        entitySnapshot.columns[columnDbName] = { key: field.key };
                     }
                 }
             }
+            for (const columnDbName of Object.keys(entitySnapshot.columns)) {
+                if (!processedColumns.has(columnDbName)) {
+                    reporter('remove column ' + e.$entityName + '.' + columnDbName);
+                    await migrationBuilder.removeColumn(e.$entityName, columnDbName);
+                    delete entitySnapshot.columns[columnDbName];
+                }
+            }
+        }
+    }
+    for (const entityDbName of Object.keys(snapshot.entities)) {
+        if (!processedEntities.has(entityDbName)) {
+            reporter('remove table ' + entityDbName);
+            await migrationBuilder.removeTable(entityDbName);
+            delete snapshot.entities[entityDbName];
         }
     }
     return snapshot;

package/esm/migrations/generate-migrations.js

@@ -9,17 +9,16 @@ import { initDataProvider } from '../server/initDataProvider.js';
  * Generates migration scripts based on changes in entities.
  *
  * @param options - Configuration options for generating migrations.
- * @param options.entities - An array of entity classes to be included in the migration.
+ * @param options.entities - An array of entity classes whose changes will be included in the migration.
  * @param options.dataProvider - The data provider instance or a function returning a promise of the data provider.
- * @param options.migrationsFolder - (Optional) The path to the folder where migration scripts will be stored. Default is 'src/server/migrations'.
+ * @param options.migrationsFolder - (Optional) The path to the folder where migration scripts will be stored. Default is 'src/migrations'.
  * @param options.snapshotFile - (Optional) The path to the file where the snapshot of the last known state will be stored. Default is 'migrations-snapshot.json' in the `migrationsFolder`.
  * @param options.migrationsTSFile - (Optional) The path to the TypeScript file where the generated migrations will be written. Default is 'migrations.ts' in the `migrationsFolder`.
- * @param options.endConnection - (Optional) Determines whether to close the database connection after generating migrations. Default is true.
- * @see [Migrations](http://remult.dev/docs/migrations.html)
+ * @param options.endConnection - (Optional) Determines whether to close the database connection after generating migrations. Default is false.
+ * @see [Migrations](https://remult.dev/docs/migrations.html)
  */
 export async function generateMigrations(options) {
-    const migrationDir = options.migrationsFolder ||
-        path.join(process.cwd(), 'src/server/migrations');
+    const migrationDir = options.migrationsFolder || path.join(process.cwd(), 'src/migrations');
     const snapshotFileName = options.snapshotFile || path.join(migrationDir, 'migrations-snapshot.json');
     const migrationsTSFilename = options.migrationsTSFile || path.join(migrationDir, 'migrations.ts');
     const dataProvider = await initDataProvider(options.dataProvider);
@@ -45,19 +44,25 @@ export async function generateMigrations(options) {
     };
     let migrationBuilder = new DefaultMigrationBuilder(code);
     if (isOfType(dataProvider, 'provideMigrationBuilder')) {
-        migrationBuilder = dataProvider.provideMigrationBuilder(code);
+        migrationBuilder = new DefaultMigrationBuilder(code, dataProvider.provideMigrationBuilder(code));
     }
+    let reporter = (what) => console.log('[generateMigrations] ' + what);
     snapshot = await compareMigrationSnapshot({
         entities: options.entities,
         snapshot,
+        reporter,
         migrationBuilder,
     });
     if (steps.length)
         await updateMigrationsFile(migrationsTSFilename, [
             `async (${hasSql ? '{sql}' : ''})=>{\n${steps.join('\n')}\n}`,
         ]);
+    else {
+        reporter('No changes detected');
+    }
     fs.writeFileSync(snapshotFileName, JSON.stringify(snapshot, null, 2));
-    if (options.endConnection !== false && isOfType(dataProvider, 'end')) {
+    if (options.endConnection && isOfType(dataProvider, 'end')) {
         await dataProvider.end();
     }
+    return steps.length > 0;
 }

package/esm/migrations/migrate.js

@@ -10,9 +10,11 @@ import { cast, isOfType } from '../src/isOfType.js';
  * @param options.migrations - An object containing the migration scripts, each keyed by a unique identifier.
  * @param options.dataProvider - The data provider instance or a function returning a promise of the data provider.
  * @param options.migrationsTable - (Optional) The name of the table that tracks applied migrations. Default is '__remult_migrations_version'.
- * @param options.endConnection - (Optional) Determines whether to close the database connection after applying migrations. Default is true.
-* @see [Migrations](http://remult.dev/docs/migrations.html)
-*/
+ * @param options.endConnection - (Optional) Determines whether to close the database connection after applying migrations. Default is false.
+ * @param options.beforeMigration - (Optional) A callback function that is called before each migration is applied. Receives an object with the migration index.
+ * @param options.afterMigration - (Optional) A callback function that is called after each migration is applied. Receives an object with the migration index and the duration of the migration.
+ * @see [Migrations](https://remult.dev/docs/migrations.html)
+ */
 export async function migrate(options) {
     let migrationTableName = options.migrationsTable || '__remult_migrations_version';
     const dataProvider = await initDataProvider(options.dataProvider);
@@ -29,48 +31,60 @@ export async function migrate(options) {
         VersionInfo = __decorate([
             Entity(migrationTableName)
         ], VersionInfo);
-        const repo = new Remult(dataProvider).repo(VersionInfo);
+        const steps = Object.entries(options.migrations).sort(([a], [b]) => parseInt(a) - parseInt(b));
         if (dataProvider.ensureSchema) {
-            await dataProvider.ensureSchema([repo.metadata]);
-        }
-        let v = await repo.findFirst();
-        if (!v) {
-            v = repo.create();
-            v.version = -1;
+            await dataProvider.ensureSchema([
+                new Remult(dataProvider).repo(VersionInfo).metadata,
+            ]);
         }
-        const steps = Object.entries(options.migrations).sort(([a], [b]) => parseInt(a) - parseInt(b));
-        for (const [stepText, action] of steps) {
-            const step = parseInt(stepText);
-            if (step < 0)
-                throw new Error('Migration step number must be a non-negative integer');
-            if (v.version >= step)
-                continue;
-            console.info('Executing migration step ' + step);
-            console.time('Completed migration step ' + step);
-            try {
-                await doTransaction(remult, async (dp) => {
+        await doTransaction(remult, async (dp) => {
+            const repo = new Remult(dataProvider).repo(VersionInfo, dp);
+            let v = await repo.findFirst();
+            if (!v) {
+                v = repo.create();
+                v.version = -1;
+            }
+            for (const [stepText, action] of steps) {
+                const step = parseInt(stepText);
+                if (step < 0)
+                    throw new Error('Migration step number must be a non-negative integer');
+                if (v.version >= step)
+                    continue;
+                if (options.beforeMigration) {
+                    await options.beforeMigration({ index: step });
+                }
+                const startTime = Date.now();
+                console.info('Executing migration step ' + step);
+                console.time('Completed migration step ' + step);
+                try {
                     const utils = {
                         sql: async (sql) => {
-                            await cast(dp, 'execute').execute(sql);
+                            return await cast(dp, 'execute').execute(sql);
                         },
                     };
                     await action(utils);
-                });
-                console.timeEnd('Completed migration step ' + step);
-            }
-            catch (err) {
-                console.error('Failed to execute migration step ' + step);
-                console.error(err);
-                throw err;
+                    if (options.afterMigration) {
+                        await options.afterMigration({
+                            index: step,
+                            duration: Date.now() - startTime,
+                        });
+                    }
+                    console.timeEnd('Completed migration step ' + step);
+                }
+                catch (err) {
+                    console.error('Failed to execute migration step ' + step);
+                    console.error(err);
+                    throw err;
+                }
+                v.version = step;
+                await v.save();
             }
-            v.version = step;
-            await v.save();
-        }
+        });
     }
     finally {
         remult.dataProvider = prev;
     }
-    if (options.endConnection !== false && isOfType(dataProvider, 'end')) {
+    if (options.endConnection && isOfType(dataProvider, 'end')) {
         await dataProvider.end();
     }
 }

package/esm/migrations/migration-types.js

@@ -1,12 +1,36 @@
 export class DefaultMigrationBuilder {
     code;
-    constructor(code) {
+    wrapped;
+    constructor(code, wrapped) {
         this.code = code;
+        this.wrapped = wrapped;
     }
     async createTable(meta) {
-        this.code.addComment('create table ' + meta.entityType.name);
+        if (this.wrapped?.createTable) {
+            await this.wrapped.createTable(meta);
+        }
+        else
+            this.code.addComment('TODO: implement create table ' + meta.dbName);
     }
     async addColumn(meta, field) {
-        this.code.addComment('add column ' + meta.entityType.name + '.' + field.key);
+        if (this.wrapped?.addColumn) {
+            await this.wrapped.addColumn(meta, field);
+        }
+        else
+            this.code.addComment('TODO: implement add column ' + meta.dbName + '.' + field.dbName);
+    }
+    async removeTable(entityDbName) {
+        if (this.wrapped?.removeTable) {
+            await this.wrapped.removeTable(entityDbName);
+        }
+        else
+            this.code.addComment('TODO: implement remove table ' + entityDbName);
+    }
+    async removeColumn(entityName, columnDbName) {
+        if (this.wrapped?.removeColumn) {
+            await this.wrapped.removeColumn(entityName, columnDbName);
+        }
+        else
+            this.code.addComment('TODO: implement remove column ' + entityName + '.' + columnDbName);
     }
 }

package/esm/migrations/update-migrations-ts-file.js

@@ -1,6 +1,9 @@
 // consider replacing with https://www.npmjs.com/package/ts-morph
 export async function updateMigrationsFile(filePath, steps) {
-    const ts = await import('typescript');
+    let ts = await import('typescript');
+    //@ts-ignore
+    if (ts.default)
+        ts = ts.default;
     const fs = await import('fs');
     let maxStep = -1;
     function buildSteps() {

package/esm/remult-knex/index.js

@@ -507,16 +507,24 @@ export function buildColumn(x, dbName, b, supportsJson = true) {
         else if (x.valueConverter.fieldTypeInDb == 'json')
             if (supportsJson)
                 b.json(dbName);
-            else
-                b.text(dbName);
+            else {
+                let c = b.text(dbName);
+                if (!x.allowNull)
+                    c.defaultTo('').notNullable();
+            }
         else
             b.specificType(dbName, x.valueConverter.fieldTypeInDb);
     }
-    else {
+    else if (x.valueType === String) {
         let c = b.string(dbName, x.options.maxLength);
         if (!x.allowNull)
             c.defaultTo('').notNullable();
     }
+    else {
+        let c = b.text(dbName);
+        if (!x.allowNull)
+            c.defaultTo('').notNullable();
+    }
 }
 function logSql(who) {
     if (KnexSchemaBuilder.logToConsole)
@@ -547,7 +555,7 @@ class KnexBridgeToSQLCommand {
     param(val) {
         if (Array.isArray(val))
             val = JSON.stringify(val);
-        const key = ':' + (this.i++);
+        const key = ':' + this.i++;
         this.values[key.substring(1)] = val;
         return key;
     }

package/esm/src/filter/filter-interfaces.js

@@ -64,6 +64,27 @@ export class Filter {
         await Filter.fromEntityFilter(metadata, filter).__applyToConsumer(result);
         return result;
     }
+    /**
+     * Retrieves information about a filter, including precise values for each property.
+     * @template entityType The type of the entity being filtered.
+     * @param metadata The metadata of the entity being filtered.
+     * @param filter The filter to analyze.
+     * @returns A promise that resolves to a FilterInfo object containing the filter information.
+     * @example
+     * const info = await where.getInfo();
+     * console.log(info.preciseValues);
+     * // Output:
+     * // {
+     * //   "customerId": ["1", "2", "3"], // Precise values inferred from the filter
+     * //   "status": undefined,           // Cannot infer precise values for 'status'
+     * // }
+    
+     */
+    async getInfo() {
+        const result = new preciseValuesCollector();
+        await this.__applyToConsumer(result);
+        return result;
+    }
     static createCustom(rawFilterTranslator, key = '') {
         let rawFilterInfo = { key: key, rawFilterTranslator };
         return Object.assign((x) => {
@@ -248,7 +269,6 @@ export class Filter {
             return await filter();
         return filter;
     }
-    //@internal
     toJson() {
         let r = new FilterSerializer();
         this.__applyToConsumer(r);
@@ -769,24 +789,55 @@ export function __updateEntityBasedOnWhere(entityDefs, where, r) {
     }
 }
 class preciseValuesCollector {
-    bad = new Set();
-    preciseValues = {};
+    rawValues = {};
+    preciseValues = new Proxy(this.rawValues, {
+        get: (target, prop) => {
+            if (prop in target) {
+                let result = target[prop];
+                if (result.bad)
+                    return undefined;
+                if (result.values.length > 0) {
+                    const relInfo = getRelationFieldInfo(result.field);
+                    if (relInfo) {
+                        if (relInfo.type === 'reference') {
+                            return result.values.map((x) => {
+                                return relInfo.toRepo.metadata.idMetadata.getIdFilter(x);
+                            });
+                        }
+                        else
+                            throw new Error('Only relations toOne without field are supported.');
+                    }
+                    return result.values;
+                }
+            }
+            return undefined;
+        },
+    });
     ok(col, ...val) {
-        if (this.bad.has(col))
-            return;
-        let x = this.preciseValues[col];
+        let x = this.rawValues[col.key];
         if (!x) {
-            this.preciseValues[col] = [...val];
+            this.rawValues[col.key] = {
+                field: col,
+                bad: false,
+                values: [...val],
+            };
         }
         else {
-            x.push(...val.filter((y) => !x.includes(y)));
+            x.values.push(...val.filter((y) => !x.values.includes(y)));
         }
     }
     notOk(col) {
-        if (this.bad.has(col))
-            return;
-        this.bad.add(col);
-        this.preciseValues[col] = undefined;
+        let x = this.rawValues[col.key];
+        if (!x) {
+            this.rawValues[col.key] = {
+                field: col,
+                bad: true,
+                values: [],
+            };
+        }
+        else {
+            x.bad = true;
+        }
     }
     or(orElements) {
         const result = orElements.map((or) => {
@@ -795,59 +846,61 @@ class preciseValuesCollector {
             return x;
         });
         for (const or of result) {
-            for (const key in or.preciseValues) {
-                if (Object.prototype.hasOwnProperty.call(or.preciseValues, key)) {
-                    const element = or.preciseValues[key];
-                    if (element)
-                        this.ok(key, ...element);
+            for (const key in or.rawValues) {
+                if (Object.prototype.hasOwnProperty.call(or.rawValues, key)) {
+                    const element = or.rawValues[key];
+                    if (element) {
+                        if (element.bad)
+                            this.notOk(element.field);
+                        else {
+                            this.ok(element.field, ...element.values);
+                        }
+                    }
                 }
             }
-            for (const key of or.bad) {
-                this.notOk(key);
-            }
         }
-        for (const key in this.preciseValues) {
-            if (Object.prototype.hasOwnProperty.call(this.preciseValues, key)) {
+        for (const key in this.rawValues) {
+            if (Object.prototype.hasOwnProperty.call(this.rawValues, key)) {
                 for (const r of result) {
-                    const element = r.preciseValues[key];
+                    const element = r.rawValues[key];
                     if (!element)
-                        this.notOk(key);
+                        this.notOk(this.rawValues[key].field);
                 }
             }
         }
     }
     isEqualTo(col, val) {
-        this.ok(col.key, val);
+        this.ok(col, val);
     }
     isDifferentFrom(col, val) {
-        this.notOk(col.key);
+        this.notOk(col);
     }
     isNull(col) {
-        this.ok(col.key, null);
+        this.ok(col, null);
     }
     isNotNull(col) {
-        this.notOk(col.key);
+        this.notOk(col);
     }
     isGreaterOrEqualTo(col, val) {
-        this.notOk(col.key);
+        this.notOk(col);
     }
     isGreaterThan(col, val) {
-        this.notOk(col.key);
+        this.notOk(col);
     }
     isLessOrEqualTo(col, val) {
-        this.notOk(col.key);
+        this.notOk(col);
     }
     isLessThan(col, val) {
-        this.notOk(col.key);
+        this.notOk(col);
     }
     containsCaseInsensitive(col, val) {
-        this.notOk(col.key);
+        this.notOk(col);
     }
     notContainsCaseInsensitive(col, val) {
-        this.notOk(col.key);
+        this.notOk(col);
     }
     isIn(col, val) {
-        this.ok(col.key, ...val);
+        this.ok(col, ...val);
     }
     custom(key, customItem) { }
     databaseCustom(databaseCustom) { }

package/esm/src/remult3/RepositoryImplementation.js

@@ -638,7 +638,7 @@ export class RepositoryImplementation {
     async count(where) {
         return this._edp.count(await this._translateWhereToFilter(where));
     }
-    async deleteMany({ where }) {
+    async deleteMany({ where, }) {
         Filter.throwErrorIfFilterIsEmpty(where, 'deleteMany');
         if (this._dataProvider.isProxy) {
             return this._edp.deleteMany(await this._translateWhereToFilter(where));
@@ -775,13 +775,16 @@ export class RepositoryImplementation {
             if (this.metadata.options.backendPreprocessFilter) {
                 where = await this.metadata.options.backendPreprocessFilter(where, {
                     metadata: this.metadata,
-                    getFilterInfo: (filter) => Filter.getInfo(this.metadata, filter || where)
+                    getFilterInfo: (filter) => Filter.getInfo(this.metadata, filter || where),
                 });
             }
             if (this.metadata.options.backendPrefilter) {
                 let z = where;
                 where = {
-                    $and: [z, await Filter.resolve(this.metadata.options.backendPrefilter)],
+                    $and: [
+                        z,
+                        await Filter.resolve(this.metadata.options.backendPrefilter),
+                    ],
                 };
             }
         }
@@ -1934,9 +1937,13 @@ class EntityFullInfo {
         getIdFilter: (...ids) => {
             if (this.idMetadata.field instanceof CompoundIdField) {
                 let field = this.idMetadata.field;
-                return {
-                    $or: ids.map((x) => field.isEqualTo(x)),
-                };
+                if (ids.length == 1) {
+                    return field.isEqualTo(ids[0]);
+                }
+                else
+                    return {
+                        $or: ids.map((x) => field.isEqualTo(x)),
+                    };
             }
             if (ids.length == 1)
                 return {
@@ -1971,7 +1978,13 @@ export function ValueListFieldType(...options) {
             ;
             (o.valueConverter = ValueListInfo.get(type)),
                 (o.displayValue = (item, val) => val?.caption);
-            o.validate = (entity, ref) => Validators.in(ValueListInfo.get(type).getValues())(entity, ref);
+            o.validate = (entity, ref) => {
+                const values = ValueListInfo.get(type).getValues();
+                if (ref.value && !values.find((v) => v === ref.value)) {
+                    ref.value = values.find((v) => v.id === ref.value.id) || ref.value;
+                }
+                return Validators.in(values)(entity, ref);
+            };
         }, ...options)(type, context);
     };
 }

package/esm/src/remult3/remult3.js

@@ -1,24 +1,23 @@
 export {};
-//p1 - It's good that there is no migrations added. (I also don't like automatic deletion), and it can be done manually if needed.
-//p1 - Maybe migrations-snapshots.json should have fields deleted to have a one to one representation of the db?
-//p1 - I needed to modify the changes Field of changeLog entity to @Fields.json() to be nvarchar(max) too. Why is @Fields.object() only nvarchar(255) ? Or is it default size and can be adjusted?
-//p1 - Filter.toJson should be public
-//p1 - improve type augmentation doc with specific of the file and with the export trick
-//y1 - JY build issue
-//p1 - JY issue with the generate migration
-//y1 - migration default folder should be source/migrations (also docs?)
-//y1 - discuss the typing issue with relations in this case vs the ValueListFieldTypes
-//y1 - JYC log the migration request
-//y1 - JYC health check approach
+//y1 - JY suggestion of having generate as part of the ensure schema - not sure
+//y1 - currently for string fields we default for 255 in knex (which is their default) why not just use text (varchar max) - and only use exact length when one is provided?
+//p1 - document custom filter without sql
 //p1 - admin url1 - pocketbase use /_ for the admin //{allowed?:Allowed,url?:string}
 //p1 - document new sqlite providers
 //p1 - article auth.js with express
 //p1 - document offline support
 //p1 - fix getting started article with using your own database connection
+//p1 - create foreign key constraints in user code
+//p1 - https://github.com/vuejs/vitepress/issues/3154#issuecomment-1784612839
 //y2 - should we validate relations
 //y1 - dependency of live query tables
+//y1 - consider the bot that updates dependencies
 //y1 - consider replacing all errors with error classes that extend the base Error class
 //y2 - should enforce integer - currently we probably round / truncate it
+//y1 - talk about filter on objects that are not loaded -  {
+//category: repo(CompoundId).create({ company: 7, index: 3, name: '' }),
+//    }
+//y1 - consider id to also support keyof (id:['company','index'])
 /*y1 - talk about modules in init express with entities/controllers,initRequest,initApi
  - support get with backend method, with url search params as the first parameter, & url as second parameter
    - support returning redirect, and plain html (For sign in scenarios)

package/migrations/compare-migration-snapshots.d.ts

@@ -1,8 +1,9 @@
 import type { MigrationBuilder } from './migration-types.js';
-export declare function compareMigrationSnapshot({ entities, snapshot, migrationBuilder: migrationBuilder, }: {
+export declare function compareMigrationSnapshot({ entities, snapshot, migrationBuilder, reporter, }: {
     entities: any[];
     snapshot: EntitiesSnapshot;
-    migrationBuilder: MigrationBuilder;
+    migrationBuilder: Required<MigrationBuilder>;
+    reporter: (what: string) => void;
 }): Promise<{
     version: number;
     entities: Record<string, {