remult 3.4.0-next.2 → 3.4.0-next.5

package/esm/src/data-providers/json-data-provider.js

@@ -49,8 +49,8 @@ class JsonEntityDataProvider {
             return x;
         }))));
     }
-    delete(id) {
-        return (this.p = this.p.then(() => this.loadEntityData((dp, save) => dp.delete(id).then(async (x) => {
+    delete(ids) {
+        return (this.p = this.p.then(() => this.loadEntityData((dp, save) => dp.delete(ids).then(async (x) => {
             await save();
             return x;
         }))));

package/esm/src/data-providers/rest-data-provider.js

@@ -303,21 +303,20 @@ export class RestEntityDataProvider {
         }
         return result;
     }
-    async delete(id) {
-        if (id == '')
-            await this.deleteMany(Filter.fromEntityFilter(this.entity, this.entity.idMetadata.getIdFilter(id)));
-        else
-            return this.http().delete(this.url() + '/' + encodeURIComponent(id));
+    async delete(ids) {
+        if (ids.length === 0)
+            return;
+        if (ids.length === 1 && ids[0] !== '')
+            return this.http().delete(this.url() + '/' + encodeURIComponent(ids[0]));
+        await this.deleteMany(Filter.fromEntityFilter(this.entity, this.entity.idMetadata.getIdFilter(...ids)));
     }
     insert(data, options) {
         return this.http()
-            .post(this.url() + (options?.select === 'none' ? '?_select=$none' : ''), this.translateToJson(data))
-            .then((y) => this.translateFromJson(y));
-    }
-    insertMany(data, options) {
-        return this.http()
-            .post(this.url() + (options?.select === 'none' ? '?_select=$none' : ''), data.map((data) => this.translateToJson(data)))
-            .then((y) => y.map((y) => this.translateFromJson(y)));
+            .post(this.url() + (options?.select === 'none' ? '?_select=$none' : ''), data.map((row) => this.translateToJson(row)))
+            .then((y) => {
+            const rows = Array.isArray(y) ? y : [y];
+            return rows.map((row) => row ? this.translateFromJson(row) : undefined);
+        });
     }
 }
 export function addFilterToUrlAndReturnTrueIfSuccessful(filter, url) {

package/esm/src/data-providers/sql-database.js

@@ -117,6 +117,8 @@ export class SqlDatabase {
                     doesNotSupportReturningSyntax: this.sql.doesNotSupportReturningSyntax,
                     doesNotSupportReturningSyntaxOnlyForUpdate: this.sql.doesNotSupportReturningSyntaxOnlyForUpdate,
                     orderByNullsFirst: this.sql.orderByNullsFirst,
+                    afterMutation: this.sql.afterMutation,
+                    maxParametersInOneSqlStatement: this.sql.maxParametersInOneSqlStatement,
                 }));
             }
             finally {
@@ -193,6 +195,35 @@ export class SqlDatabase {
     createdEntities = [];
     end;
 }
+const defaultMaxParametersInOneSqlStatement = 2000;
+function splitByMaxParameters(items, parametersInItem, maxParameters) {
+    const batches = [];
+    let batch = [];
+    let params = 0;
+    for (const item of items) {
+        const n = parametersInItem(item);
+        if (batch.length > 0 && params + n > maxParameters) {
+            batches.push(batch);
+            batch = [];
+            params = 0;
+        }
+        batch.push(item);
+        params += n;
+    }
+    if (batch.length)
+        batches.push(batch);
+    return batches;
+}
+function countInsertBindParameters(entity, e, row) {
+    let n = 0;
+    for (const x of entity.fields) {
+        if (isDbReadonly(x, e))
+            continue;
+        if (x.valueConverter.toDb(row[x.key]) != undefined)
+            n++;
+    }
+    return n;
+}
 const icons = new Map([
     // CRUD
     ['INSERT', '⚪'], // Used to insert new data into a database.
@@ -441,11 +472,13 @@ class ActualSQLEntityDataProvider {
             return this.buildResultRow(colKeys, sqlResult.rows[0], sqlResult);
         });
     }
-    async delete(id) {
+    async delete(ids) {
+        if (ids.length === 0)
+            return;
         let e = await this.init();
         let r = this.sql.createCommand();
         let f = new FilterConsumerBridgeToSqlRequest(r, e);
-        Filter.fromEntityFilter(this.entity, this.entity.idMetadata.getIdFilter(id)).__applyToConsumer(f);
+        Filter.fromEntityFilter(this.entity, this.entity.idMetadata.getIdFilter(...ids)).__applyToConsumer(f);
         let statement = 'delete from ' + e.$entityName;
         statement += await f.resolveWhere();
         return r.execute(statement).then(() => {
@@ -453,56 +486,90 @@ class ActualSQLEntityDataProvider {
         });
     }
     async insert(data, options) {
+        if (data.length === 0)
+            return [];
         let e = await this.init();
+        const maxParams = this.strategy.maxParametersInOneSqlStatement ??
+            defaultMaxParametersInOneSqlStatement;
+        const batches = splitByMaxParameters(data, (row) => countInsertBindParameters(this.entity, e, row), maxParams);
+        const result = [];
+        for (const batch of batches)
+            result.push(...(await this.insertBatch(batch, e, options)));
+        return result;
+    }
+    //@internal
+    async insertOne(data, options) {
+        return (await this.insert([data], options))[0];
+    }
+    async insertBatch(batch, e, options) {
         let r = this.sql.createCommand();
-        let cols = '';
-        let vals = '';
-        let added = false;
+        const cols = [];
         for (const x of this.entity.fields) {
-            if (isDbReadonly(x, e)) {
-            }
-            else {
-                let v = x.valueConverter.toDb(data[x.key]);
-                if (v != undefined) {
-                    if (!added)
-                        added = true;
-                    else {
-                        cols += ', ';
-                        vals += ', ';
-                    }
-                    cols += e.$dbNameOf(x);
+            if (isDbReadonly(x, e))
+                continue;
+            if (batch.some((row) => x.valueConverter.toDb(row[x.key]) != undefined))
+                cols.push(x);
+        }
+        let colSql = '';
+        let vals = '';
+        for (let i = 0; i < cols.length; i++) {
+            if (i)
+                colSql += ', ';
+            colSql += e.$dbNameOf(cols[i]);
+        }
+        for (let i = 0; i < batch.length; i++) {
+            if (i)
+                vals += ',';
+            vals += '(';
+            for (let j = 0; j < cols.length; j++) {
+                if (j)
+                    vals += ', ';
+                const x = cols[j];
+                const v = x.valueConverter.toDb(batch[i][x.key]);
+                if (v != undefined)
                     vals += toDbSql(r, x, v);
-                }
+                else if (x.allowNull)
+                    vals += 'null';
+                else
+                    vals += 'DEFAULT';
             }
+            vals += ')';
         }
-        let statement = `insert into ${e.$entityName} (${cols}) values (${vals})`;
+        let statement = `insert into ${e.$entityName} (${colSql}) values ${vals}`;
         let { colKeys, select } = await this.buildSelect(e, r, undefined, undefined);
-        if (!this.sql._getSourceSql().doesNotSupportReturningSyntax &&
-            !(options?.select === 'none'))
+        const source = this.sql._getSourceSql();
+        const wantRows = options?.select !== 'none';
+        if (!source.doesNotSupportReturningSyntax && wantRows)
             statement += ' returning ' + select;
-        return await r.execute(statement).then((sql) => {
-            this.sql._getSourceSql().afterMutation?.();
-            if (this.sql._getSourceSql().doesNotSupportReturningSyntax) {
-                if (isAutoIncrement(this.entity.idMetadata.field)) {
-                    const id = sql.rows[0];
-                    if (typeof id !== 'number')
-                        throw new Error('Auto increment, for a database that is does not support returning syntax, should return an array with the single last added id. Instead it returned: ' +
-                            JSON.stringify(id));
-                    if (options?.select === 'none')
-                        return undefined;
-                    return this.find({
-                        where: new Filter((x) => x.isEqualTo(this.entity.idMetadata.field, id)),
-                    }).then((r) => r[0]);
-                }
-                else {
-                    if (options?.select === 'none')
-                        return undefined;
-                    return getRowAfterUpdate(this.entity, this, data, undefined, 'insert');
-                }
+        const sql = await r.execute(statement);
+        source.afterMutation?.();
+        if (!wantRows)
+            return batch.map(() => undefined);
+        if (source.doesNotSupportReturningSyntax) {
+            if (isAutoIncrement(this.entity.idMetadata.field)) {
+                const lastId = sql.rows[0];
+                if (typeof lastId !== 'number')
+                    throw new Error('Auto increment, for a database that is does not support returning syntax, should return an array with the single last added id. Instead it returned: ' +
+                        JSON.stringify(lastId));
+                const ids = [];
+                for (let id = lastId - batch.length + 1; id <= lastId; id++)
+                    ids.push(id);
+                return this.loadRowsByIds(ids);
             }
-            if (options?.select === 'none')
-                return undefined;
-            return this.buildResultRow(colKeys, sql.rows[0], sql);
+            return this.loadRowsByIds(batch.map((row) => this.entity.idMetadata.getId(row)));
+        }
+        return sql.rows.map((row) => this.buildResultRow(colKeys, row, sql));
+    }
+    async loadRowsByIds(ids) {
+        const found = await this.find({
+            where: Filter.fromEntityFilter(this.entity, this.entity.idMetadata.getIdFilter(...ids)),
+        });
+        const byId = new Map(found.map((row) => [this.entity.idMetadata.getId(row) + '', row]));
+        return ids.map((id) => {
+            const row = byId.get(id + '');
+            if (!row)
+                throw new Error(`Failed to insert row - result contained ${found.length} rows`);
+            return row;
         });
     }
 }

package/esm/src/live-query/LiveQueryClient.js

@@ -46,7 +46,11 @@ export class LiveQueryClient {
                     });
                 }
                 catch (err) {
-                    onResult.error(err);
+                    // drop the half-registered channel so the next subscribe retries
+                    // instead of joining a channel that was never subscribed
+                    this.channels.delete(key);
+                    q.listeners.forEach((l) => l.error(err));
+                    this.closeIfNoListeners();
                     throw err;
                 }
             }
@@ -98,6 +102,8 @@ export class LiveQueryClient {
             if (!q) {
                 this.queries.set(eventTypeKey, (q = new LiveQuerySubscriber(repo, { entityKey: repo.metadata.key, options }, this.getUserId())));
                 q.subscribeCode = () => {
+                    q.snapshotReady = false;
+                    q.droppedWhilePending = false;
                     q.unsubscribeChannel();
                     let unsubscribeToChannel = () => { };
                     q.unsubscribeChannel = () => {
@@ -119,7 +125,10 @@ export class LiveQueryClient {
                             this.runPromise(r.unsubscribe());
                         };
                         snapshotDone = true;
-                        return this.runPromise(q.setAllItems(r.result));
+                        return this.runPromise(q.setAllItems(r.result).then(() => {
+                            if (q.droppedWhilePending)
+                                return this.runKeepAlive();
+                        }));
                     };
                     this.runPromise(this.subscribeChannel(q.queryChannel, {
                         next: (value) => this.runPromise(q.handle(value)),
@@ -135,7 +144,7 @@ export class LiveQueryClient {
                         }
                         unsubscribeToChannel = unsub;
                         if (snapshotDone)
-                            return this.runPromise(this.runKeepAlive({ checkVersions: true }));
+                            return this.runPromise(this.runKeepAlive());
                     })
                         .catch((err) => {
                         q.listeners.forEach((l) => l.error(err));
@@ -185,21 +194,24 @@ export class LiveQueryClient {
             return false;
         return Date.now() - conn.lastServerEvent > flags.sseStaleMs;
     }
-    async runKeepAlive(opts) {
+    async runKeepAlive() {
         if (this.keepAliveInFlight)
             return this.keepAliveInFlight;
-        this.keepAliveInFlight = this.sendKeepAlive(opts).finally(() => {
+        this.keepAliveInFlight = this.sendKeepAlive().finally(() => {
             this.keepAliveInFlight = undefined;
         });
         return this.keepAliveInFlight;
     }
-    async sendKeepAlive(opts) {
+    /** Resolves true when a query was refetched. */
+    async sendKeepAlive() {
         const ids = [];
         for (const q of this.queries.values()) {
+            if (!q.snapshotReady)
+                continue;
             ids.push(q.queryChannel);
         }
         if (ids.length === 0)
-            return;
+            return false;
         let p = this.apiProvider();
         const raw = await this.runPromise(remultStatic.actionInfo.runActionWithoutBlockingUI(() => buildRestDataProvider(p.httpClient).post(p.url + '/' + liveQueryKeepAliveRoute, { queryIds: ids })));
         const unknownIds = Array.isArray(raw)
@@ -208,12 +220,13 @@ export class LiveQueryClient {
         const versions = Array.isArray(raw)
             ? {}
             : (raw?.versions ?? {});
-        const stale = opts?.checkVersions || this.isSseStale();
         let reloaded = false;
         for (const q of this.queries.values()) {
             const serverVersion = versions[q.queryChannel];
             const unknown = unknownIds.includes(q.queryChannel);
-            const mismatch = stale && serverVersion !== undefined && serverVersion !== q.version;
+            // server returns versions on every keep-alive, so a missed message is
+            // caught even while SSE pings look healthy
+            const mismatch = serverVersion !== undefined && serverVersion !== q.version;
             if (unknown || mismatch) {
                 reloaded = true;
                 if (serverVersion !== undefined)
@@ -221,14 +234,7 @@ export class LiveQueryClient {
                 q.subscribeCode();
             }
         }
-        if (this.isSseStale()) {
-            this.keepAliveBackoffMs = reloaded
-                ? flags.liveQueryPollWhenStaleMs
-                : Math.min(this.keepAliveBackoffMs * 2, flags.liveQueryKeepAliveMs);
-        }
-        else {
-            this.keepAliveBackoffMs = flags.liveQueryPollWhenStaleMs;
-        }
+        return reloaded;
     }
     async maybeKeepAlive() {
         const stale = this.isSseStale();
@@ -242,7 +248,18 @@ export class LiveQueryClient {
         this.lastKeepAliveAt = Date.now();
         if (stale)
             this.openedConnection?.resume?.(true);
-        await this.runKeepAlive();
+        let reloaded = false;
+        try {
+            reloaded = await this.runKeepAlive();
+        }
+        finally {
+            // runs on a failed post too, so an unreachable server is not polled
+            // every second; channel-only clients (no query ids) back off as well
+            if (this.isSseStale())
+                this.keepAliveBackoffMs = reloaded
+                    ? flags.liveQueryPollWhenStaleMs
+                    : Math.min(this.keepAliveBackoffMs * 2, flags.liveQueryKeepAliveMs);
+        }
     }
     detachForeground = () => { };
     attachForeground() {
@@ -255,7 +272,7 @@ export class LiveQueryClient {
                 this.foregroundKeepAliveTimer = undefined;
                 this.keepAliveBackoffMs = flags.liveQueryPollWhenStaleMs;
                 this.lastKeepAliveAt = 0;
-                this.runPromise(this.runKeepAlive({ checkVersions: true }));
+                this.runPromise(this.runKeepAlive());
             }, 300);
         });
         this.detachForeground = () => {
@@ -267,19 +284,30 @@ export class LiveQueryClient {
             this.detachForeground = () => { };
         };
     }
+    startKeepAlive() {
+        // openConnection may reject and be retried with queries still alive
+        if (this.interval !== undefined)
+            return;
+        this.lastKeepAliveAt = Date.now();
+        this.keepAliveBackoffMs = flags.liveQueryPollWhenStaleMs;
+        this.attachForeground();
+        this.interval = setInterval(() => {
+            // a failed post is transient: the backoff covers it, next tick retries
+            this.runPromise(this.maybeKeepAlive().catch(() => { }));
+        }, Math.min(flags.liveQueryPollWhenStaleMs, flags.liveQueryKeepAliveMs));
+    }
     openIfNoOpened() {
         if (!this.client) {
-            this.lastKeepAliveAt = Date.now();
-            this.keepAliveBackoffMs = flags.liveQueryPollWhenStaleMs;
-            this.attachForeground();
-            this.interval = setInterval(() => {
-                this.runPromise(this.maybeKeepAlive());
-            }, Math.min(flags.liveQueryPollWhenStaleMs, flags.liveQueryKeepAliveMs));
+            this.startKeepAlive();
             return this.runPromise((this.client = this.apiProvider()
                 .subscriptionClient.openConnection(() => {
                 for (const q of this.queries.values()) {
                     q.subscribeCode();
                 }
+                for (const c of this.channels.values()) {
+                    for (const l of c.listeners)
+                        l.reconnect?.();
+                }
             })
                 .then((c) => {
                 this.openedConnection = c;

package/esm/src/live-query/SseSubscriptionClient.js

@@ -1,17 +1,21 @@
 import { buildRestDataProvider } from '../buildRestDataProvider.js';
 import { remult } from '../remult-proxy.js';
 import { remultStatic } from '../remult-static.js';
+import { flags } from '../remult3/remult3.js';
 import { streamUrl } from './SubscriptionChannel.js';
 export class SseSubscriptionClient {
     openConnection(onReconnect) {
         let connectionId;
         const channels = new Map();
+        const errorHandlers = new Map();
         const provider = buildRestDataProvider(remult.apiClient.httpClient);
         let source;
         let retryCount = 0;
         let closed = false;
         let connected = false;
+        let everConnected = false;
         let reconnectTimer;
+        let lastConnectAt = 0;
         let connectionReady;
         let connectionPromise = new Promise((res) => {
             connectionReady = res;
@@ -36,15 +40,16 @@ export class SseSubscriptionClient {
         function resume(force) {
             if (closed)
                 return;
-            retryCount = 0;
-            clearReconnectTimer();
+            const open = source && source.readyState !== EVENT_SOURCE_CLOSED;
             if (force) {
+                // a source still inside its handshake window is not a zombie yet
+                if (open && Date.now() - lastConnectAt < flags.sseStaleMs)
+                    return;
                 createConnection();
                 return;
             }
-            const open = source &&
-                typeof EventSource !== 'undefined' &&
-                source.readyState !== EventSource.CLOSED;
+            retryCount = 0;
+            clearReconnectTimer();
             if (!open)
                 createConnection();
         }
@@ -56,23 +61,39 @@ export class SseSubscriptionClient {
                 clearReconnectTimer();
                 source?.close();
             },
-            async subscribe(channel, handler) {
+            async subscribe(channel, handler, onError) {
                 let listeners = channels.get(channel);
+                let onErrors = errorHandlers.get(channel);
                 if (!listeners) {
                     channels.set(channel, (listeners = []));
-                    await connectionPromise;
-                    if (!closed)
-                        await subscribeToChannel(channel);
+                    errorHandlers.set(channel, (onErrors = []));
+                    try {
+                        await connectionPromise;
+                        if (!closed)
+                            await subscribeToChannel(channel);
+                    }
+                    catch (err) {
+                        // otherwise the next subscribe sees the entry and never retries
+                        channels.delete(channel);
+                        errorHandlers.delete(channel);
+                        throw err;
+                    }
                 }
                 listeners.push(handler);
+                onErrors.push(onError);
                 return () => {
-                    listeners.splice(listeners.indexOf(handler, 1));
+                    const i = listeners.indexOf(handler);
+                    if (i >= 0) {
+                        listeners.splice(i, 1);
+                        onErrors.splice(i, 1);
+                    }
                     if (listeners.length == 0) {
                         remultStatic.actionInfo.runActionWithoutBlockingUI(() => provider.post(remult.apiClient.url + '/' + streamUrl + '/unsubscribe', {
                             channel: channel,
                             clientId: connectionId,
                         }));
                         channels.delete(channel);
+                        errorHandlers.delete(channel);
                     }
                 };
             },
@@ -86,6 +107,7 @@ export class SseSubscriptionClient {
             clearReconnectTimer();
             if (source)
                 source.close();
+            lastConnectAt = Date.now();
             source = SseSubscriptionClient.createEventSource(remult.apiClient.url + '/' + streamUrl);
             source.onmessage = (e) => {
                 noteServerEvent();
@@ -109,16 +131,23 @@ export class SseSubscriptionClient {
                 connectionId = e.data;
                 retryCount = 0;
                 noteServerEvent();
-                if (connected) {
-                    for (const channel of channels.keys()) {
-                        await subscribeToChannel(channel);
+                // a connection the server forgot counts as a reconnect too: every
+                // channel must be re-subscribed and live queries refetched
+                const wasConnected = everConnected;
+                connected = everConnected = true;
+                if (wasConnected) {
+                    for (const channel of [...channels.keys()]) {
+                        try {
+                            await subscribeToChannel(channel);
+                        }
+                        catch (err) {
+                            // one refused channel must not block the others
+                            errorHandlers.get(channel)?.forEach((f) => f(err));
+                        }
                     }
                     onReconnect();
                 }
-                else {
-                    connected = true;
-                    connectionReady();
-                }
+                connectionReady();
             });
         }
         createConnection();
@@ -130,15 +159,13 @@ export class SseSubscriptionClient {
                     clientId: connectionId,
                 });
             });
-            if (result === ConnectionNotFoundError) {
+            if (result === ConnectionNotFoundError && connected) {
                 connected = false;
                 connectionPromise = new Promise((res) => {
                     connectionReady = res;
                 });
                 createConnection();
                 await connectionPromise;
-                if (!closed)
-                    await subscribeToChannel(channel);
             }
         }
     }
@@ -149,3 +176,5 @@ export class SseSubscriptionClient {
     }
 }
 export const ConnectionNotFoundError = 'client connection not found';
+// EventSource.CLOSED; the global is absent in Node
+const EVENT_SOURCE_CLOSED = 2;

package/esm/src/live-query/SubscriptionChannel.js

@@ -14,14 +14,19 @@ export class LiveQuerySubscriber {
     unsubscribe = () => { };
     unsubscribeChannel = () => { };
     unsubscribeQuery = () => { };
+    snapshotReady = false;
+    droppedWhilePending = false;
     async setAllItems(result) {
+        // server re-registers the query at version 0; reset before the await so a
+        // version message landing mid-parse is judged against the new baseline
+        this.version = 0;
         const items = await getRepositoryInternals(this.repo)._fromJsonArray(result, this.query.options);
         this.forListeners((listener) => {
             listener(() => {
                 return items;
             });
         }, this.allItemsMessage(items));
-        this.version = 0;
+        this.snapshotReady = true;
     }
     allItemsMessage(items) {
         return [
@@ -55,16 +60,24 @@ export class LiveQuerySubscriber {
         };
     }
     async handle(messages) {
+        // the pending snapshot supersedes anything in flight; the keep-alive
+        // after it lands catches whatever was dropped here
+        if (!this.snapshotReady) {
+            this.droppedWhilePending = true;
+            return;
+        }
         const ver = messages.find((m) => m.type === 'version');
-        if (ver) {
-            if (ver.from < this.version)
-                return;
-            if (ver.from > this.version) {
-                this.version = ver.to;
-                this.subscribeCode?.();
-                return;
-            }
+        // any gap, forward or backward, means a message was missed or two servers
+        // published the same version with different diffs: refetch instead of drop
+        if (ver && ver.from !== this.version) {
+            this.version = ver.to;
+            this.subscribeCode?.();
+            return;
         }
+        // bump before the await below, so a second message delivered in the same
+        // tick is judged against this one and not mistaken for a gap
+        if (ver)
+            this.version = ver.to;
         const data = messages.filter((m) => m.type !== 'version');
         {
             let x = data.filter(({ type }) => type == 'add' || type == 'replace');
@@ -102,7 +115,8 @@ export class LiveQuerySubscriber {
                 return items;
             });
         }, data);
-        this.version = ver ? ver.to : this.version + (data.length > 0 ? 1 : 0);
+        if (!ver && data.length > 0)
+            this.version++;
     }
     defaultQueryState = [];
     listeners = [];
@@ -200,6 +214,16 @@ export const liveQueryKeepAliveRoute = '_liveQueryKeepAlive';
  *  });
  *  ```
  *
+ *  #### Recovering after a dropped connection
+ *  Messages published while the connection was down are not replayed.
+ *  Use the `reconnect` listener to refetch state:
+ *  ```ts
+ *  chart.subscribe({
+ *    next: (message) => (chartData = message.data),
+ *    reconnect: () => loadChartData(),
+ *  });
+ *  ```
+ *
  */
 export class SubscriptionChannel {
     channelKey;