remult 3.3.18 → 3.4.0-next.1

package/SseSubscriptionServer.d.ts

@@ -1,6 +1,8 @@
 import type { GenericResponse } from './server/index.js';
 import type { Remult } from './src/context.js';
 import type { SubscriptionServer } from './src/live-query/SubscriptionServer.js';
+/** Keep below typical proxy idle (30s) and Heroku H15 (55s). */
+export declare const ssePingMs = 15000;
 export declare class SseSubscriptionServer implements SubscriptionServer {
     private canUserConnectToChannel?;
     constructor(canUserConnectToChannel?: ((channel: string, remult: Remult) => boolean) | undefined);

package/SseSubscriptionServer.js

@@ -1,8 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.clientConnection = exports.SseSubscriptionServer = void 0;
+exports.clientConnection = exports.SseSubscriptionServer = exports.ssePingMs = void 0;
 var tslib_1 = require("tslib");
 var SseSubscriptionClient_js_1 = require("./src/live-query/SseSubscriptionClient.js");
+/** Keep below typical proxy idle (30s) and Heroku H15 (55s). */
+exports.ssePingMs = 15000;
 var SseSubscriptionServer = /** @class */ (function () {
     function SseSubscriptionServer(canUserConnectToChannel) {
         this.canUserConnectToChannel = canUserConnectToChannel;
@@ -137,7 +139,7 @@ var clientConnection = /** @class */ (function () {
         this.write('', 'keep-alive');
         this.timeOutRef = setTimeout(function () {
             _this.sendLiveMessage();
-        }, 45000);
+        }, exports.ssePingMs);
     };
     return clientConnection;
 }());

package/esm/SseSubscriptionServer.js

@@ -1,4 +1,6 @@
 import { ConnectionNotFoundError } from './src/live-query/SseSubscriptionClient.js';
+/** Keep below typical proxy idle (30s) and Heroku H15 (55s). */
+export const ssePingMs = 15_000;
 export class SseSubscriptionServer {
     canUserConnectToChannel;
     //@internal
@@ -101,6 +103,6 @@ export class clientConnection {
         this.write('', 'keep-alive');
         this.timeOutRef = setTimeout(() => {
             this.sendLiveMessage();
-        }, 45000);
+        }, ssePingMs);
     }
 }

package/esm/live-query/data-provider-live-query-storage.js

@@ -52,12 +52,14 @@ export class DataProviderLiveQueryStorage {
     }
     async keepAliveAndReturnUnknownQueryIds(queryIds) {
         const repo = await this.repo;
+        const versions = {};
         for (const query of await repo.find({ where: { id: queryIds } })) {
             query.lastUsedIso = new Date().toISOString();
             await repo.save(query);
+            versions[query.id] = query.data?.version ?? 0;
             queryIds = queryIds.filter((x) => x !== query.id);
         }
-        return queryIds;
+        return { unknownQueryIds: queryIds, versions };
     }
 }
 let LiveQueryStorageEntity = class LiveQueryStorageEntity extends EntityBase {

package/esm/server/remult-api-server.js

@@ -281,7 +281,12 @@ export class RemultServerImplementation {
                 }));
             }
             r.route(this.options.rootPath + '/' + liveQueryKeepAliveRoute).post(this.process(async (remult, req, res, reqInfo, origRes, origReq) => {
-                res.success(await remult.liveQueryStorage.keepAliveAndReturnUnknownQueryIds(await this.coreOptions.getRequestBody(origReq)));
+                const body = await this.coreOptions.getRequestBody(origReq);
+                const queryIds = Array.isArray(body)
+                    ? body
+                    : (body?.queryIds ?? []);
+                const result = await remult.liveQueryStorage.keepAliveAndReturnUnknownQueryIds(queryIds);
+                res.success(Array.isArray(body) ? result.unknownQueryIds : result);
             }));
         }
         this.entities.forEach((e) => {

package/esm/src/data-api.js

@@ -358,6 +358,7 @@ export class DataApi {
                 requestJson: await serializeContext(),
                 findOptionsJson: findOptionsToJson(r.findOptions, this.repository.metadata),
                 lastIds: r.r.map((y) => this.repository.metadata.idMetadata.getId(y)),
+                version: 0,
             };
             await this.remult.liveQueryStorage.add({
                 entityKey: this.repository.metadata.key,

package/esm/src/live-query/LiveQueryClient.js

@@ -1,8 +1,9 @@
 import { buildRestDataProvider } from '../buildRestDataProvider.js';
 import { RestDataProvider } from '../data-providers/rest-data-provider.js';
 import { remultStatic } from '../remult-static.js';
+import { flags } from '../remult3/remult3.js';
 import { getRepositoryInternals } from '../remult3/repository-internals.js';
-import { liveQueryKeepAliveRoute, LiveQuerySubscriber, } from './SubscriptionChannel.js';
+import { liveQueryKeepAliveRoute, LiveQuerySubscriber, onBrowserForeground, } from './SubscriptionChannel.js';
 /* @internal*/
 export class LiveQueryClient {
     apiProvider;
@@ -15,10 +16,10 @@ export class LiveQueryClient {
             handleMessage();
     }
     queries = new Map();
+    channels = new Map();
     hasQueriesForTesting() {
         return this.queries.size > 0;
     }
-    channels = new Map();
     constructor(apiProvider, getUserId) {
         this.apiProvider = apiProvider;
         this.getUserId = getUserId;
@@ -29,6 +30,7 @@ export class LiveQueryClient {
     close() {
         this.queries.clear();
         this.channels.clear();
+        this.detachForeground();
         this.closeIfNoListeners();
     }
     async subscribeChannel(key, onResult) {
@@ -68,13 +70,15 @@ export class LiveQueryClient {
         };
     }
     closeIfNoListeners() {
-        if (this.client)
-            if (this.queries.size === 0 && this.channels.size === 0) {
-                this.runPromise(this.client.then((x) => x.close()));
-                this.client = undefined;
-                clearInterval(this.interval);
-                this.interval = undefined;
-            }
+        if (this.queries.size === 0 && this.channels.size === 0) {
+            if (this.client)
+                this.runPromise(this.client.then((x) => x.close()).catch(() => { }));
+            this.client = undefined;
+            this.openedConnection = undefined;
+            this.detachForeground();
+            clearInterval(this.interval);
+            this.interval = undefined;
+        }
     }
     subscribe(repo, options, listener) {
         let alive = true;
@@ -94,43 +98,56 @@ export class LiveQueryClient {
             if (!q) {
                 this.queries.set(eventTypeKey, (q = new LiveQuerySubscriber(repo, { entityKey: repo.metadata.key, options }, this.getUserId())));
                 q.subscribeCode = () => {
-                    if (q.unsubscribe) {
-                        q.unsubscribe();
+                    q.unsubscribeChannel();
+                    let unsubscribeToChannel = () => { };
+                    q.unsubscribeChannel = () => {
+                        unsubscribeToChannel();
+                        unsubscribeToChannel = () => { };
+                    };
+                    q.unsubscribe = () => {
                         q.unsubscribe = () => { };
-                    }
+                        q.unsubscribeChannel();
+                        q.unsubscribeQuery();
+                    };
+                    let snapshotDone = false;
+                    const applySnapshot = (r) => {
+                        if (q.listeners.length === 0) {
+                            r.unsubscribe();
+                            return;
+                        }
+                        q.unsubscribeQuery = () => {
+                            this.runPromise(r.unsubscribe());
+                        };
+                        snapshotDone = true;
+                        return this.runPromise(q.setAllItems(r.result));
+                    };
                     this.runPromise(this.subscribeChannel(q.queryChannel, {
                         next: (value) => this.runPromise(q.handle(value)),
                         complete: () => { },
                         error: (er) => {
                             q.listeners.forEach((l) => l.error(er));
                         },
-                    }).then((unsubscribeToChannel) => {
+                    })
+                        .then((unsub) => {
                         if (q.listeners.length == 0) {
-                            unsubscribeToChannel();
+                            unsub();
                             return;
                         }
-                        this.runPromise(subscribe(q.queryChannel)
-                            .then((r) => {
-                            if (q.listeners.length === 0) {
-                                r.unsubscribe();
-                                unsubscribeToChannel();
-                                return;
-                            }
-                            this.runPromise(q.setAllItems(r.result));
-                            q.unsubscribe = () => {
-                                q.unsubscribe = () => { };
-                                unsubscribeToChannel();
-                                this.runPromise(r.unsubscribe());
-                            };
-                        })
-                            .catch((err) => {
-                            q.listeners.forEach((l) => l.error(err));
-                            unsubscribeToChannel();
-                            this.queries.delete(eventTypeKey);
-                        }));
-                    })).catch((err) => {
+                        unsubscribeToChannel = unsub;
+                        if (snapshotDone)
+                            return this.runPromise(this.runKeepAlive({ checkVersions: true }));
+                    })
+                        .catch((err) => {
                         q.listeners.forEach((l) => l.error(err));
-                    });
+                    }));
+                    this.runPromise(subscribe(q.queryChannel)
+                        .then(applySnapshot)
+                        .catch((err) => {
+                        q.listeners.forEach((l) => l.error(err));
+                        unsubscribeToChannel();
+                        this.queries.delete(eventTypeKey);
+                        this.closeIfNoListeners();
+                    }));
                 };
                 q.subscribeCode();
             }
@@ -156,29 +173,123 @@ export class LiveQueryClient {
         };
     }
     client;
+    openedConnection;
     interval;
+    lastKeepAliveAt = 0;
+    keepAliveBackoffMs = flags.liveQueryPollWhenStaleMs;
+    keepAliveInFlight;
+    foregroundKeepAliveTimer;
+    isSseStale() {
+        const conn = this.openedConnection;
+        if (!conn || conn.lastServerEvent === undefined)
+            return false;
+        return Date.now() - conn.lastServerEvent > flags.sseStaleMs;
+    }
+    async runKeepAlive(opts) {
+        if (this.keepAliveInFlight)
+            return this.keepAliveInFlight;
+        this.keepAliveInFlight = this.sendKeepAlive(opts).finally(() => {
+            this.keepAliveInFlight = undefined;
+        });
+        return this.keepAliveInFlight;
+    }
+    async sendKeepAlive(opts) {
+        const ids = [];
+        for (const q of this.queries.values()) {
+            ids.push(q.queryChannel);
+        }
+        if (ids.length === 0)
+            return;
+        let p = this.apiProvider();
+        const raw = await this.runPromise(remultStatic.actionInfo.runActionWithoutBlockingUI(() => buildRestDataProvider(p.httpClient).post(p.url + '/' + liveQueryKeepAliveRoute, { queryIds: ids })));
+        const unknownIds = Array.isArray(raw)
+            ? raw
+            : (raw?.unknownQueryIds ?? []);
+        const versions = Array.isArray(raw)
+            ? {}
+            : (raw?.versions ?? {});
+        const stale = opts?.checkVersions || this.isSseStale();
+        let reloaded = false;
+        for (const q of this.queries.values()) {
+            const serverVersion = versions[q.queryChannel];
+            const unknown = unknownIds.includes(q.queryChannel);
+            const mismatch = stale && serverVersion !== undefined && serverVersion !== q.version;
+            if (unknown || mismatch) {
+                reloaded = true;
+                if (serverVersion !== undefined)
+                    q.version = serverVersion;
+                q.subscribeCode();
+            }
+        }
+        if (this.isSseStale()) {
+            this.keepAliveBackoffMs = reloaded
+                ? flags.liveQueryPollWhenStaleMs
+                : Math.min(this.keepAliveBackoffMs * 2, flags.liveQueryKeepAliveMs);
+        }
+        else {
+            this.keepAliveBackoffMs = flags.liveQueryPollWhenStaleMs;
+        }
+    }
+    async maybeKeepAlive() {
+        const stale = this.isSseStale();
+        if (!stale)
+            this.keepAliveBackoffMs = flags.liveQueryPollWhenStaleMs;
+        const interval = stale
+            ? this.keepAliveBackoffMs
+            : flags.liveQueryKeepAliveMs;
+        if (Date.now() - this.lastKeepAliveAt < interval)
+            return;
+        this.lastKeepAliveAt = Date.now();
+        if (stale)
+            this.openedConnection?.resume?.(true);
+        await this.runKeepAlive();
+    }
+    detachForeground = () => { };
+    attachForeground() {
+        this.detachForeground();
+        const unsub = onBrowserForeground(() => {
+            this.openedConnection?.resume?.();
+            if (this.foregroundKeepAliveTimer !== undefined)
+                clearTimeout(this.foregroundKeepAliveTimer);
+            this.foregroundKeepAliveTimer = setTimeout(() => {
+                this.foregroundKeepAliveTimer = undefined;
+                this.keepAliveBackoffMs = flags.liveQueryPollWhenStaleMs;
+                this.lastKeepAliveAt = 0;
+                this.runPromise(this.runKeepAlive({ checkVersions: true }));
+            }, 300);
+        });
+        this.detachForeground = () => {
+            unsub();
+            if (this.foregroundKeepAliveTimer !== undefined) {
+                clearTimeout(this.foregroundKeepAliveTimer);
+                this.foregroundKeepAliveTimer = undefined;
+            }
+            this.detachForeground = () => { };
+        };
+    }
     openIfNoOpened() {
         if (!this.client) {
-            this.interval = setInterval(async () => {
-                const ids = [];
-                for (const q of this.queries.values()) {
-                    ids.push(q.queryChannel);
-                }
-                if (ids.length > 0) {
-                    let p = this.apiProvider();
-                    const invalidIds = await this.runPromise(await remultStatic.actionInfo.runActionWithoutBlockingUI(() => buildRestDataProvider(p.httpClient).post(p.url + '/' + liveQueryKeepAliveRoute, ids)));
-                    for (const id of invalidIds) {
-                        for (const q of this.queries.values()) {
-                            if (q.queryChannel === id)
-                                q.subscribeCode();
-                        }
-                    }
-                }
-            }, 30000);
-            return this.runPromise((this.client = this.apiProvider().subscriptionClient.openConnection(() => {
+            this.lastKeepAliveAt = Date.now();
+            this.keepAliveBackoffMs = flags.liveQueryPollWhenStaleMs;
+            this.attachForeground();
+            this.interval = setInterval(() => {
+                this.runPromise(this.maybeKeepAlive());
+            }, Math.min(flags.liveQueryPollWhenStaleMs, flags.liveQueryKeepAliveMs));
+            return this.runPromise((this.client = this.apiProvider()
+                .subscriptionClient.openConnection(() => {
                 for (const q of this.queries.values()) {
                     q.subscribeCode();
                 }
+            })
+                .then((c) => {
+                this.openedConnection = c;
+                return c;
+            })
+                .catch((err) => {
+                this.client = undefined;
+                this.openedConnection = undefined;
+                this.closeIfNoListeners();
+                throw err;
             })));
         }
         return this.client;

package/esm/src/live-query/SseSubscriptionClient.js

@@ -1,24 +1,68 @@
 import { buildRestDataProvider } from '../buildRestDataProvider.js';
 import { remult } from '../remult-proxy.js';
 import { remultStatic } from '../remult-static.js';
-import { flags } from '../remult3/remult3.js';
 import { streamUrl } from './SubscriptionChannel.js';
 export class SseSubscriptionClient {
     openConnection(onReconnect) {
         let connectionId;
         const channels = new Map();
         const provider = buildRestDataProvider(remult.apiClient.httpClient);
-        let connected = false;
         let source;
+        let retryCount = 0;
+        let closed = false;
+        let connected = false;
+        let reconnectTimer;
+        let connectionReady;
+        let connectionPromise = new Promise((res) => {
+            connectionReady = res;
+        });
+        function clearReconnectTimer() {
+            if (reconnectTimer !== undefined) {
+                clearTimeout(reconnectTimer);
+                reconnectTimer = undefined;
+            }
+        }
+        function scheduleReconnect() {
+            if (closed)
+                return;
+            clearReconnectTimer();
+            const delay = Math.min(500 * 2 ** Math.min(retryCount, 5), 8_000);
+            retryCount++;
+            reconnectTimer = setTimeout(() => {
+                reconnectTimer = undefined;
+                createConnection();
+            }, delay);
+        }
+        function resume(force) {
+            if (closed)
+                return;
+            retryCount = 0;
+            clearReconnectTimer();
+            if (force) {
+                createConnection();
+                return;
+            }
+            const open = source &&
+                typeof EventSource !== 'undefined' &&
+                source.readyState !== EventSource.CLOSED;
+            if (!open)
+                createConnection();
+        }
         const client = {
+            lastServerEvent: Date.now(),
+            resume,
             close() {
-                source.close();
+                closed = true;
+                clearReconnectTimer();
+                source?.close();
             },
             async subscribe(channel, handler) {
                 let listeners = channels.get(channel);
                 if (!listeners) {
                     channels.set(channel, (listeners = []));
-                    await subscribeToChannel(channel);
+                    await connectionPromise;
+                    if (!closed)
+                        await subscribeToChannel(channel);
                 }
                 listeners.push(handler);
                 return () => {
@@ -33,45 +77,52 @@ export class SseSubscriptionClient {
                 };
             },
         };
-        const createConnectionPromise = () => new Promise((res) => {
-            createConnection();
-            let retryCount = 0;
-            function createConnection() {
-                if (source)
-                    source.close();
-                source = SseSubscriptionClient.createEventSource(remult.apiClient.url + '/' + streamUrl);
-                source.onmessage = (e) => {
-                    let message = JSON.parse(e.data);
-                    const listeners = channels.get(message.channel);
-                    if (listeners)
-                        listeners.forEach((x) => x(message.data));
-                };
-                source.onerror = (e) => {
-                    console.error('Live Query Event Source Error', e);
-                    source.close();
-                    if (retryCount++ < flags.error500RetryCount) {
-                        setTimeout(() => {
-                            createConnection();
-                        }, 500);
-                    }
-                };
-                source.addEventListener('connectionId', async (e) => {
-                    //@ts-ignore
-                    connectionId = e.data;
-                    if (connected) {
-                        for (const channel of channels.keys()) {
-                            await subscribeToChannel(channel);
-                        }
-                        onReconnect();
-                    }
-                    else {
-                        connected = true;
-                        res(client);
+        function noteServerEvent() {
+            client.lastServerEvent = Date.now();
+        }
+        function createConnection() {
+            if (closed)
+                return;
+            clearReconnectTimer();
+            if (source)
+                source.close();
+            source = SseSubscriptionClient.createEventSource(remult.apiClient.url + '/' + streamUrl);
+            source.onmessage = (e) => {
+                noteServerEvent();
+                let message = JSON.parse(e.data);
+                const listeners = channels.get(message.channel);
+                if (listeners)
+                    listeners.forEach((x) => x(message.data));
+            };
+            source.addEventListener('keep-alive', () => {
+                noteServerEvent();
+            });
+            source.onerror = (e) => {
+                console.error('Live Query Event Source Error', e);
+                source.close();
+                if (closed)
+                    return;
+                scheduleReconnect();
+            };
+            source.addEventListener('connectionId', async (e) => {
+                //@ts-ignore
+                connectionId = e.data;
+                retryCount = 0;
+                noteServerEvent();
+                if (connected) {
+                    for (const channel of channels.keys()) {
+                        await subscribeToChannel(channel);
                     }
-                });
-            }
-        });
-        return createConnectionPromise();
+                    onReconnect();
+                }
+                else {
+                    connected = true;
+                    connectionReady();
+                }
+            });
+        }
+        createConnection();
+        return Promise.resolve(client);
         async function subscribeToChannel(channel) {
             const result = await remultStatic.actionInfo.runActionWithoutBlockingUI(() => {
                 return provider.post(remult.apiClient.url + '/' + streamUrl + '/subscribe', {
@@ -80,7 +131,14 @@ export class SseSubscriptionClient {
                 });
             });
             if (result === ConnectionNotFoundError) {
-                await createConnectionPromise();
+                connected = false;
+                connectionPromise = new Promise((res) => {
+                    connectionReady = res;
+                });
+                createConnection();
+                await connectionPromise;
+                if (!closed)
+                    await subscribeToChannel(channel);
             }
         }
     }

package/esm/src/live-query/SubscriptionChannel.js

@@ -12,6 +12,8 @@ export class LiveQuerySubscriber {
     queryChannel;
     subscribeCode;
     unsubscribe = () => { };
+    unsubscribeChannel = () => { };
+    unsubscribeQuery = () => { };
     async setAllItems(result) {
         const items = await getRepositoryInternals(this.repo)._fromJsonArray(result, this.query.options);
         this.forListeners((listener) => {
@@ -19,6 +21,7 @@ export class LiveQuerySubscriber {
                 return items;
             });
         }, this.allItemsMessage(items));
+        this.version = 0;
     }
     allItemsMessage(items) {
         return [
@@ -52,8 +55,19 @@ export class LiveQuerySubscriber {
         };
     }
     async handle(messages) {
+        const ver = messages.find((m) => m.type === 'version');
+        if (ver) {
+            if (ver.from < this.version)
+                return;
+            if (ver.from > this.version) {
+                this.version = ver.to;
+                this.subscribeCode?.();
+                return;
+            }
+        }
+        const data = messages.filter((m) => m.type !== 'version');
         {
-            let x = messages.filter(({ type }) => type == 'add' || type == 'replace');
+            let x = data.filter(({ type }) => type == 'add' || type == 'replace');
             let loadedItems = await getRepositoryInternals(this.repo)._fromJsonArray(x.map((m) => m.data.item), this.query.options);
             for (let index = 0; index < x.length; index++) {
                 const element = x[index];
@@ -64,7 +78,7 @@ export class LiveQuerySubscriber {
             listener((items) => {
                 if (!items)
                     items = [];
-                for (const message of messages) {
+                for (const message of data) {
                     switch (message.type) {
                         case 'all':
                             this.setAllItems(message.data);
@@ -87,11 +101,13 @@ export class LiveQuerySubscriber {
                 }
                 return items;
             });
-        }, messages);
+        }, data);
+        this.version = ver ? ver.to : this.version + (data.length > 0 ? 1 : 0);
     }
     defaultQueryState = [];
     listeners = [];
     id = String(crypto.randomUUID());
+    version = 0;
     constructor(repo, query, userId) {
         this.repo = repo;
         this.query = query;
@@ -99,6 +115,24 @@ export class LiveQuerySubscriber {
         this.id = this.queryChannel;
     }
 }
+//@internal
+export function onBrowserForeground(handler) {
+    if (typeof document === 'undefined')
+        return () => { };
+    function onForeground() {
+        if (document.visibilityState === 'hidden')
+            return;
+        handler();
+    }
+    document.addEventListener('visibilitychange', onForeground);
+    window.addEventListener('online', onForeground);
+    window.addEventListener('pageshow', onForeground);
+    return () => {
+        document.removeEventListener('visibilitychange', onForeground);
+        window.removeEventListener('online', onForeground);
+        window.removeEventListener('pageshow', onForeground);
+    };
+}
 export const liveQueryKeepAliveRoute = '_liveQueryKeepAlive';
 /**
  * The `SubscriptionChannel` class is used to send messages from the backend to the frontend,

package/esm/src/live-query/SubscriptionServer.js

@@ -57,6 +57,11 @@ export class LiveQueryPublisher {
                     messages,
                 });
                 query.lastIds = currentIds;
+                if (messages.length > 0) {
+                    const from = query.version ?? 0;
+                    query.version = from + 1;
+                    messages.push({ type: 'version', from, to: query.version });
+                }
                 await setData(query);
                 if (messages.length > 0)
                     this.subscriptionServer().publishMessage(q.id, messages);
@@ -70,21 +75,24 @@ export class InMemoryLiveQueryStorage {
         this.debugFileSaver(this.queries);
     }
     async keepAliveAndReturnUnknownQueryIds(ids) {
-        const result = [];
+        const unknownQueryIds = [];
+        const versions = {};
         for (const id of ids) {
             let q = this.queries.find((q) => q.id === id);
             if (q) {
                 q.lastUsed = new Date().toISOString();
+                versions[id] = q.data?.version ?? 0;
             }
             else
-                result.push(id);
+                unknownQueryIds.push(id);
         }
         this.debug();
-        return result;
+        return { unknownQueryIds, versions };
     }
     queries = [];
     constructor() { }
     async add(query) {
+        this.queries = this.queries.filter((q) => q.id !== query.id);
         this.queries.push({ ...query, lastUsed: new Date().toISOString() });
         this.debug();
     }