prostgles-server 4.2.535 → 4.2.538

package/dist/PubSubManager/SyncReplication/syncData.js

@@ -0,0 +1,132 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.syncData = syncData;
+const prostgles_types_1 = require("prostgles-types");
+const WAL_1 = require("prostgles-types/dist/WAL");
+const getSyncUtilFunctions_1 = require("./getSyncUtilFunctions");
+/**
+ * Server or client requested data sync
+ */
+async function syncData(sync, clientData, source) {
+    const logSyncData = (state) => {
+        return this._log({
+            type: "sync",
+            command: "syncData",
+            channelName: sync.channel_name,
+            tableName: sync.table_name,
+            sid: sync.sid,
+            source,
+            ...(0, prostgles_types_1.pickKeys)(sync, ["socket_id", "condition", "last_synced", "is_syncing"]),
+            lr: JSON.stringify(sync.lr),
+            connectedSocketIds: this.dboBuilder.prostgles.connectedSockets.map((s) => s.id),
+            localParams: undefined,
+            duration: -1,
+            socketId: sync.socket_id,
+            syncParams: sync,
+            state,
+        });
+    };
+    await logSyncData("start");
+    const { socket_id, table_name, synced_field, id_fields = [], batch_size, throttle = 0 } = sync;
+    const socket = this.sockets[socket_id];
+    if (!socket?.connected) {
+        await logSyncData("socket?.connected");
+        return;
+    }
+    const tableHandler = this.dbo[table_name];
+    if (!tableHandler?.find) {
+        throw `dbo.${table_name}.find missing or not allowed`;
+    }
+    const { upsertData, updateSyncLR, getServerRowInfo, deleteData, getLastSynced, pushData, syncBatch, rowsFullyMatch, } = (0, getSyncUtilFunctions_1.getSyncUtilFunctions)({
+        tableHandler,
+        socket,
+        sync,
+        pubSubManager: this,
+        logSyncData,
+    });
+    /* Used to throttle and merge incoming updates */
+    sync.wal ??= new WAL_1.WAL({
+        id_fields,
+        synced_field,
+        throttle,
+        batch_size,
+        DEBUG_MODE: this.dboBuilder.prostgles.opts.DEBUG_MODE,
+        onSendStart: () => {
+            sync.is_syncing = true;
+        },
+        onSend: async (data) => {
+            const res = await upsertData(data);
+            return res;
+        },
+        onSendEnd: (batch) => {
+            updateSyncLR(batch);
+            sync.is_syncing = false;
+            /**
+             * After all data was inserted request SyncInfo from client and sync again if necessary
+             */
+            void this.syncData(sync, undefined, source);
+        },
+    });
+    /* Debounce sync requests */
+    if (!sync.wal.isSending() && sync.is_syncing) {
+        if (!this.syncTimeout) {
+            this.syncTimeout = setTimeout(() => {
+                this.syncTimeout = undefined;
+                void this.syncData(sync, undefined, source);
+            }, throttle);
+        }
+        return;
+    }
+    /**
+     * Express data sent from a client that has already been synced
+     * Add to WAL manager which will sync at the end
+     */
+    if (clientData) {
+        if (clientData.data && Array.isArray(clientData.data) && clientData.data.length) {
+            return sync.wal.addData(clientData.data.map((d) => ({ current: d })));
+            /* Not expecting this anymore. use normal db.table.delete channel */
+        }
+        else if (clientData.deleted &&
+            Array.isArray(clientData.deleted) &&
+            clientData.deleted.length) {
+            await deleteData(clientData.deleted);
+        }
+    }
+    else {
+        // do nothing
+    }
+    if (sync.wal.isSending()) {
+        await logSyncData("sync.wal.isSending()");
+        return;
+    }
+    sync.is_syncing = true;
+    // from synced does not make sense. It should be sync.lr only!!!
+    let from_synced = null;
+    /** Sync was already synced */
+    if (sync.lr) {
+        const { s_lr } = await getServerRowInfo();
+        /* Make sure trigger is not firing on freshly synced data */
+        if (!rowsFullyMatch(sync.lr, s_lr)) {
+            from_synced = sync.last_synced;
+            await logSyncData("sync.last_synced");
+        }
+        else {
+            await logSyncData("rowsFullyMatch");
+        }
+    }
+    else {
+        await logSyncData("getLastSynced(clientData)");
+        from_synced = await getLastSynced(clientData);
+    }
+    if (from_synced !== null) {
+        await logSyncData("syncBatch.start");
+        await syncBatch(from_synced);
+        await logSyncData("syncBatch.end");
+    }
+    else {
+        await logSyncData("nothingToSync");
+    }
+    await pushData([], true);
+    sync.is_syncing = false;
+}
+//# sourceMappingURL=syncData.js.map

package/dist/PubSubManager/SyncReplication/syncData.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"syncData.js","sourceRoot":"","sources":["../../../lib/PubSubManager/SyncReplication/syncData.ts"],"names":[],"mappings":";;AA+CA,4BAmJC;AAjMD,qDAA2C;AAC3C,kDAA+C;AAE/C,iEAA8D;AAwC9D;;GAEG;AACI,KAAK,UAAU,QAAQ,CAE5B,IAAgB,EAChB,UAAyC,EACzC,MAA4B;IAE5B,MAAM,WAAW,GAAG,CAAC,KAAiE,EAAE,EAAE;QACxF,OAAO,IAAI,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,UAAU;YACnB,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM;YACN,GAAG,IAAA,0BAAQ,EAAC,IAAI,EAAE,CAAC,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;YAC1E,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,kBAAkB,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,WAAW,EAAE,SAAS;YACtB,QAAQ,EAAE,CAAC,CAAC;YACZ,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,UAAU,EAAE,IAAI;YAChB,KAAK;SACN,CAAC,CAAC;IACL,CAAC,CAAC;IACF,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IAE3B,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,SAAS,GAAG,EAAE,EAAE,UAAU,EAAE,QAAQ,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC;IAE/F,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACvC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;QACvB,MAAM,WAAW,CAAC,mBAAmB,CAAC,CAAC;QACvC,OAAO;IACT,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC1C,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC;QACxB,MAAM,OAAO,UAAU,8BAA8B,CAAC;IACxD,CAAC;IAED,MAAM,EACJ,UAAU,EACV,YAAY,EACZ,gBAAgB,EAChB,UAAU,EACV,aAAa,EACb,QAAQ,EACR,SAAS,EACT,cAAc,GACf,GAAG,IAAA,2CAAoB,EAAC;QACvB,YAAY;QACZ,MAAM;QACN,IAAI;QACJ,aAAa,EAAE,IAAI;QACnB,WAAW;KACZ,CAAC,CAAC;IAEH,iDAAiD;IACjD,IAAI,CAAC,GAAG,KAAK,IAAI,SAAG,CAAC;QACnB,SAAS;QACT,YAAY;QACZ,QAAQ;QACR,UAAU;QACV,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU;QACrD,WAAW,EAAE,GAAG,EAAE;YAChB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACrB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;YACnC,OAAO,GAAG,CAAC;QACb,CAAC;QACD,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;YAExB;;eAEG;YACH,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAC9C,CAAC;KACF,CAAC,CAAC;IAEH,4BAA4B;IAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,GAAG,EAAE;gBACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;gBAC7B,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC9C,CAAC,EAAE,QAAQ,CAAC,CAAC;QACf,CAAC;QACD,OAAO;IACT,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAChF,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACtE,oEAAoE;QACtE,CAAC;aAAM,IACL,UAAU,CAAC,OAAO;YAClB,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;YACjC,UAAU,CAAC,OAAO,CAAC,MAAM,EACzB,CAAC;YACD,MAAM,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,aAAa;IACf,CAAC;IAED,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC;QACzB,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAC;QAC1C,OAAO;IACT,CAAC;IAED,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IAEvB,gEAAgE;IAChE,IAAI,WAAW,GAAG,IAAI,CAAC;IAEvB,8BAA8B;IAC9B,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,gBAAgB,EAAE,CAAC;QAE1C,4DAA4D;QAC5D,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;YACnC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;YAC/B,MAAM,WAAW,CAAC,kBAAkB,CAAC,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,MAAM,WAAW,CAAC,gBAAgB,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAC;QAC/C,WAAW,GAAG,MAAM,aAAa,CAAC,UAAU,CAAC,CAAC;IAChD,CAAC;IAED,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,WAAW,CAAC,iBAAiB,CAAC,CAAC;QACrC,MAAM,SAAS,CAAC,WAAW,CAAC,CAAC;QAC7B,MAAM,WAAW,CAAC,eAAe,CAAC,CAAC;IACrC,CAAC;SAAM,CAAC;QACN,MAAM,WAAW,CAAC,eAAe,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAEzB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;AAC1B,CAAC"}

package/dist/PubSubManager/addSync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"addSync.d.ts","sourceRoot":"","sources":["../../lib/PubSubManager/addSync.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAiB,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGnF;;;GAGG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,aAAa,EACnB,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC,CA0IlC"}
+{"version":3,"file":"addSync.d.ts","sourceRoot":"","sources":["../../lib/PubSubManager/addSync.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAiB,aAAa,EAAc,MAAM,iBAAiB,CAAC;AAI/F;;;GAGG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,aAAa,EACnB,UAAU,EAAE,aAAa,GACxB,OAAO,CAAC;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC,CAqIlC"}

package/dist/PubSubManager/addSync.js

@@ -10,22 +10,19 @@ const PubSubManagerUtils_1 = require("./PubSubManagerUtils");
 async function addSync(syncParams) {
     const sid = this.dboBuilder.prostgles.authHandler.getSIDNoError({ socket: syncParams.socket });
     const res = await (0, prostgles_types_1.tryCatchV2)(async () => {
-        const { socket = null, table_info = null, table_rules, synced_field = null, filter = {}, params, condition = "", } = syncParams;
+        const { socket, table_info, table_rules, filter, params, condition } = syncParams;
         const conditionParsed = (0, PubSubManagerUtils_1.parseCondition)(condition);
-        if (!socket || !table_info)
-            throw "socket or table_info missing";
         const { name: table_name } = table_info;
         const channelName = (0, prostgles_types_1.getSyncChannelName)({
             tableName: table_name,
             filter,
             select: params.select,
         });
-        if (!synced_field)
-            throw "synced_field missing from table_rules";
         this.upsertSocket(socket);
         const syncConfig = this.dboBuilder.prostgles.tableConfigurator?.getTableSyncConfig(table_name);
-        if (!syncConfig)
+        if (!syncConfig) {
             throw `Sync not configured for table ${table_name}`;
+        }
         const upsertSync = () => {
             const newSync = {
                 channel_name: channelName,
@@ -45,7 +42,7 @@ async function addSync(syncParams) {
                 socket,
                 params,
             };
-            /* Only a sync per socket per table per condition allowed */
+            /* Only a sync per socket per table/condition/select allowed */
             const existing = (0, prostgles_types_1.find)(this.syncs, { socket_id: socket.id, channel_name: channelName });
             if (!existing) {
                 this.syncs.push(newSync);
@@ -58,9 +55,11 @@ async function addSync(syncParams) {
                         socketId: socket.id,
                         tableName: table_name,
                         condition,
+                        channelName,
                         sid,
                         connectedSocketIds: this.connectedSocketIds,
                         duration: -1,
+                        syncParams: newSync,
                     });
                     socket.removeAllListeners(channelName);
                     socket.removeAllListeners(unsyncChn);
@@ -104,9 +103,9 @@ async function addSync(syncParams) {
             }
             return newSync;
         };
-        upsertSync();
+        const newSync = upsertSync();
         await this.addTrigger({ table_name, condition: conditionParsed, tracked_columns: undefined }, undefined, socket);
-        return { channelName };
+        return { channelName, newSync };
     });
     await this._log({
         type: "sync",
@@ -118,6 +117,8 @@ async function addSync(syncParams) {
         duration: res.duration,
         error: res.error,
         sid,
+        channelName: res.data?.channelName || "",
+        syncParams: res.data?.newSync ?? {},
     });
     if (res.hasError)
         throw res.error;

package/dist/PubSubManager/addSync.js.map

@@ -1 +1 @@
-{"version":3,"file":"addSync.js","sourceRoot":"","sources":["../../lib/PubSubManager/addSync.ts"],"names":[],"mappings":";;AASA,0BA6IC;AAtJD,qDAA6F;AAG7F,6DAAsD;AAEtD;;;GAGG;AACI,KAAK,UAAU,OAAO,CAE3B,UAAyB;IAEzB,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/F,MAAM,GAAG,GAAG,MAAM,IAAA,4BAAU,EAAC,KAAK,IAAI,EAAE;QACtC,MAAM,EACJ,MAAM,GAAG,IAAI,EACb,UAAU,GAAG,IAAI,EACjB,WAAW,EACX,YAAY,GAAG,IAAI,EACnB,MAAM,GAAG,EAAE,EACX,MAAM,EACN,SAAS,GAAG,EAAE,GACf,GAAG,UAAU,CAAC;QACf,MAAM,eAAe,GAAG,IAAA,mCAAc,EAAC,SAAS,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU;YAAE,MAAM,8BAA8B,CAAC;QAEjE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,UAAU,CAAC;QACxC,MAAM,WAAW,GAAG,IAAA,oCAAkB,EAAC;YACrC,SAAS,EAAE,UAAU;YACrB,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY;YAAE,MAAM,uCAAuC,CAAC;QAEjE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC/F,IAAI,CAAC,UAAU;YAAE,MAAM,iCAAiC,UAAU,EAAE,CAAC;QACrE,MAAM,UAAU,GAAG,GAAG,EAAE;YACtB,MAAM,OAAO,GAAG;gBACd,YAAY,EAAE,WAAW;gBACzB,UAAU;gBACV,MAAM;gBACN,SAAS,EAAE,eAAe;gBAC1B,GAAG;gBACH,WAAW;gBACX,GAAG,UAAU;gBACb,SAAS,EAAE,MAAM,CAAC,EAAE;gBACpB,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,CAAC;gBACd,EAAE,EAAE,SAAS;gBACb,UAAU;gBACV,UAAU,EAAE,KAAK;gBACjB,GAAG,EAAE,SAAS;gBACd,MAAM;gBACN,MAAM;aACP,CAAC;YAEF,4DAA4D;YAC5D,MAAM,QAAQ,GAAG,IAAA,sBAAI,EAAC,IAAI,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC;YACvF,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAEzB,MAAM,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;gBACzC,MAAM,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;gBACrC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,KAAU,EAAE,EAAiB,EAAE,EAAE;oBACvD,KAAK,IAAI,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,QAAQ;wBACjB,QAAQ,EAAE,MAAM,CAAC,EAAE;wBACnB,SAAS,EAAE,UAAU;wBACrB,SAAS;wBACT,GAAG;wBACH,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;wBAC3C,QAAQ,EAAE,CAAC,CAAC;qBACb,CAAC,CAAC;oBACH,MAAM,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;oBACvC,MAAM,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;oBACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;wBACnC,MAAM,OAAO,GACX,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC,YAAY,KAAK,WAAW,CAAC;wBAC7E,OAAO,CAAC,OAAO,CAAC;oBAClB,CAAC,CAAC,CAAC;oBACH,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1B,CAAC,CAAC,CAAC;gBAEH,MAAM,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;gBACvC,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,IAAuC,EAAE,EAAiB,EAAE,EAAE;oBACpF,IAAI,CAAC,IAAI,EAAE,CAAC;wBACV,EAAE,CAAC,EAAE,GAAG,EAAE,gDAAgD,EAAE,CAAC,CAAC;wBAC9D,OAAO;oBACT,CAAC;oBAED;;;;;;;;;;;sBAWE;oBAEF,IAAI,eAAe,IAAI,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;wBAClD,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;4BACjE,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,GAAG,CAAC,CAAC;4BACvD,EAAE,CAAC,EAAE,GAAG,EAAE,IAAA,sCAAoB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACzC,CAAC,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE,IAAI,CAAC,CAAC;oBACpE,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,kDAAkD,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;YAC1F,CAAC;YAED,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC;QAEF,UAAU,EAAE,CAAC;QAEb,MAAM,IAAI,CAAC,UAAU,CACnB,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,EACtE,SAAS,EACT,MAAM,CACP,CAAC;QAEF,OAAO,EAAE,WAAW,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,CAAC,IAAI,CAAC;QACd,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,SAAS;QAClB,SAAS,EAAE,UAAU,CAAC,UAAU,CAAC,IAAI;QACrC,SAAS,EAAE,UAAU,CAAC,SAAS;QAC/B,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE;QAC9B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;QAC3C,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,GAAG;KACJ,CAAC,CAAC;IAEH,IAAI,GAAG,CAAC,QAAQ;QAAE,MAAM,GAAG,CAAC,KAAK,CAAC;IAElC,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC"}
+{"version":3,"file":"addSync.js","sourceRoot":"","sources":["../../lib/PubSubManager/addSync.ts"],"names":[],"mappings":";;AASA,0BAwIC;AAjJD,qDAA6F;AAE7F,6DAAsD;AAGtD;;;GAGG;AACI,KAAK,UAAU,OAAO,CAE3B,UAAyB;IAEzB,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/F,MAAM,GAAG,GAAG,MAAM,IAAA,4BAAU,EAAC,KAAK,IAAI,EAAE;QACtC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,CAAC;QAClF,MAAM,eAAe,GAAG,IAAA,mCAAc,EAAC,SAAS,CAAC,CAAC;QAElD,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,UAAU,CAAC;QACxC,MAAM,WAAW,GAAG,IAAA,oCAAkB,EAAC;YACrC,SAAS,EAAE,UAAU;YACrB,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC/F,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,iCAAiC,UAAU,EAAE,CAAC;QACtD,CAAC;QACD,MAAM,UAAU,GAAG,GAAG,EAAE;YACtB,MAAM,OAAO,GAAG;gBACd,YAAY,EAAE,WAAW;gBACzB,UAAU;gBACV,MAAM;gBACN,SAAS,EAAE,eAAe;gBAC1B,GAAG;gBACH,WAAW;gBACX,GAAG,UAAU;gBACb,SAAS,EAAE,MAAM,CAAC,EAAE;gBACpB,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,CAAC;gBACd,EAAE,EAAE,SAAS;gBACb,UAAU;gBACV,UAAU,EAAE,KAAK;gBACjB,GAAG,EAAE,SAAS;gBACd,MAAM;gBACN,MAAM;aACP,CAAC;YAEF,+DAA+D;YAC/D,MAAM,QAAQ,GAAG,IAAA,sBAAI,EAAC,IAAI,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC;YACvF,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAEzB,MAAM,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;gBACzC,MAAM,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;gBACrC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,KAAU,EAAE,EAAiB,EAAE,EAAE;oBACvD,KAAK,IAAI,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,QAAQ;wBACjB,QAAQ,EAAE,MAAM,CAAC,EAAE;wBACnB,SAAS,EAAE,UAAU;wBACrB,SAAS;wBACT,WAAW;wBACX,GAAG;wBACH,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;wBAC3C,QAAQ,EAAE,CAAC,CAAC;wBACZ,UAAU,EAAE,OAAO;qBACpB,CAAC,CAAC;oBACH,MAAM,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;oBACvC,MAAM,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;oBACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;wBACnC,MAAM,OAAO,GACX,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC,YAAY,KAAK,WAAW,CAAC;wBAC7E,OAAO,CAAC,OAAO,CAAC;oBAClB,CAAC,CAAC,CAAC;oBACH,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1B,CAAC,CAAC,CAAC;gBAEH,MAAM,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;gBACvC,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,IAAuC,EAAE,EAAiB,EAAE,EAAE;oBACpF,IAAI,CAAC,IAAI,EAAE,CAAC;wBACV,EAAE,CAAC,EAAE,GAAG,EAAE,gDAAgD,EAAE,CAAC,CAAC;wBAC9D,OAAO;oBACT,CAAC;oBAED;;;;;;;;;;;sBAWE;oBAEF,IAAI,eAAe,IAAI,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;wBAClD,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;4BACjE,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,GAAG,CAAC,CAAC;4BACvD,EAAE,CAAC,EAAE,GAAG,EAAE,IAAA,sCAAoB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACzC,CAAC,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE,IAAI,CAAC,CAAC;oBACpE,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,kDAAkD,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;YAC1F,CAAC;YAED,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;QAE7B,MAAM,IAAI,CAAC,UAAU,CACnB,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,EACtE,SAAS,EACT,MAAM,CACP,CAAC;QAEF,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,CAAC,IAAI,CAAC;QACd,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,SAAS;QAClB,SAAS,EAAE,UAAU,CAAC,UAAU,CAAC,IAAI;QACrC,SAAS,EAAE,UAAU,CAAC,SAAS;QAC/B,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE;QAC9B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;QAC3C,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,GAAG;QACH,WAAW,EAAE,GAAG,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE;QACxC,UAAU,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,IAAK,EAAiB;KACpD,CAAC,CAAC;IAEH,IAAI,GAAG,CAAC,QAAQ;QAAE,MAAM,GAAG,CAAC,KAAK,CAAC;IAElC,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC"}

package/lib/DboBuilder/DboBuilderTypes.ts

@@ -6,7 +6,6 @@ import type {
   EXISTS_KEY,
   RawJoinPath,
   SQLHandler,
-  SyncTableInfo,
   TableInfo as TInfo,
   UserLike,
 } from "prostgles-types";
@@ -23,6 +22,7 @@ import type { FieldSpec } from "./QueryBuilder/Functions/Functions";
 import type { TableHandler } from "./TableHandler/TableHandler";
 import type { ParsedJoinPath } from "./ViewHandler/parseJoinPath";
 import pg = require("pg-promise/typescript/pg-subset");
+import type { SyncTableInfo } from "prostgles-types/dist/WAL";
 
 type PGP = pgPromise.IMain<{}, pg.IClient>;
 
@@ -188,6 +188,7 @@ export type PRGLIOSocket = {
   };
 
   _user?: AnyObject;
+  connected: boolean;
 
   /** Used for publish error caching */
   prostgles?: Map<string, ClientSchema>;

package/lib/DboBuilder/TableHandler/TableHandler.ts

@@ -4,12 +4,14 @@ import type {
   DeleteParams,
   FieldFilter,
   InsertParams,
+  ReplicationState,
   Select,
   UpdateParams,
 } from "prostgles-types";
 import { asName, isDefined } from "prostgles-types";
 import type { DB } from "../../Prostgles";
 import type { InsertRule, ParsedTableRule, UpdateRule } from "../../PublishParser/PublishParser";
+import { getSyncBatchOptions } from "../../PubSubManager/SyncReplication/getSyncBatchOptions";
 import type TableConfigurator from "../../TableConfig/TableConfig";
 import type { TableDefinition } from "../../TableConfig/TableConfig";
 import type { DboBuilder, Filter, LocalParams, TableHandlers } from "../DboBuilder";
@@ -198,7 +200,7 @@ export class TableHandler extends ViewHandler {
   async sync(
     filter: Filter,
     params: { select?: FieldFilter },
-    param3_unused: undefined,
+    _param3_unused: undefined,
     table_rules: ParsedTableRule,
     localParams: LocalParams,
   ) {
@@ -223,7 +225,7 @@ export class TableHandler extends ViewHandler {
       const ALLOWED_PARAMS = ["select"];
       const invalidParams = Object.keys(params).filter((k) => !ALLOWED_PARAMS.includes(k));
       if (invalidParams.length) {
-        throw "Invalid or dissallowed params found: " + invalidParams.join(", ");
+        throw "Invalid or disallowed params found: " + invalidParams.join(", ");
       }
 
       const { synced_field } = syncConfig;
@@ -250,13 +252,20 @@ export class TableHandler extends ViewHandler {
       });
 
       /* Step 1: parse command and params */
-      const result = await this.find(
+      const syncOpts = getSyncBatchOptions({
         filter,
-        { select, limit: 0 },
+        params: { select },
+        ...syncConfig,
+        from_synced: undefined,
+        offset: undefined,
+      });
+      const syncResult = await this.find(
+        syncOpts.syncBatchFilter,
+        { select: syncOpts.select, limit: syncOpts.limit, orderBy: syncOpts.orderBy },
         undefined,
         table_rules,
         localParams,
-      ).then(async (_isValid) => {
+      ).then(async (data: AnyObject[]) => {
         const { filterFields, forcedFilter } = table_rules.select || {};
         const condition = (
           await this.prepareWhere({
@@ -281,7 +290,16 @@ export class TableHandler extends ViewHandler {
             filter: { ...filter },
             params: { select },
           })
-          .then(({ channelName }) => ({ channelName, id_fields, synced_field }));
+          .then(
+            ({ channelName }) =>
+              ({
+                channelName,
+                data,
+                id_fields,
+                synced_field,
+                isSynced: data.length < syncOpts.limit,
+              }) satisfies ReplicationState["channels"]["CHANNEL_PREFIX"]["client.emit"]["server.response"]["data"],
+          );
       });
       await this._log({
         command: "sync",
@@ -289,7 +307,7 @@ export class TableHandler extends ViewHandler {
         data: { filter, params },
         duration: Date.now() - start,
       });
-      return result;
+      return syncResult;
     } catch (e) {
       await this._log({
         command: "sync",
@@ -309,7 +327,7 @@ export class TableHandler extends ViewHandler {
     /*
     REPLICATION
 
-        1 Sync proccess (NO DELETES ALLOWED):
+        1 Sync process (NO DELETES ALLOWED):
 
             Client sends:
                 "sync-request"

package/lib/DboBuilder/schema/getTablesForSchemaPostgresSQL.ts

@@ -31,10 +31,6 @@ export async function getTablesForSchemaPostgresSQL(
     ddlWithRollback,
   }: { schemaFilter: ProstglesInitOptions["schemaFilter"]; ddlWithRollback?: string },
 ) {
-  // : Promise<{
-  //   result: TableSchema[];
-  //   durations: Record<string, number>;
-  // }>
   const { sql, schemaNames } = getSchemaFilter(schemaFilter);
 
   return db.tx(async (t) => {

package/lib/Logging.ts

@@ -1,6 +1,6 @@
 import type { AnyObject, ClientSchema, TableHandler } from "prostgles-types";
 import type { LocalParams } from "./DboBuilder/DboBuilder";
-import type { PubSubManagerTriggers } from "./PubSubManager/PubSubManager";
+import type { PubSubManagerTriggers, SyncParams } from "./PubSubManager/PubSubManager";
 import type { NotifTypeName } from "./PubSubManager/PubSubManagerUtils";
 
 type ClientInfo = {
@@ -30,11 +30,26 @@ export namespace EventTypes {
       tableName: string;
       localParams?: LocalParams;
       connectedSocketIds: string[];
+      channelName: string;
+      syncParams: SyncParams;
     } & (
       | {
           command: "syncData";
           source: "client" | "trigger";
           lr: string;
+          state:
+            | "start"
+            | "sync.wal.isSending()"
+            | "initial"
+            | "socket?.connected"
+            | "sync.last_synced"
+            | "rowsFullyMatch"
+            | "getLastSynced.rowsFullyMatch(lr)"
+            | "getLastSynced.nothingToSync"
+            | "syncBatch.start"
+            | "syncBatch.end"
+            | "nothingToSync"
+            | "getLastSynced(clientData)";
         }
       | {
           command: "upsertData" | "pushData";

package/lib/PubSubManager/PubSubManager.ts

@@ -15,11 +15,10 @@ import { refreshTriggers } from "./refreshTriggers";
 
 import type {
   AnyObject,
+  EqualityFilter,
   FieldFilter,
   SelectParams,
-  SyncTableInfo,
   TableSchema,
-  WAL,
 } from "prostgles-types";
 import { CHANNELS, getSerialisableError, type SubscribeOptions } from "prostgles-types";
 
@@ -28,11 +27,12 @@ import type { OnData } from "../DboBuilder/ViewHandler/subscribe";
 import { matchesLocalFuncs } from "../DboBuilder/ViewHandler/subscribe";
 import type { EventTypes } from "../Logging";
 import type { ParsedTableRule } from "../PublishParser/PublishParser";
-import { syncData } from "../SyncReplication";
+import { syncData } from "./SyncReplication/syncData";
 import { addSub } from "./addSub";
 import { notifListener } from "./notifListener";
 import { log } from "./PubSubManagerUtils";
 import { pushSubData } from "./pushSubData";
+import type { SyncTableInfo, WAL } from "prostgles-types/dist/WAL";
 
 export type BasicCallback = (err?: any, res?: any) => void;
 
@@ -41,11 +41,11 @@ export type SyncParams = {
   sid: string | undefined;
   channel_name: string;
   table_name: string;
-  table_rules?: ParsedTableRule;
+  table_rules: ParsedTableRule;
   synced_field: string;
   id_fields: string[];
   batch_size: number;
-  filter: object;
+  filter: EqualityFilter<AnyObject>;
   params: {
     select: FieldFilter;
   };
@@ -361,6 +361,7 @@ export class PubSubManager {
           type: "sync",
           command: "upsertSocket.disconnect",
           tableName: "",
+          channelName: "*",
           duration: 0,
           sid: this.dboBuilder.prostgles.authHandler.getSIDNoError({ socket }),
           socketId: socket.id,
@@ -374,6 +375,7 @@ export class PubSubManager {
           remainingSyncs: JSON.stringify(
             this.syncs.map((s) => pickKeys(s, ["table_name", "condition"])),
           ),
+          syncParams: {} as SyncParams,
         });
 
         return "ok";

package/lib/PubSubManager/SyncReplication/fetchSyncServerData.ts

@@ -0,0 +1,55 @@
+import type { AnyObject } from "prostgles-types";
+import type { PRGLIOSocket } from "../../DboBuilder/DboBuilder";
+import type { TableHandler } from "../../DboBuilder/TableHandler/TableHandler";
+import type { SyncParams } from "../PubSubManager";
+import { getSyncOrderByAndFields } from "./getSyncOrderByAndFields";
+import { getSyncBatchOptions } from "./getSyncBatchOptions";
+
+export const fetchSyncServerData = async (
+  {
+    tableHandler,
+    socket,
+    from_synced,
+    offset,
+  }: {
+    tableHandler: TableHandler;
+    socket: PRGLIOSocket;
+    from_synced: number | undefined;
+    offset: number | undefined;
+  },
+  {
+    filter,
+    id_fields,
+    params,
+    synced_field,
+    batch_size,
+    table_rules,
+  }: Pick<
+    SyncParams,
+    "filter" | "params" | "id_fields" | "synced_field" | "batch_size" | "table_rules"
+  >,
+) => {
+  const { syncBatchFilter } = getSyncBatchOptions({
+    filter,
+    id_fields,
+    params,
+    synced_field,
+    batch_size,
+    from_synced,
+    offset,
+  });
+  const batchRows = await tableHandler.find(
+    syncBatchFilter,
+    {
+      select: params.select,
+      orderBy: getSyncOrderByAndFields({ synced_field, id_fields }).orderByAsc,
+      offset,
+      limit: batch_size,
+    },
+    undefined,
+    table_rules,
+    { clientReq: { socket } },
+  );
+
+  return batchRows as AnyObject[];
+};

package/lib/PubSubManager/SyncReplication/getSyncBatchOptions.ts

@@ -0,0 +1,31 @@
+import type { SyncParams } from "../PubSubManager";
+import { getSyncOrderByAndFields } from "./getSyncOrderByAndFields";
+
+export const getSyncBatchOptions = ({
+  from_synced,
+  offset = 0,
+  filter,
+  id_fields,
+  params,
+  synced_field,
+  batch_size,
+}: Pick<SyncParams, "filter" | "params" | "id_fields" | "synced_field" | "batch_size"> & {
+  from_synced: number | undefined;
+  offset: number | undefined;
+}) => {
+  const syncBatchFilter =
+    from_synced === undefined ? filter : (
+      {
+        ...filter,
+        [synced_field]: { $gte: from_synced },
+      }
+    );
+
+  return {
+    select: params.select,
+    orderBy: getSyncOrderByAndFields({ synced_field, id_fields }).orderByAsc,
+    offset,
+    limit: batch_size,
+    syncBatchFilter,
+  };
+};

package/lib/PubSubManager/SyncReplication/getSyncOrderByAndFields.ts

@@ -0,0 +1,11 @@
+import type { OrderBy } from "prostgles-types";
+import type { SyncParams } from "../PubSubManager";
+
+export const getSyncOrderByAndFields = ({
+  synced_field,
+  id_fields,
+}: Pick<SyncParams, "id_fields" | "synced_field">) => {
+  const sync_fields = [synced_field, ...id_fields.sort()],
+    orderByAsc: OrderBy = sync_fields.reduce((a, v) => ({ ...a, [v]: true }), {});
+  return { orderByAsc, sync_fields };
+};