prostgles-server 4.2.535 → 4.2.538

package/lib/{SyncReplication.ts → PubSubManager/SyncReplication/getSyncUtilFunctions.ts}

@@ -1,98 +1,52 @@
-import type { AnyObject, OrderBy, SyncBatchParams } from "prostgles-types";
-import { WAL, omitKeys, pickKeys } from "prostgles-types";
-import type { TableHandler } from "./DboBuilder/TableHandler/TableHandler";
-import type { PubSubManager, SyncParams } from "./PubSubManager/PubSubManager";
-import { log } from "./PubSubManager/PubSubManagerUtils";
-
-export type ClientSyncInfo = Partial<{
-  c_fr: AnyObject;
-  c_lr: AnyObject;
-  /**
-   * PG count is ussually string due to bigint
-   */
-  c_count: number | string;
-}>;
-
-export type ServerSyncInfo = Partial<{
-  s_fr: AnyObject;
-  s_lr: AnyObject;
-  /**
-   * PG count is ussually string due to bigint
-   */
-  s_count: number | string;
-}>;
-
-export type SyncBatchInfo = Partial<{
-  from_synced: number | null;
-  to_synced: number | null;
-  end_offset: number | null;
-}>;
-
-export type onSyncRequestResponse =
-  | {
-      onSyncRequest?: ClientSyncInfo;
-    }
-  | {
-      err: AnyObject | string;
-    };
-
-export type ClientExpressData = ClientSyncInfo & {
-  data?: AnyObject[];
-  deleted?: AnyObject[];
+import {
+  isDefined,
+  omitKeys,
+  pickKeys,
+  type AnyObject,
+  type SyncBatchParams,
+} from "prostgles-types";
+import type { PRGLIOSocket } from "../../DboBuilder/DboBuilder";
+import type { TableHandler } from "../../DboBuilder/TableHandler/TableHandler";
+import { getSyncOrderByAndFields } from "./getSyncOrderByAndFields";
+import type { PubSubManager, SyncParams } from "../PubSubManager";
+import { fetchSyncServerData } from "./fetchSyncServerData";
+import type {
+  ClientSyncInfo,
+  onSyncRequestResponse,
+  ServerSyncInfo,
+  SyncBatchInfo,
+} from "./syncData";
+import { log } from "../PubSubManagerUtils";
+import type { EventTypes } from "../../Logging";
+
+type Args = {
+  socket: PRGLIOSocket;
+  tableHandler: TableHandler;
+  sync: SyncParams;
+  pubSubManager: PubSubManager;
+  logSyncData: (state: Extract<EventTypes.Sync, { command: "syncData" }>["state"]) => void;
 };
 
-function getNumbers(numberArr: (null | undefined | string | number)[]): number[] {
-  return numberArr.filter((v) => v !== null && v !== undefined && Number.isFinite(+v)) as number[];
-}
-
-/**
- * Server or client requested data sync
- */
-export async function syncData(
-  this: PubSubManager,
-  sync: SyncParams,
-  clientData: ClientExpressData | undefined,
-  source: "trigger" | "client",
-) {
-  await this._log({
-    type: "sync",
-    command: "syncData",
-    tableName: sync.table_name,
-    sid: sync.sid,
-    source,
-    ...pickKeys(sync, ["socket_id", "condition", "last_synced", "is_syncing"]),
-    lr: JSON.stringify(sync.lr),
-    connectedSocketIds: this.dboBuilder.prostgles.connectedSockets.map((s) => s.id),
-    localParams: undefined,
-    duration: -1,
-    socketId: sync.socket_id,
-  });
-
+export const getSyncUtilFunctions = ({
+  socket,
+  tableHandler,
+  sync,
+  pubSubManager,
+  logSyncData,
+}: Args) => {
   const {
+    synced_field,
     socket_id,
-    channel_name,
-    table_name,
+    id_fields,
     filter,
     table_rules,
-    params,
-    synced_field,
-    id_fields = [],
     batch_size,
-    wal,
-    throttle = 0,
+    channel_name,
+    table_name,
+    params,
   } = sync;
 
-  const socket = this.sockets[socket_id];
-  if (!socket) {
-    return;
-  }
-  const tableHandler = this.dbo[table_name];
-  if (!tableHandler?.find) {
-    throw `dbo.${table_name}.find missing or not allowed`;
-  }
-
-  const sync_fields = [synced_field, ...id_fields.sort()],
-    orderByAsc: OrderBy = sync_fields.reduce((a, v) => ({ ...a, [v]: true }), {}),
+  const { orderByAsc, sync_fields } = getSyncOrderByAndFields({ synced_field, id_fields }),
     rowsIdsMatch = (a?: AnyObject, b?: AnyObject) => {
       return a && b && !id_fields.find((key) => a[key].toString() !== b[key].toString());
     },
@@ -100,30 +54,31 @@ export async function syncData(
       return rowsIdsMatch(a, b) && a?.[synced_field].toString() === b?.[synced_field].toString();
     },
     getServerRowInfo = async (args: SyncBatchParams = {}): Promise<ServerSyncInfo> => {
-      const { from_synced = null, to_synced = null, offset = 0, limit } = args;
-      const _filter: AnyObject = { ...filter };
+      const { from_synced, to_synced, offset = 0, limit } = args;
+      const batchFilter = { ...filter };
 
-      if (from_synced || to_synced) {
-        _filter[synced_field] = {
-          ...(from_synced ? { $gte: from_synced } : {}),
-          ...(to_synced ? { $lte: to_synced } : {}),
+      if (isDefined(from_synced) || isDefined(to_synced)) {
+        batchFilter[synced_field] = {
+          ...(isDefined(from_synced) ? { $gte: from_synced } : {}),
+          ...(isDefined(to_synced) ? { $lte: to_synced } : {}),
         };
       }
 
-      const first_rows = await tableHandler.find(
-        _filter,
+      const first_rows = (await tableHandler.find(
+        batchFilter,
         { orderBy: orderByAsc, select: sync_fields, limit, offset },
         undefined,
         table_rules,
         { clientReq: { socket } },
-      );
+      )) as AnyObject[];
+
       const last_rows = first_rows.slice(-1); // Why not logic below?
       // const last_rows = await _this?.dbo[table_name]?.find?.(_filter, { orderBy: (orderByDesc as OrderBy), select: sync_fields, limit: 1, offset: -offset || 0 }, null, table_rules);
-      const count = await tableHandler.count(_filter, undefined, undefined, table_rules);
+      const count = await tableHandler.count(batchFilter, undefined, undefined, table_rules);
 
       return {
-        s_fr: first_rows[0] || null,
-        s_lr: last_rows[0] || null,
+        s_fr: first_rows[0],
+        s_lr: last_rows[0],
         s_count: count,
       };
     },
@@ -147,16 +102,15 @@ export async function syncData(
 
       return res;
     },
-    getClientData = (from_synced = 0, offset = 0): Promise<AnyObject[]> => {
+    getClientData = (from_synced: number | undefined, offset = 0): Promise<AnyObject[]> => {
       return new Promise((resolve, reject) => {
         const onPullRequest = {
-          from_synced: from_synced || 0,
+          from_synced: from_synced,
           offset: offset || 0,
           limit: batch_size,
         };
         socket.emit(channel_name, { onPullRequest }, (resp?: { data?: AnyObject[] }) => {
           if (resp && resp.data && Array.isArray(resp.data)) {
-            // console.log({ onPullRequest, resp }, socket._user)
             resolve(sortClientData(resp.data));
           } else {
             reject("unexpected onPullRequest response: " + JSON.stringify(resp));
@@ -182,31 +136,11 @@ export async function syncData(
         });
       }
     },
-    getServerData = async (from_synced = 0, offset = 0): Promise<AnyObject[]> => {
-      const _filter = {
-        ...filter,
-        [synced_field]: { $gte: from_synced || 0 },
-      };
-
-      try {
-        const res = await tableHandler.find(
-          _filter,
-          {
-            select: params.select,
-            orderBy: orderByAsc as OrderBy,
-            offset: offset || 0,
-            limit: batch_size,
-          },
-          undefined,
-          table_rules,
-          { clientReq: { socket } },
-        );
-
-        return res;
-      } catch (e) {
-        console.error("Sync getServerData failed: ", e);
-        throw "INTERNAL ERROR";
-      }
+    getServerData = async (from_synced: number | undefined, offset = 0): Promise<AnyObject[]> => {
+      return fetchSyncServerData(
+        { tableHandler, socket, from_synced, offset },
+        { filter, id_fields, params, synced_field, batch_size, table_rules },
+      );
     },
     deleteData = async (deleted: AnyObject[]) => {
       // console.log("deleteData deleteData  deleteData " + deleted.length);
@@ -215,7 +149,7 @@ export async function syncData(
       //     deleted.map(async (d) => {
       //       const id_filter = pickKeys(d, id_fields);
       //       try {
-      //         await (this.dbo[table_name] as TableHandler).delete(
+      //         await (pubSubManager.dbo[table_name] as TableHandler).delete(
       //           id_filter,
       //           undefined,
       //           undefined,
@@ -238,66 +172,59 @@ export async function syncData(
      */
     upsertData = async (data: AnyObject[]) => {
       const start = Date.now();
-      const result = await this.dboBuilder
+      const result = await pubSubManager.dboBuilder
         .getTX(async (dbTX) => {
           const tableHandlerTx = dbTX[table_name] as TableHandler;
           const existingData = await tableHandlerTx.find(
             { $or: data.map((d) => pickKeys(d, id_fields)) },
             {
               select: [synced_field, ...id_fields],
-              orderBy: orderByAsc as OrderBy,
+              orderBy: orderByAsc,
             },
             undefined,
             table_rules,
             { clientReq: { socket } },
           );
-          let inserts = data.filter((d) => !existingData.find((ed) => rowsIdsMatch(ed, d)));
-          let updates = data.filter((d) =>
-            existingData.find((ed) => rowsIdsMatch(ed, d) && +ed[synced_field] < +d[synced_field]),
+          let rowsToInsert = data.filter((d) => !existingData.find((ed) => rowsIdsMatch(ed, d)));
+          let rowsToUpdate = data.filter((d) =>
+            existingData.find(
+              (ed) => rowsIdsMatch(ed, d) && Number(ed[synced_field]) < Number(d[synced_field]),
+            ),
           );
-          try {
-            if (!table_rules) throw "table_rules missing";
-
-            if (table_rules.update && updates.length) {
-              const updateData: [any, any][] = [];
-              await Promise.all(
-                updates.map((upd) => {
-                  const id_filter = pickKeys(upd, id_fields);
-                  const syncSafeFilter = {
-                    $and: [id_filter, { [synced_field]: { "<": upd[synced_field] } }],
-                  };
-
-                  updateData.push([syncSafeFilter, omitKeys(upd, id_fields)]);
-                }),
-              );
-              await tableHandlerTx.updateBatch(
-                updateData,
-                { removeDisallowedFields: true },
-                undefined,
-                table_rules,
-                { clientReq: { socket } },
-              );
-            } else {
-              updates = [];
-            }
-
-            if (table_rules.insert && inserts.length) {
-              await tableHandlerTx.insert(
-                inserts,
-                { removeDisallowedFields: true },
-                undefined,
-                table_rules,
-                { clientReq: { socket } },
-              );
-            } else {
-              inserts = [];
-            }
-
-            return { inserts, updates };
-          } catch (e) {
-            console.trace(e);
-            throw e;
+
+          if (table_rules.update && rowsToUpdate.length) {
+            const batchUpdates: [AnyObject, AnyObject][] = rowsToUpdate.map((rowToUpdate) => {
+              const id_filter = pickKeys(rowToUpdate, id_fields);
+              const syncSafeFilter = {
+                $and: [id_filter, { [synced_field]: { "<": rowToUpdate[synced_field] } }],
+              };
+
+              return [syncSafeFilter, omitKeys(rowToUpdate, id_fields)];
+            });
+            await tableHandlerTx.updateBatch(
+              batchUpdates,
+              { removeDisallowedFields: true },
+              undefined,
+              table_rules,
+              { clientReq: { socket } },
+            );
+          } else {
+            rowsToUpdate = [];
+          }
+
+          if (table_rules.insert && rowsToInsert.length) {
+            await tableHandlerTx.insert(
+              rowsToInsert,
+              { removeDisallowedFields: true },
+              undefined,
+              table_rules,
+              { clientReq: { socket } },
+            );
+          } else {
+            rowsToInsert = [];
           }
+
+          return { inserts: rowsToInsert, updates: rowsToUpdate };
         })
         .then(({ inserts, updates }) => {
           log(
@@ -318,15 +245,17 @@ export async function syncData(
           return Promise.reject(new Error("Something went wrong with syncing to server: "));
         });
 
-      await this._log({
+      await pubSubManager._log({
         type: "sync",
         command: "upsertData",
+        channelName: channel_name,
         tableName: sync.table_name,
         rows: data.length,
         socketId: socket_id,
         sid: sync.sid,
         duration: Date.now() - start,
-        connectedSocketIds: this.dboBuilder.prostgles.connectedSockets.map((s) => s.id),
+        connectedSocketIds: pubSubManager.dboBuilder.prostgles.connectedSockets.map((s) => s.id),
+        syncParams: sync,
       });
 
       return result;
@@ -335,13 +264,18 @@ export async function syncData(
      * Pushes the given data to client
      * @param isSynced = true if
      */
-    pushData = async (data?: AnyObject[], isSynced = false, err: unknown = null) => {
+    pushData = async (data: AnyObject[], isSynced = false) => {
       const start = Date.now();
-      const result = await new Promise((resolve, reject) => {
+      const result = await new Promise<{
+        pushed: number;
+        resp: {
+          ok: boolean;
+        };
+      }>((resolve, reject) => {
         socket.emit(channel_name, { data, isSynced }, (resp?: { ok: boolean }) => {
           if (resp && resp.ok) {
             // console.log("PUSHED to client: fr/lr", data[0], data[data.length - 1]);
-            resolve({ pushed: data?.length, resp });
+            resolve({ pushed: data.length, resp });
           } else {
             reject(resp);
             console.error("Unexpected response");
@@ -349,15 +283,17 @@ export async function syncData(
         });
       });
 
-      await this._log({
+      await pubSubManager._log({
         type: "sync",
         command: "pushData",
         tableName: sync.table_name,
-        rows: data?.length ?? 0,
+        rows: data.length,
         socketId: socket_id,
         duration: Date.now() - start,
         sid: sync.sid,
-        connectedSocketIds: this.dboBuilder.prostgles.connectedSockets.map((s) => s.id),
+        connectedSocketIds: pubSubManager.dboBuilder.prostgles.connectedSockets.map((s) => s.id),
+        channelName: channel_name,
+        syncParams: sync,
       });
 
       return result;
@@ -374,20 +310,24 @@ export async function syncData(
 
       // console.log("getLastSynced", clientData, socket._user )
 
-      let result = null;
+      let result: number | null = null;
 
       /* Nothing to sync */
-      if ((!c_fr && !s_fr) || rowsFullyMatch(c_lr, s_lr)) {
-        //  c_count === s_count &&
-        // sync.last_synced = null;
-        result = null;
+      if (!c_fr && !s_fr) {
+        logSyncData("getLastSynced.nothingToSync");
+        return null;
+      }
+
+      if (rowsFullyMatch(c_lr, s_lr)) {
+        logSyncData("getLastSynced.rowsFullyMatch(lr)");
+        return null;
+      }
 
-        /* Sync Everything */
-      } else if (!rowsFullyMatch(c_fr, s_fr)) {
+      if (!rowsFullyMatch(c_fr, s_fr)) {
         if (c_fr && s_fr) {
           result = Math.min(c_fr[synced_field], s_fr[synced_field]);
-        } else if (c_fr || s_fr) {
-          result = (c_fr || s_fr)![synced_field];
+        } else {
+          result = (c_fr ?? s_fr)![synced_field];
         }
 
         /* Sync from last matching synced value */
@@ -408,15 +348,15 @@ export async function syncData(
             to_synced: result,
             end_offset,
           });
-          // console.log("getLastSynced... end_offset > " + end_offset);
-          let server_row;
+
+          let server_rows: AnyObject[] | undefined;
 
           if (c_lr) {
             const _filter: AnyObject = {};
             sync_fields.map((key) => {
               _filter[key] = c_lr[key];
             });
-            server_row = await this.dbo[table_name]?.find(
+            server_rows = await tableHandler.find(
               _filter,
               { select: sync_fields, limit: 1 },
               undefined,
@@ -425,11 +365,9 @@ export async function syncData(
             );
           }
 
-          // if(rowsFullyMatch(c_lr, s_lr)){ //c_count === s_count &&
-          if (server_row && server_row.length) {
-            server_row = server_row[0];
-
-            result = +server_row[synced_field];
+          const first_server_row = server_rows?.[0];
+          if (first_server_row) {
+            result = +first_server_row[synced_field];
             end_offset = min_count;
             // console.log(`getLastSynced found for ${table_name} -> ${result}`);
           } else {
@@ -443,20 +381,21 @@ export async function syncData(
 
       return result;
     },
-    updateSyncLR = (data: AnyObject) => {
-      if (data.length) {
-        const lastRow = data[data.length - 1];
-        if (sync.lr?.[synced_field] && +sync.lr[synced_field] > +lastRow[synced_field]) {
-          console.error(
-            {
-              syncIssue: "sync.lr[synced_field] is greater than lastRow[synced_field]",
-            },
-            sync.table_name,
-          );
-        }
-        sync.lr = lastRow;
-        sync.last_synced = +sync.lr?.[synced_field];
+    updateSyncLR = (data: AnyObject[]) => {
+      const lastRow = data.at(-1);
+      if (!lastRow) {
+        return;
       }
+      if (sync.lr?.[synced_field] && +sync.lr[synced_field] > +lastRow[synced_field]) {
+        console.error(
+          {
+            syncIssue: "sync.lr[synced_field] is greater than lastRow[synced_field]",
+          },
+          sync.table_name,
+        );
+      }
+      sync.lr = lastRow;
+      sync.last_synced = +sync.lr[synced_field];
     },
     /**
      * Will push pull sync between client and server from a given from_synced value
@@ -465,8 +404,7 @@ export async function syncData(
       let offset = 0,
         canContinue = true;
       const limit = batch_size,
-        min_synced = from_synced || 0,
-        max_synced = from_synced;
+        min_synced = from_synced ?? undefined;
 
       let inserted = 0,
         updated = 0,
@@ -490,7 +428,7 @@ export async function syncData(
           serverData = await getServerData(min_synced, offset);
         } catch (e) {
           console.trace("sync getServerData err", e);
-          await pushData(undefined, undefined, "Internal error. Check server logs");
+          // await pushData(undefined, undefined, "Internal error. Check server logs");
           throw " d";
         }
 
@@ -503,7 +441,7 @@ export async function syncData(
         //   await Promise.all(
         //     to_delete.map((d) => {
         //       deleted++;
-        //       return (this.dbo[table_name] as TableHandler).delete(
+        //       return (pubSubManager.dbo[table_name] as TableHandler).delete(
         //         pickKeys(d, id_fields),
         //         {},
         //         undefined,
@@ -520,7 +458,7 @@ export async function syncData(
           );
         });
         if (forClient.length) {
-          const res: any = await pushData(
+          const res = await pushData(
             forClient.filter((d) => !sync.wal || !sync.wal.isInHistory(d)),
           );
           pushed += res.pushed;
@@ -532,9 +470,7 @@ export async function syncData(
         }
         offset += serverData.length;
 
-        // canContinue = offset >= limit;
         canContinue = serverData.length >= limit;
-        // console.log(`sData ${sData.length}      limit ${limit}`);
       }
       log(
         `server.syncBatch ${table_name}: inserted( ${inserted} )    updated( ${updated} )   deleted( ${deleted} )    pushed to client( ${pushed} )     total( ${total} )`,
@@ -544,115 +480,22 @@ export async function syncData(
       return true;
     };
 
-  if (!wal) {
-    /* Used to throttle and merge incomming updates */
-    sync.wal = new WAL({
-      id_fields,
-      synced_field,
-      throttle,
-      batch_size,
-      DEBUG_MODE: this.dboBuilder.prostgles.opts.DEBUG_MODE,
-      onSendStart: () => {
-        sync.is_syncing = true;
-      },
-      onSend: async (data) => {
-        // console.log("WAL upsertData START", data)
-        const res = await upsertData(data);
-        // const max_incoming_synced = Math.max(...data.map(d => +d[synced_field]));
-        // if(Number.isFinite(max_incoming_synced) && max_incoming_synced > +sync.last_synced){
-        //     sync.last_synced = max_incoming_synced;
-        // }
-        // console.log("WAL upsertData END")
-
-        /******** */
-        /* TO DO -> Store and push patch updates instead of full data if and where possible */
-        /******** */
-        // 1. Store successfully upserted wal items for a couple of seconds
-        // 2. When pushing data to clients check if any matching wal items exist
-        // 3. Replace text fields with matching patched data
-
-        return res;
-      },
-      onSendEnd: (batch) => {
-        updateSyncLR(batch);
-        sync.is_syncing = false;
-        // console.log("syncData from WAL.onSendEnd")
-
-        /**
-         * After all data was inserted request SyncInfo from client and sync again if necessary
-         */
-        void this.syncData(sync, undefined, source);
-      },
-    });
-  }
-
-  /* Debounce sync requests */
-  if (!sync.wal) throw "sync.wal missing";
-  if (!sync.wal.isSending() && sync.is_syncing) {
-    if (!this.syncTimeout) {
-      this.syncTimeout = setTimeout(() => {
-        this.syncTimeout = undefined;
-        // console.log("SYNC FROM TIMEOUT")
-        void this.syncData(sync, undefined, source);
-      }, throttle);
-    }
-    // console.log("SYNC THROTTLE")
-    return;
-  }
-
-  // console.log("syncData", clientData)
-
-  /**
-   * Express data sent from a client that has already been synced
-   * Add to WAL manager which will sync at the end
-   */
-  if (clientData) {
-    if (clientData.data && Array.isArray(clientData.data) && clientData.data.length) {
-      return sync.wal.addData(clientData.data.map((d) => ({ current: d })));
-
-      // await upsertData(clientData.data, true);
-
-      /* Not expecting this anymore. use normal db.table.delete channel */
-    } else if (
-      clientData.deleted &&
-      Array.isArray(clientData.deleted) &&
-      clientData.deleted.length
-    ) {
-      await deleteData(clientData.deleted);
-    }
-  } else {
-    // do nothing
-  }
-  if (sync.wal.isSending()) return;
-
-  sync.is_syncing = true;
-
-  // from synced does not make sense. It should be sync.lr only!!!
-  let from_synced = null;
-
-  /** Sync was already synced */
-  if (sync.lr) {
-    const { s_lr } = await getServerRowInfo();
-
-    /* Make sure trigger is not firing on freshly synced data */
-    if (!rowsFullyMatch(sync.lr, s_lr)) {
-      from_synced = sync.last_synced;
-    } else {
-      // console.log("rowsFullyMatch")
-    }
-    // console.log(table_name, sync.lr[synced_field])
-  } else {
-    from_synced = await getLastSynced(clientData);
-  }
-
-  if (from_synced !== null) {
-    await syncBatch(from_synced);
-  } else {
-    // console.log("from_synced is null")
-  }
-
-  await pushData([], true);
-
-  sync.is_syncing = false;
-  // console.log(`Finished sync for ${table_name}`, socket._user);
+  return {
+    rowsIdsMatch,
+    rowsFullyMatch,
+    getServerRowInfo,
+    getClientRowInfo,
+    getClientData,
+    getServerData,
+    deleteData,
+    upsertData,
+    pushData,
+    getLastSynced,
+    updateSyncLR,
+    syncBatch,
+  };
+};
+
+function getNumbers(numberArr: (null | undefined | string | number)[]): number[] {
+  return numberArr.filter((v) => v !== null && v !== undefined && Number.isFinite(+v)) as number[];
 }