prostgles-server 4.2.159 → 4.2.161

package/lib/SyncReplication.ts

@@ -0,0 +1,557 @@
+
+import { PubSubManager, SyncParams, pickKeys, omitKeys, log } from "./PubSubManager/PubSubManager";
+import { OrderBy, WAL, AnyObject, SyncBatchParams } from "prostgles-types";
+import { TableHandler } from "./DboBuilder/TableHandler/TableHandler";
+
+export type ClientSyncInfo = Partial<{
+  c_fr: AnyObject;
+  c_lr: AnyObject;
+  /**
+   * PG count is ussually string due to bigint
+   */
+  c_count: number | string;
+}>;
+
+export type ServerSyncInfo = Partial<{
+  s_fr: AnyObject;
+  s_lr: AnyObject;
+  /**
+   * PG count is ussually string due to bigint
+   */
+  s_count: number | string;
+}>
+
+export type SyncBatchInfo = Partial<{
+  from_synced: number | null;
+  to_synced: number | null;
+  end_offset: number | null;
+}>;
+
+export type onSyncRequestResponse = {
+  onSyncRequest?: ClientSyncInfo
+} | {
+  err: AnyObject | string;
+};
+
+export type ClientExpressData = ClientSyncInfo & {
+  data?: AnyObject[];
+  deleted?: AnyObject[];
+}
+
+function getNumbers(numberArr: (null | undefined | string | number)[]): number[] {
+  return numberArr.filter(v => v !== null && v !== undefined && Number.isFinite(+v)) as any;
+}
+
+/**
+ * Server or client requested data sync
+ */
+export async function syncData (this: PubSubManager, sync: SyncParams, clientData: ClientExpressData | undefined, source: "trigger" | "client"){
+  await this._log({ 
+    type: "sync",
+    command: "syncData", 
+    tableName: sync.table_name,
+    sid: sync.sid,
+    source,
+    ...pickKeys(sync, ["socket_id", "condition", "last_synced", "is_syncing"]),
+    lr: JSON.stringify(sync.lr),
+    connectedSocketIds: this.dboBuilder.prostgles.connectedSockets.map(s => s.id),
+    localParams: undefined,
+    duration: -1,
+    socketId: sync.socket_id
+  });
+
+  const {
+    socket_id, channel_name, table_name, filter,
+    table_rules, allow_delete = false, params,
+    synced_field, id_fields = [], batch_size,
+    wal, throttle = 0
+  } = sync;
+
+  const socket = this.sockets[socket_id];
+  if (!socket) {
+    return;
+  }
+
+  const sync_fields = [synced_field, ...id_fields.sort()],
+    orderByAsc: OrderBy = sync_fields.reduce((a, v) => ({ ...a, [v]: true }), {}),
+    rowsIdsMatch = (a?: AnyObject, b?: AnyObject) => {
+      return a && b && !id_fields.find(key => (a[key]).toString() !== (b[key]).toString())
+    },
+    rowsFullyMatch = (a?: AnyObject, b?: AnyObject) => {
+      return rowsIdsMatch(a, b) && a?.[synced_field].toString() === b?.[synced_field].toString();
+    },
+    getServerRowInfo = async (args: SyncBatchParams = {}): Promise<ServerSyncInfo> => {
+      const { from_synced = null, to_synced = null, offset = 0, limit } = args;
+      const _filter: AnyObject = { ...filter };
+
+      if (from_synced || to_synced) {
+        _filter[synced_field] = {
+          ...(from_synced ? { $gte: from_synced } : {}),
+          ...(to_synced ? { $lte: to_synced } : {})
+        }
+      }
+      if (this.dbo?.[table_name]?.find === undefined || this?.dbo?.[table_name]?.count === undefined) {
+        throw `dbo.${table_name}.find or .count are missing or not allowed`;
+      }
+
+      const first_rows = await this.dbo?.[table_name]?.find?.(_filter, { orderBy: orderByAsc, select: sync_fields, limit, offset }, undefined, table_rules);
+      const last_rows = first_rows?.slice(-1); // Why not logic below?
+      // const last_rows = await _this?.dbo[table_name]?.find?.(_filter, { orderBy: (orderByDesc as OrderBy), select: sync_fields, limit: 1, offset: -offset || 0 }, null, table_rules);
+      const count = await this.dbo?.[table_name]?.count?.(_filter, undefined, undefined, table_rules);
+
+      return { s_fr: first_rows?.[0] || null, s_lr: last_rows?.[0] || null, s_count: count }
+    },
+    getClientRowInfo = (args: SyncBatchInfo = {}) => {
+      const { from_synced = null, to_synced = null, end_offset = null } = args;
+      const res = new Promise<any>((resolve, reject) => {
+        const onSyncRequest = { from_synced, to_synced, end_offset };//, forReal: true };
+        socket.emit(channel_name, { onSyncRequest }, (resp?: onSyncRequestResponse) => {
+          if (resp && "onSyncRequest" in resp && resp?.onSyncRequest) {
+            const c_fr = resp.onSyncRequest.c_fr,
+              c_lr = resp.onSyncRequest.c_lr,
+              c_count = resp.onSyncRequest.c_count;
+
+            // console.log(onSyncRequest, { c_fr, c_lr, c_count }, socket._user);
+            return resolve({ c_fr, c_lr, c_count });
+          } else if (resp && "err" in resp && resp?.err) {
+            reject(resp.err);
+          }
+        });
+      });
+
+      return res;
+    },
+    getClientData = (from_synced = 0, offset = 0): Promise<AnyObject[]> => {
+      return new Promise((resolve, reject) => {
+        const onPullRequest = { from_synced: from_synced || 0, offset: offset || 0, limit: batch_size };
+        socket.emit(channel_name, { onPullRequest }, async (resp?: { data?: AnyObject[] }) => {
+          if (resp && resp.data && Array.isArray(resp.data)) {
+            // console.log({ onPullRequest, resp }, socket._user)
+            resolve(sortClientData(resp.data));
+          } else {
+            reject("unexpected onPullRequest response: " + JSON.stringify(resp));
+          }
+        });
+      });
+
+      function sortClientData(data: AnyObject[]) {
+        return data.sort((a, b) => {
+          /* Order by increasing synced and ids (sorted alphabetically) */
+          return (+a[synced_field] - +b[synced_field]) || id_fields.sort().map(idKey => a[idKey] < b[idKey] ? -1 : a[idKey] > b[idKey] ? 1 : 0).find(v => v) || 0;
+        });
+      }
+    },
+    getServerData = async (from_synced = 0, offset = 0): Promise<AnyObject[]> => {
+      const _filter = {
+        ...filter,
+        [synced_field]: { $gte: from_synced || 0 }
+      };
+
+      if (!this?.dbo?.[table_name]?.find) throw "_this?.dbo?.[table_name]?.find is missing";
+
+      try {
+        const res = this?.dbo?.[table_name]?.find?.(
+          _filter,
+          {
+            select: params.select,
+            orderBy: (orderByAsc as OrderBy),
+            offset: offset || 0,
+            limit: batch_size
+          },
+          undefined,
+          table_rules
+        );
+
+        if (!res) throw "_this?.dbo?.[table_name]?.find is missing";
+
+        return res;
+      } catch (e) {
+        console.error("Sync getServerData failed: ", e);
+        throw "INTERNAL ERROR"
+      }
+    },
+    deleteData = async (deleted: AnyObject[]) => {
+      // console.log("deleteData deleteData  deleteData " + deleted.length);
+      if (allow_delete) {
+        return Promise.all(deleted.map(async d => {
+          const id_filter = pickKeys(d, id_fields);
+          try {
+            await (this.dbo[table_name] as TableHandler).delete(id_filter, undefined, undefined, table_rules);
+            return 1;
+          } catch (e) {
+            console.error(e)
+          }
+          return 0;
+        }))
+      } else {
+        console.warn("client tried to delete data without permission (allow_delete is false)")
+      }
+      return false;
+    },
+
+    /**
+     * Upserts the given client data where synced_field is higher than on server  
+     */
+    upsertData = async (data: AnyObject[]) => {
+      
+      const start = Date.now();
+      const result = await this.dboBuilder.getTX(async (dbTX) => {
+        const tbl = dbTX[table_name] as TableHandler;
+        const existingData = await tbl.find(
+          { $or: data.map(d => pickKeys(d, id_fields)) },
+          {
+            select: [synced_field, ...id_fields],
+            orderBy: (orderByAsc as OrderBy),
+          },
+          undefined,
+          table_rules
+        );
+        let inserts = data.filter(d => !existingData.find(ed => rowsIdsMatch(ed, d)));
+        let updates = data.filter(d => existingData.find(ed => rowsIdsMatch(ed, d) && +ed[synced_field] < +d[synced_field]));
+        try {
+          if (!table_rules) throw "table_rules missing";
+          
+          if (table_rules.update && updates.length) {
+            const updateData: [any, any][] = [];
+            await Promise.all(updates.map(upd => {
+              const id_filter = pickKeys(upd, id_fields);
+              const syncSafeFilter = { $and: [id_filter, { [synced_field]: { "<": upd[synced_field] } }] }
+
+              updateData.push([syncSafeFilter, omitKeys(upd, id_fields)])
+            }));
+            await tbl.updateBatch(updateData, { fixIssues: true }, table_rules);
+          } else {
+            updates = [];
+          }
+
+          if (table_rules.insert && inserts.length) {
+            await tbl.insert(inserts, { fixIssues: true }, undefined, table_rules);
+          } else {
+            inserts = [];
+          }
+
+          return { inserts, updates };
+        } catch (e) {
+          console.trace(e);
+          throw e;
+        }
+
+      }).then(({ inserts, updates }) => {
+        log(`upsertData: inserted( ${inserts.length} )    updated( ${updates.length} )     total( ${data.length} ) \n last insert ${JSON.stringify(inserts.at(-1))} \n last update ${JSON.stringify(updates.at(-1))}`);
+        return { inserted: inserts.length, updated: updates.length, total: data.length };
+      })
+      .catch(err => {
+        console.trace("Something went wrong with syncing to server: \n ->", err, data.length, id_fields);
+        return Promise.reject("Something went wrong with syncing to server: ")
+      });
+
+      await this._log({ 
+        type: "sync", 
+        command: "upsertData", 
+        tableName: sync.table_name, 
+        rows: data.length, 
+        socketId: socket_id,
+        sid: sync.sid,
+        duration: Date.now() - start,
+        connectedSocketIds: this.dboBuilder.prostgles.connectedSockets.map(s => s.id)
+      });
+
+      return result;
+    },
+
+    /**
+     * Pushes the given data to client
+     * @param isSynced = true if 
+     */
+    pushData = async (data?: AnyObject[], isSynced = false, err: any = null) => {
+      const start = Date.now();
+      const result = await new Promise((resolve, reject) => {
+        socket.emit(channel_name, { data, isSynced }, (resp?: { ok: boolean }) => {
+
+          if (resp && resp.ok) {
+            // console.log("PUSHED to client: fr/lr", data[0], data[data.length - 1]);
+            resolve({ pushed: data?.length, resp })
+          } else {
+            reject(resp);
+            console.error("Unexpected response");
+          }
+        });
+      });
+
+      await this._log({ 
+        type: "sync", 
+        command: "pushData", 
+        tableName: sync.table_name, 
+        rows: data?.length ?? 0, 
+        socketId: socket_id,
+        duration: Date.now() - start,
+        sid: sync.sid,
+        connectedSocketIds: this.dboBuilder.prostgles.connectedSockets.map(s => s.id)
+      });
+
+      return result;
+    },
+
+    /**
+     * Returns the lowest synced_field between server and client by checking client and server sync data. 
+     * If last rows don't match it will find an earlier matching last row and use that last matching from_synced
+     * If no rows or fully synced (c_lr and s_lr match) then returns null
+     */
+    getLastSynced = async (clientSyncInfo?: ClientSyncInfo): Promise<number | null> => {
+
+      // Get latest row info
+      const { c_fr, c_lr, c_count } = clientSyncInfo || await getClientRowInfo();
+      const { s_fr, s_lr, s_count } = await getServerRowInfo();
+
+      // console.log("getLastSynced", clientData, socket._user )
+
+      let result = null;
+
+      /* Nothing to sync */
+      if (!c_fr && !s_fr || rowsFullyMatch(c_lr, s_lr)) {  //  c_count === s_count && 
+        // sync.last_synced = null;
+        result = null;
+
+        /* Sync Everything */
+      } else if (!rowsFullyMatch(c_fr, s_fr)) {
+        if (c_fr && s_fr) {
+          result = Math.min(c_fr[synced_field], s_fr[synced_field]);
+
+        } else if (c_fr || s_fr) {
+          result = (c_fr || s_fr)[synced_field];
+        }
+
+        /* Sync from last matching synced value */
+      } else if (rowsFullyMatch(c_fr, s_fr)) {
+
+        if (s_lr && c_lr) {
+          result = Math.min(...getNumbers([c_lr[synced_field], s_lr[synced_field]]));
+        } else {
+          result = Math.min(...getNumbers([c_fr[synced_field], s_fr?.[synced_field]]));
+        }
+
+        const min_count = Math.min(...getNumbers([c_count, s_count]));
+        let end_offset = 1;// Math.min(s_count, c_count) - 1;
+        let step = 0;
+
+        while (min_count > 5 && end_offset < min_count) {
+          const { c_lr = null } = await getClientRowInfo({ from_synced: 0, to_synced: result, end_offset });
+          // console.log("getLastSynced... end_offset > " + end_offset);
+          let server_row;
+
+          if (c_lr) {
+            const _filter: AnyObject = {};
+            sync_fields.map(key => {
+              _filter[key] = c_lr[key];
+            });
+            server_row = await this?.dbo?.[table_name]?.find?.(_filter, { select: sync_fields, limit: 1 }, undefined, table_rules);
+          }
+
+          // if(rowsFullyMatch(c_lr, s_lr)){ //c_count === s_count && 
+          if (server_row && server_row.length) {
+            server_row = server_row[0];
+
+            result = +server_row[synced_field];
+            end_offset = min_count;
+            // console.log(`getLastSynced found for ${table_name} -> ${result}`);
+          } else {
+            end_offset += 1 + step * (step > 4 ? 2 : 1);
+            // console.log(`getLastSynced NOT found for ${table_name} -> ${result}`);
+          }
+
+          step++;
+        }
+      }
+
+      return result;
+    },
+
+    updateSyncLR = (data: AnyObject) => {
+      if (data.length) {
+        const lastRow = data[data.length - 1];
+        if (sync.lr?.[synced_field] && +sync.lr?.[synced_field] > +lastRow[synced_field]) {
+          console.error({ syncIssue: "sync.lr[synced_field] is greater than lastRow[synced_field]" }, sync.table_name)
+        }
+        sync.lr = lastRow;
+        sync.last_synced = +sync.lr?.[synced_field];
+      }
+    },
+
+    /**
+     * Will push pull sync between client and server from a given from_synced value
+     */
+    syncBatch = async (from_synced: SyncBatchInfo["from_synced"]) => {
+      let offset = 0,
+        canContinue = true;
+      const limit = batch_size,
+        min_synced = from_synced || 0,
+        max_synced = from_synced;
+
+      let inserted = 0, updated = 0, pushed = 0, deleted = 0, total = 0;
+
+      // console.log("syncBatch", from_synced)
+
+      while (canContinue) {
+        const cData = await getClientData(min_synced, offset);
+
+
+        if (cData.length) {
+          const res = await upsertData(cData);
+          inserted += res.inserted;
+          updated += res.updated;
+        }
+        let sData: AnyObject[] | undefined;
+
+        try {
+          sData = await getServerData(min_synced, offset);
+        } catch (e) {
+          console.trace("sync getServerData err", e);
+          await pushData(undefined, undefined, "Internal error. Check server logs");
+          throw " d"
+        }
+
+        // console.log("allow_delete", table_rules.delete);
+        if (allow_delete && table_rules?.delete) {
+          const to_delete = sData.filter(d => {
+            !cData.find(c => rowsIdsMatch(c, d))
+          });
+          await Promise.all(to_delete.map(d => {
+            deleted++;
+            return (this.dbo[table_name] as TableHandler).delete(pickKeys(d, id_fields), {}, undefined, table_rules);
+          }));
+          sData = await getServerData(min_synced, offset);
+        }
+
+        const forClient = sData.filter(s => {
+          return !cData.find(c =>
+            rowsIdsMatch(c, s) &&
+            +c[synced_field] >= +s[synced_field]
+          );
+        });
+        if (forClient.length) {
+          const res: any = await pushData(forClient.filter(d => !sync.wal || !sync.wal.isInHistory(d)));
+          pushed += res.pushed;
+        }
+
+        if (sData.length) {
+          updateSyncLR(sData);
+          total += sData.length;
+        }
+        offset += sData.length;
+
+        // canContinue = offset >= limit;
+        canContinue = sData.length >= limit;
+        // console.log(`sData ${sData.length}      limit ${limit}`);
+      }
+      log(`server.syncBatch ${table_name}: inserted( ${inserted} )    updated( ${updated} )   deleted( ${deleted} )    pushed to client( ${pushed} )     total( ${total} )`, socket._user );
+
+      return true;
+    };
+
+  if (!wal) {
+    /* Used to throttle and merge incomming updates */
+    sync.wal = new WAL({
+      id_fields, synced_field, throttle, batch_size,
+      DEBUG_MODE: this.dboBuilder.prostgles.opts.DEBUG_MODE,
+      onSendStart: () => {
+        sync.is_syncing = true;
+      },
+      onSend: async (data) => {
+        // console.log("WAL upsertData START", data)
+        const res = await upsertData(data);
+        // const max_incoming_synced = Math.max(...data.map(d => +d[synced_field]));
+        // if(Number.isFinite(max_incoming_synced) && max_incoming_synced > +sync.last_synced){
+        //     sync.last_synced = max_incoming_synced;
+        // }
+        // console.log("WAL upsertData END")
+
+        /******** */
+        /* TO DO -> Store and push patch updates instead of full data if and where possible */
+        /******** */
+        // 1. Store successfully upserted wal items for a couple of seconds
+        // 2. When pushing data to clients check if any matching wal items exist
+        // 3. Replace text fields with matching patched data
+
+        return res;
+      },
+      onSendEnd: (batch) => {
+        updateSyncLR(batch);
+        sync.is_syncing = false;
+        // console.log("syncData from WAL.onSendEnd")
+
+        /**
+         * After all data was inserted request SyncInfo from client and sync again if necessary
+         */
+        this.syncData(sync, undefined, source);
+      },
+    })
+  }
+
+  /* Debounce sync requests */
+  if (!sync.wal) throw "sync.wal missing";
+  if (!sync.wal.isSending() && sync.is_syncing) {
+    if (!this.syncTimeout) {
+      this.syncTimeout = setTimeout(() => {
+        this.syncTimeout = undefined;
+        // console.log("SYNC FROM TIMEOUT")
+        this.syncData(sync, undefined, source);
+      }, throttle)
+    }
+    // console.log("SYNC THROTTLE")
+    return;
+  }
+
+  // console.log("syncData", clientData)
+
+  /**
+   * Express data sent from a client that has already been synced
+   * Add to WAL manager which will sync at the end
+   */
+  if (clientData) {
+    if (clientData.data && Array.isArray(clientData.data) && clientData.data.length) {
+      if (!sync.wal) throw "sync.wal missing";
+
+      sync.wal.addData(clientData.data.map(d => ({ current: d })));
+      return;
+      // await upsertData(clientData.data, true);
+
+      /* Not expecting this anymore. use normal db.table.delete channel */
+    } else if (clientData.deleted && Array.isArray(clientData.deleted) && clientData.deleted.length) {
+      await deleteData(clientData.deleted);
+    }
+  } else {
+    // do nothing
+  }
+  if (sync.wal.isSending()) return;
+
+  sync.is_syncing = true;
+
+  // from synced does not make sense. It should be sync.lr only!!!
+  let from_synced = null;
+
+  /** Sync was already synced */
+  if (sync.lr) {
+    const { s_lr } = await getServerRowInfo();
+
+    /* Make sure trigger is not firing on freshly synced data */
+    if (!rowsFullyMatch(sync.lr, s_lr)) {
+      from_synced = sync.last_synced;
+    } else {
+      // console.log("rowsFullyMatch")
+    }
+    // console.log(table_name, sync.lr[synced_field])
+  } else {
+    from_synced = await getLastSynced(clientData);
+  }
+
+  if (from_synced !== null) {
+    await syncBatch(from_synced);
+  } else {
+    // console.log("from_synced is null")
+  }
+
+  await pushData([], true);
+
+  sync.is_syncing = false;
+  // console.log(`Finished sync for ${table_name}`, socket._user);
+}