prostgles-server 2.0.157 → 2.0.158

package/lib/SyncReplication.ts

@@ -1,5 +1,5 @@
 
-import { PubSubManager, SyncParams, filterObj, log } from "./PubSubManager";
+import { PubSubManager, SyncParams, pickKeys, omitKeys, log } from "./PubSubManager";
 import { OrderBy, WAL, AnyObject, SyncBatchParams } from "prostgles-types";
 import { TableHandler } from "./DboBuilder";
 
@@ -42,8 +42,8 @@ function getNumbers(numberArr: (null | undefined | string | number)[]): number[]
     return numberArr.filter(v => v !== null && v !== undefined && Number.isFinite(+v)) as any;
 }
 
-export const syncData = async (_this: PubSubManager, sync: SyncParams, clientData?: ClientExpressData) => {
-    // console.log("S", clientData)
+export const syncData = async (_this: PubSubManager, sync: SyncParams, clientData: ClientExpressData | undefined, source: "trigger" | "client") => {
+    log("syncData", { clientData, sync: pickKeys(sync, ["filter", "last_synced", "lr", "is_syncing"]) , source })
     const { 
             socket_id, channel_name, table_name, filter, 
             table_rules, allow_delete = false, params,
@@ -162,7 +162,7 @@ export const syncData = async (_this: PubSubManager, sync: SyncParams, clientDat
             // console.log("deleteData deleteData  deleteData " + deleted.length);
             if(allow_delete){
                 return Promise.all(deleted.map(async d => {
-                    const id_filter = filterObj(d, id_fields);
+                    const id_filter = pickKeys(d, id_fields);
                     try {
                         await (_this.dbo[table_name] as TableHandler).delete(id_filter, undefined, undefined, table_rules);
                         return 1;
@@ -181,16 +181,13 @@ export const syncData = async (_this: PubSubManager, sync: SyncParams, clientDat
          * Upserts the given client data where synced_field is higher than on server  
          */
         upsertData = (data: AnyObject[], isExpress = false) => {
-            let inserted = 0,
-                updated = 0,
-                total = data.length;
 
             // console.log("isExpress", isExpress, data);
 
-            return _this.dboBuilder.getTX(async dbTX => {
+            return _this.dboBuilder.getTX(async (dbTX) => {
                 const tbl = dbTX[table_name] as TableHandler;
                 const existingData = await tbl.find(
-                    { $or: data.map(d => filterObj(d, id_fields)) }, 
+                    { $or: data.map(d => pickKeys(d, id_fields)) }, 
                     { 
                         select: [synced_field, ...id_fields] , 
                         orderBy: (orderByAsc as OrderBy), 
@@ -198,37 +195,40 @@ export const syncData = async (_this: PubSubManager, sync: SyncParams, clientDat
                     undefined, 
                     table_rules
                 );
-                const inserts = data.filter(d => !existingData.find(ed => rowsIdsMatch(ed, d)));
-                const updates = data.filter(d =>  existingData.find(ed => rowsIdsMatch(ed, d) && +ed[synced_field] < +d[synced_field]) );
+                let inserts = data.filter(d => !existingData.find(ed => rowsIdsMatch(ed, d)));
+                let updates = data.filter(d =>  existingData.find(ed => rowsIdsMatch(ed, d) && +ed[synced_field] < +d[synced_field]) );
                 try {
                     if(!table_rules) throw "table_rules missing"
                     if(table_rules.update && updates.length){
                         let updateData: [any, any][] = [];
                         await Promise.all(updates.map(upd => {
-                            const id_filter = filterObj(upd, id_fields);
+                            const id_filter = pickKeys(upd, id_fields);
                             const syncSafeFilter = { $and: [id_filter, { [synced_field]: { "<": upd[synced_field] } } ] }
-                            // return tbl.update(syncSafeFilter, filterObj(upd, [], id_fields), { fixIssues: true }, table_rules)
-                            updateData.push([syncSafeFilter, filterObj(upd, [], id_fields)])
+                            
+                            updateData.push([syncSafeFilter, omitKeys(upd, id_fields)])
                         }));
-                        await tbl.updateBatch(updateData, { fixIssues: true }, table_rules)
-                        updated = updates.length;
+                        await tbl.updateBatch(updateData, { fixIssues: true }, table_rules);
+                    } else {
+                        updates = [];
                     }
+
                     if(table_rules.insert && inserts.length){
                         // const qs = await tbl.insert(inserts, { fixIssues: true }, null, table_rules, { returnQuery: true });
                         // console.log("inserts", qs)
                         await tbl.insert(inserts, { fixIssues: true }, undefined, table_rules);
-                        inserted = inserts.length;                       
+                    } else {
+                        inserts = [];
                     }
                     
-                    return true;
+                    return { inserts, updates };
                 } catch(e) {
                     console.trace(e);
                     throw e;
                 }
 
-            }).then(res => {
-                log(`upsertData: inserted( ${inserted} )    updated( ${updated} )     total( ${total} )`);
-                return { inserted , updated , total };
+            }).then(({ inserts, updates }) => {
+                log(`upsertData: inserted( ${inserts.length} )    updated( ${updates.length} )     total( ${data.length} ) \n last insert ${JSON.stringify(inserts.at(-1))} \n last update ${JSON.stringify(updates.at(-1))}`);
+                return { inserted: inserts.length , updated: updates.length , total: data.length };
             })
             .catch(err => {
                 console.trace("Something went wrong with syncing to server: \n ->", err, data.length, id_fields);
@@ -381,7 +381,7 @@ export const syncData = async (_this: PubSubManager, sync: SyncParams, clientDat
                     });
                     await Promise.all(to_delete.map(d => {
                         deleted++;
-                        return (_this.dbo[table_name] as TableHandler).delete(filterObj(d, id_fields), { }, undefined, table_rules);
+                        return (_this.dbo[table_name] as TableHandler).delete(pickKeys(d, id_fields), { }, undefined, table_rules);
                     }));
                     sData = await getServerData(min_synced, offset);
                 }
@@ -447,7 +447,7 @@ export const syncData = async (_this: PubSubManager, sync: SyncParams, clientDat
                 /**
                  * After all data was inserted request SyncInfo from client and sync again if necessary
                  */
-                _this.syncData(sync, undefined);
+                _this.syncData(sync, undefined, source);
             },
         })
     }
@@ -459,7 +459,7 @@ export const syncData = async (_this: PubSubManager, sync: SyncParams, clientDat
             _this.syncTimeout = setTimeout(() => {
                 _this.syncTimeout = undefined;
                 // console.log("SYNC FROM TIMEOUT")
-                _this.syncData(sync, undefined);
+                _this.syncData(sync, undefined, source);
             }, throttle)
         }
         // console.log("SYNC THROTTLE")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prostgles-server",
-  "version": "2.0.157",
+  "version": "2.0.158",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -29,7 +29,7 @@
     "bluebird": "^3.7.2",
     "file-type": "^16.5.3",
     "pg-promise": "^10.11.1",
-    "prostgles-types": "^1.5.130",
+    "prostgles-types": "^1.5.131",
     "sharp": "^0.30.5"
   },
   "devDependencies": {

package/tests/client/PID.txt

@@ -1 +1 @@
-12615
+23125

package/tests/client/package-lock.json

@@ -11,7 +11,7 @@
       "dependencies": {
         "@types/node": "^14.14.16",
         "@types/socket.io-client": "^1.4.35",
-        "prostgles-client": "^1.5.143",
+        "prostgles-client": "^1.5.144",
         "prostgles-types": "^1.5.68",
         "socket.io-client": "^4.5.1"
       }
@@ -73,17 +73,17 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "node_modules/prostgles-client": {
-      "version": "1.5.143",
-      "resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-1.5.143.tgz",
-      "integrity": "sha512-1+YpgEAvDCdbl0MWZnSjr75Wjo7wDYOSAVCflunJpUs9zsELqIumB1dCmrMRDCKM4UsQxgoFLHDtU6MxJDm7Ug==",
+      "version": "1.5.144",
+      "resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-1.5.144.tgz",
+      "integrity": "sha512-IbLQ6PC3bUgRUHiWQs0p3PXdyAQWprPbRM7VZ4OCAuPzAnOdjyriB+psa04li1AMs9EtKpMkAil+Hks6tgyHpA==",
       "dependencies": {
-        "prostgles-types": "^1.5.130"
+        "prostgles-types": "^1.5.131"
       }
     },
     "node_modules/prostgles-types": {
-      "version": "1.5.130",
-      "resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-1.5.130.tgz",
-      "integrity": "sha512-Kv3c6ennbBVITVjdFYFS9WVNzbC1XVnPN59kA73zgzbhRbRjcHRCzW9MbZjZeHvX5PTICHF4a+XnpHMVF58tTA=="
+      "version": "1.5.131",
+      "resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-1.5.131.tgz",
+      "integrity": "sha512-gJUrlG2JcDFglKMLAf7flMHOQ5+rPSkBct4zhMTJU35wDEU4c9QWuoLaSMj3zhvw4q/wCb4tz4l3nCrNbdPCGw=="
     },
     "node_modules/socket.io-client": {
       "version": "4.5.1",
@@ -187,17 +187,17 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "prostgles-client": {
-      "version": "1.5.143",
-      "resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-1.5.143.tgz",
-      "integrity": "sha512-1+YpgEAvDCdbl0MWZnSjr75Wjo7wDYOSAVCflunJpUs9zsELqIumB1dCmrMRDCKM4UsQxgoFLHDtU6MxJDm7Ug==",
+      "version": "1.5.144",
+      "resolved": "https://registry.npmjs.org/prostgles-client/-/prostgles-client-1.5.144.tgz",
+      "integrity": "sha512-IbLQ6PC3bUgRUHiWQs0p3PXdyAQWprPbRM7VZ4OCAuPzAnOdjyriB+psa04li1AMs9EtKpMkAil+Hks6tgyHpA==",
       "requires": {
-        "prostgles-types": "^1.5.130"
+        "prostgles-types": "^1.5.131"
       }
     },
     "prostgles-types": {
-      "version": "1.5.130",
-      "resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-1.5.130.tgz",
-      "integrity": "sha512-Kv3c6ennbBVITVjdFYFS9WVNzbC1XVnPN59kA73zgzbhRbRjcHRCzW9MbZjZeHvX5PTICHF4a+XnpHMVF58tTA=="
+      "version": "1.5.131",
+      "resolved": "https://registry.npmjs.org/prostgles-types/-/prostgles-types-1.5.131.tgz",
+      "integrity": "sha512-gJUrlG2JcDFglKMLAf7flMHOQ5+rPSkBct4zhMTJU35wDEU4c9QWuoLaSMj3zhvw4q/wCb4tz4l3nCrNbdPCGw=="
     },
     "socket.io-client": {
       "version": "4.5.1",

package/tests/client/package.json

@@ -14,7 +14,7 @@
   "dependencies": {
     "@types/node": "^14.14.16",
     "@types/socket.io-client": "^1.4.35",
-    "prostgles-client": "^1.5.143",
+    "prostgles-client": "^1.5.144",
     "prostgles-types": "^1.5.68",
     "socket.io-client": "^4.5.1"
   }

package/tests/client_only_queries.js

@@ -4,14 +4,13 @@ const assert_1 = require("assert");
 const isomorphic_queries_1 = require("./isomorphic_queries");
 async function client_only(db, auth, log, methods) {
     const testRealtime = () => {
-        log("Started testRealtime");
         return new Promise(async (resolve, reject) => {
             try {
                 /* METHODS */
                 const t222 = await methods.get();
                 assert_1.strict.equal(t222, 222, "methods.get() failed");
                 /* RAWSQL */
-                await (0, isomorphic_queries_1.tryRunP)("SQL Full result", async (resolve, reject) => {
+                await (0, isomorphic_queries_1.tryRun)("SQL Full result", async () => {
                     const sqlStatement = await db.sql("SELECT $1", [1], { returnType: "statement" });
                     assert_1.strict.equal(sqlStatement, "SELECT 1", "db.sql statement query failed");
                     const arrayMode = await db.sql("SELECT 1 as a, 2 as a", undefined, { returnType: "arrayMode" });
@@ -34,29 +33,35 @@ async function client_only(db, auth, log, methods) {
                             udt_name: 'int4',
                             tsDataType: "number"
                         }], "db.sql query failed");
-                    resolve(true);
-                }, log);
+                });
                 await (0, isomorphic_queries_1.tryRunP)("sql LISTEN NOTIFY events", async (resolve, reject) => {
-                    const sub = await db.sql("LISTEN chnl ");
-                    // console.log({ sub })
-                    sub.addListener(notif => {
-                        // console.log({ notif })
-                        if (notif === "hello")
-                            resolve(true);
-                        else
-                            reject("Something went bad");
-                    });
-                    db.sql("NOTIFY chnl , 'hello'; ");
-                }, log);
+                    try {
+                        const sub = await db.sql("LISTEN chnl ", {}, { returnType: "allowListen" });
+                        if (!("addListener" in sub)) {
+                            reject("addListener missing");
+                            return;
+                        }
+                        sub.addListener(notif => {
+                            const expected = "hello";
+                            if (notif === expected)
+                                resolve(true);
+                            else
+                                reject(`Notif value is not what we expect: ${JSON.stringify(notif)} is not ${JSON.stringify(expected)} (expected) `);
+                        });
+                        db.sql("NOTIFY chnl , 'hello'; ");
+                    }
+                    catch (e) {
+                        reject(e);
+                    }
+                });
                 await (0, isomorphic_queries_1.tryRunP)("sql NOTICE events", async (resolve, reject) => {
                     const sub = await db.sql("", {}, { returnType: "noticeSubscription" });
-                    // console.log({ sub })
                     sub.addListener(notice => {
-                        // console.log({ notif })
-                        if (notice.message === "hello2")
+                        const expected = "hello2";
+                        if (notice.message === expected)
                             resolve(true);
                         else
-                            reject("Something went bad");
+                            reject(`Notice value is not what we expect: ${JSON.stringify(notice)} is not ${JSON.stringify(expected)} (expected) `);
                     });
                     db.sql(`
           DO $$ 
@@ -68,7 +73,8 @@ async function client_only(db, auth, log, methods) {
         `);
                 }, log);
                 /* REPLICATION */
-                let start = Date.now();
+                log("Started testRealtime");
+                const start = Date.now();
                 await db.planes.delete();
                 let inserts = new Array(100).fill(null).map((d, i) => ({ id: i, flight_number: `FN${i}`, x: Math.random(), y: i }));
                 await db.planes.insert(inserts);

package/tests/client_only_queries.ts

@@ -7,7 +7,7 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
   
 
   const testRealtime = () => {
-    log("Started testRealtime")
+
     return new Promise(async (resolve, reject) => {
       try {
 
@@ -16,7 +16,7 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
       assert.equal(t222, 222, "methods.get() failed");
   
       /* RAWSQL */
-      await tryRunP("SQL Full result", async (resolve, reject) => {
+      await tryRun("SQL Full result", async () => {
         const sqlStatement = await db.sql("SELECT $1", [1], { returnType: "statement" });
         assert.equal(sqlStatement, "SELECT 1", "db.sql statement query failed");
       
@@ -42,29 +42,37 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
           udt_name: 'int4', 
           tsDataType: "number" 
         }] , "db.sql query failed");
-        resolve(true);
-      }, log);
+      });
 
       await tryRunP("sql LISTEN NOTIFY events", async (resolve, reject) => {
         
-        const sub = await db.sql("LISTEN chnl ");
-        // console.log({ sub })
-        sub.addListener(notif => {
-          // console.log({ notif })
-          if(notif === "hello") resolve(true);
-          else reject("Something went bad")
-        });
-        db.sql("NOTIFY chnl , 'hello'; ");
-      }, log);
+        try {
+          
+          const sub = await db.sql("LISTEN chnl ", {}, { returnType: "allowListen" });
+          if(!("addListener" in sub)) {
+            reject("addListener missing");
+            return
+          }
+
+          sub.addListener(notif => {
+            const expected = "hello"
+            if(notif === expected) resolve(true);
+            else reject(`Notif value is not what we expect: ${JSON.stringify(notif)} is not ${JSON.stringify(expected)} (expected) `)
+          });
+          db.sql("NOTIFY chnl , 'hello'; ");
+        } catch(e){
+          reject(e);
+        }
+      });
 
       await tryRunP("sql NOTICE events", async (resolve, reject) => {
         
         const sub = await db.sql("", {}, { returnType: "noticeSubscription" });
-        // console.log({ sub })
+        
         sub.addListener(notice => {
-          // console.log({ notif })
-          if(notice.message === "hello2") resolve(true);
-          else reject("Something went bad")
+          const expected = "hello2"
+          if(notice.message === expected) resolve(true);
+          else reject(`Notice value is not what we expect: ${JSON.stringify(notice)} is not ${JSON.stringify(expected)} (expected) `)
         });
         db.sql(`
           DO $$ 
@@ -78,7 +86,8 @@ export default async function client_only(db: DBHandlerClient, auth: Auth, log:
 
   
       /* REPLICATION */
-      let start = Date.now();
+      log("Started testRealtime")
+      const start = Date.now();
   
       await db.planes.delete();
       let inserts = new Array(100).fill(null).map((d, i) => ({ id: i, flight_number: `FN${i}`, x: Math.random(), y: i }));