prostgles-server 4.2.639 → 4.2.640

package/lib/Logging.ts

@@ -43,6 +43,11 @@ export namespace EventTypes {
       channelName: string;
       syncParams: SyncParams;
     } & (
+      | {
+          command: "replicationError";
+          source: "client" | "trigger" | "WAL";
+          error: unknown;
+        }
       | {
           command: "syncData";
           source: "client" | "trigger";

package/lib/Prostgles.ts

@@ -13,7 +13,7 @@ import type { ProstglesInitOptions } from "./ProstglesTypes";
 import { RestApi } from "./RestApi";
 import { TableConfigurator } from "./TableConfig/TableConfigurator";
 
-import type { PRGLIOSocket } from "./DboBuilder/DboBuilder";
+import type { ClientSocketWithCachedData } from "./DboBuilder/DboBuilder";
 import { DBHandlerServer, DboBuilder } from "./DboBuilder/DboBuilder";
 export { DBHandlerServer };
 export type PGP = pgPromise.IMain<{}, pg.IClient>;
@@ -401,7 +401,7 @@ export class Prostgles {
 
   init = initProstgles.bind(this);
 
-  connectedSockets: PRGLIOSocket[] = [];
+  connectedSockets: ClientSocketWithCachedData[] = [];
   setupSocketIO() {
     this.checkDb();
 

package/lib/ProstglesTypes.ts

@@ -16,7 +16,7 @@ import type { TableConfig } from "./TableConfig/TableConfigTypes";
 import type { TableHooks } from "./TableHooks/TableHooks";
 import type { ResolvedAuditConfig, SchemaConfigAudit } from "./Audit/AuditTypes";
 
-import type { PRGLIOSocket } from "./DboBuilder/DboBuilder";
+import type { ClientSocketWithCachedData } from "./DboBuilder/DboBuilder";
 
 import type e from "express";
 import type pgPromise from "pg-promise";
@@ -248,14 +248,14 @@ export type ProstglesInitOptions<
    * Use for connection verification. Will disconnect socket on any errors
    */
   onSocketConnect?: (
-    args: AuthRequestParams<S, SUser> & { socket: PRGLIOSocket },
+    args: AuthRequestParams<S, SUser> & { socket: ClientSocketWithCachedData },
   ) => void | Promise<void>;
 
   /**
    * Called when a socket disconnects
    */
   onSocketDisconnect?: (
-    args: AuthRequestParams<S, SUser> & { socket: PRGLIOSocket },
+    args: AuthRequestParams<S, SUser> & { socket: ClientSocketWithCachedData },
   ) => void | Promise<void>;
 
   /**

package/lib/PubSubManager/PubSubManager.ts

@@ -7,7 +7,7 @@ import type {
   DBHandlerServer,
   DboBuilder,
   LocalParams,
-  PRGLIOSocket,
+  ClientSocketWithCachedData,
 } from "../DboBuilder/DboBuilder";
 import type { PostgresNotifListenManager } from "../PostgresNotifListenManager";
 import type { DB } from "../Prostgles";
@@ -60,12 +60,13 @@ export type SyncParams = {
   throttle?: number;
   lr?: AnyObject;
   is_syncing: boolean;
+  syncQueue?: Promise<void>;
   handlers: ReturnType<(typeof ReplicationProtocol)["getServerHandlers"]>;
   localParams: LocalParams;
 };
 
 export type AddSyncParams = SyncTableInfo & {
-  socket: PRGLIOSocket;
+  socket: ClientSocketWithCachedData;
   table_info: TableSchema;
   table_rules: ParsedTableRule;
   allow_delete?: boolean;
@@ -120,7 +121,7 @@ export type SubscriptionParams = {
   subscribeOptions: SubscribeOptions;
 
   onData?: OnData;
-  socket: PRGLIOSocket | undefined;
+  socket: ClientSocketWithCachedData | undefined;
   localParams?: LocalParams;
 
   lastPushed: number;
@@ -196,7 +197,7 @@ export class PubSubManager {
    * Updated through refreshTriggers()
    */
   _triggers: PubSubManagerTriggers = new Map();
-  sockets: Map<string, PRGLIOSocket> = new Map();
+  sockets: Map<string, ClientSocketWithCachedData> = new Map();
 
   subs: Subscription[] = [];
   syncs: SyncParams[] = [];
@@ -266,7 +267,7 @@ export class PubSubManager {
 
   removeSubscription = (
     channelName: string,
-    subInfo: { type: "local"; onData: OnData } | { type: "ws"; socket: PRGLIOSocket },
+    subInfo: { type: "local"; onData: OnData } | { type: "ws"; socket: ClientSocketWithCachedData },
   ) => {
     const matchingSubIdx = this.subs.findIndex(
       (s) =>
@@ -364,7 +365,7 @@ export class PubSubManager {
 
   pushSubData = pushSubData.bind(this);
 
-  upsertSocket(socket: PRGLIOSocket | undefined) {
+  upsertSocket(socket: ClientSocketWithCachedData | undefined) {
     if (socket && !this.sockets.get(socket.id)) {
       this.sockets.set(socket.id, socket);
       socket.on("disconnect", () => {

package/lib/PubSubManager/SyncReplication/getSyncUtilFunctions.ts

@@ -6,7 +6,7 @@ import {
   type AnyObject,
   type SyncBatchParams,
 } from "prostgles-types";
-import type { LocalParams, PRGLIOSocket } from "../../DboBuilder/DboBuilder";
+import type { LocalParams, ClientSocketWithCachedData } from "../../DboBuilder/DboBuilder";
 import type { TableHandler } from "../../DboBuilder/TableHandler/TableHandler";
 import { getSyncOrderByAndFields } from "./getSyncOrderByAndFields";
 import type { PubSubManager, SyncParams } from "../PubSubManager";
@@ -21,7 +21,7 @@ import { log } from "../PubSubManagerUtils";
 import type { EventTypes } from "../../Logging";
 
 type Args = {
-  socket: PRGLIOSocket;
+  socket: ClientSocketWithCachedData;
   tableHandler: TableHandler;
   sync: SyncParams;
   pubSubManager: PubSubManager;
@@ -389,12 +389,7 @@ export const getSyncUtilFunctions = ({
         return;
       }
       if (sync.lr?.[synced_field] && +sync.lr[synced_field] > +lastRow[synced_field]) {
-        console.error(
-          {
-            syncIssue: "sync.lr[synced_field] is greater than lastRow[synced_field]",
-          },
-          sync.table_name,
-        );
+        return;
       }
       sync.lr = lastRow;
     },

package/lib/PubSubManager/SyncReplication/logSyncError.ts

@@ -0,0 +1,22 @@
+import type { EventTypes } from "../../Logging";
+import type { PubSubManager, SyncParams } from "../PubSubManager";
+
+export const logSyncError = (
+  pubSubManager: PubSubManager,
+  sync: SyncParams,
+  source: Extract<EventTypes.Sync, { command: "replicationError" }>["source"],
+  error: unknown,
+) =>
+  pubSubManager._log({
+    type: "sync",
+    command: "replicationError",
+    source,
+    error,
+    tableName: sync.table_name,
+    channelName: sync.channel_name,
+    socketId: sync.socket_id,
+    sid: sync.sid,
+    connectedSocketIds: pubSubManager.connectedSocketIds,
+    syncParams: sync,
+    duration: -1,
+  });

package/lib/PubSubManager/SyncReplication/syncData.ts

@@ -4,6 +4,7 @@ import { WAL } from "prostgles-types/dist/WAL";
 import type { PubSubManager, SyncParams } from "../PubSubManager";
 import { getSyncUtilFunctions } from "./getSyncUtilFunctions";
 import type { EventTypes } from "../../Logging";
+import { logSyncError } from "./logSyncError";
 
 export type ClientSyncInfo = Partial<{
   c_fr: AnyObject;
@@ -45,7 +46,26 @@ export type ClientExpressData = ClientSyncInfo & {
 /**
  * Server or client requested data sync
  */
-export async function syncData(
+export function syncData(
+  this: PubSubManager,
+  sync: SyncParams,
+  clientData: ClientExpressData | undefined,
+  source: "trigger" | "client",
+) {
+  const run = () =>
+    runSyncData.call(this, sync, clientData, source).catch(async (error) => {
+      if (this.sockets.get(sync.socket_id)?.connected) {
+        await logSyncError(this, sync, source, error);
+      }
+      throw error;
+    });
+  const queued = (sync.syncQueue ?? Promise.resolve()).then(run);
+  // Report failures to the caller while allowing the next request to run.
+  sync.syncQueue = queued.catch(() => undefined);
+  return queued;
+}
+
+async function runSyncData(
   this: PubSubManager,
   sync: SyncParams,
   clientData: ClientExpressData | undefined,
@@ -111,8 +131,12 @@ export async function syncData(
       sync.is_syncing = true;
     },
     onSend: async (data) => {
-      const res = await upsertData(data, "WAL");
-      return res;
+      try {
+        return await upsertData(data, "WAL");
+      } catch (error) {
+        await logSyncError(this, sync, "WAL", error);
+        throw error;
+      }
     },
     onSendEnd: (batch, _, error) => {
       if (error === undefined) {
@@ -164,36 +188,36 @@ export async function syncData(
   }
 
   sync.is_syncing = true;
+  try {
+    let from_synced = null;
+
+    /** Client runs must reconcile the client's snapshot after reattachment. */
+    if (sync.lr && !clientData) {
+      const { s_lr } = await getServerRowInfo();
+
+      /* Make sure trigger is not firing on freshly synced data */
+      if (!rowsFullyMatch(sync.lr, s_lr)) {
+        from_synced = Number(sync.lr[synced_field]);
+        await logSyncData("sync.lr");
+      } else {
+        await logSyncData("rowsFullyMatch");
+      }
+    } else {
+      await logSyncData("getLastSynced(clientData).start");
+      from_synced = await getLastSynced(clientData);
+      await logSyncData("getLastSynced(clientData).end");
+    }
 
-  // from synced does not make sense. It should be sync.lr only!!!
-  let from_synced = null;
-
-  /** Sync was already synced */
-  if (sync.lr) {
-    const { s_lr } = await getServerRowInfo();
-
-    /* Make sure trigger is not firing on freshly synced data */
-    if (!rowsFullyMatch(sync.lr, s_lr)) {
-      from_synced = Number(sync.lr[synced_field]);
-      await logSyncData("sync.lr");
+    if (from_synced !== null) {
+      await logSyncData("syncBatch.start");
+      await syncBatch(from_synced);
+      await logSyncData("syncBatch.end");
     } else {
-      await logSyncData("rowsFullyMatch");
+      await logSyncData("nothingToSync");
     }
-  } else {
-    await logSyncData("getLastSynced(clientData).start");
-    from_synced = await getLastSynced(clientData);
-    await logSyncData("getLastSynced(clientData).end");
-  }
 
-  if (from_synced !== null) {
-    await logSyncData("syncBatch.start");
-    await syncBatch(from_synced);
-    await logSyncData("syncBatch.end");
-  } else {
-    await logSyncData("nothingToSync");
+    await pushData({ state: "synced" });
+  } finally {
+    sync.is_syncing = false;
   }
-
-  await pushData({ state: "synced" });
-
-  sync.is_syncing = false;
 }

package/lib/PubSubManager/addSync.ts

@@ -40,7 +40,6 @@ export async function addSync(
       /* Only a sync per socket per table/condition/select allowed */
       const existing = find(this.syncs, { socket_id: socket.id, channel_name: channelName });
       if (existing) {
-        console.warn("addSync: Client tried to create a duplicate sync", existing.channel_name);
         return existing;
       }
 
@@ -112,18 +111,18 @@ export async function addSync(
         cb(null, { res: "ok" });
       });
 
+      this.syncs.push(newSync);
       return newSync;
     };
 
-    const newSync = upsertSync();
-
     await this.addTrigger(
       { table_name, condition: conditionParsed, tracked_columns: undefined },
       undefined,
       socket,
     );
 
-    this.syncs.push(newSync);
+    // Check and register without yielding so concurrent attachments share one sync.
+    const newSync = upsertSync();
     return { channelName, newSync };
   });
 

package/lib/PubSubManager/addTrigger.ts

@@ -1,6 +1,6 @@
 import * as crypto from "crypto";
 import { asName, pickKeys, tryCatchV2 } from "prostgles-types";
-import type { PRGLIOSocket } from "../DboBuilder/DboBuilderTypes";
+import type { ClientSocketWithCachedData } from "../DboBuilder/DboBuilderTypes";
 import type { TableHandler } from "../DboBuilder/TableHandler/TableHandler";
 import type { ViewSubscriptionOptions } from "./PubSubManager";
 import { type PubSubManager } from "./PubSubManager";
@@ -16,7 +16,7 @@ export async function addTrigger(
   this: PubSubManager,
   params: AddTriggerParams,
   viewOptions: ViewSubscriptionOptions | undefined,
-  socket: PRGLIOSocket | undefined,
+  socket: ClientSocketWithCachedData | undefined,
 ) {
   const addedTrigger = await tryCatchV2(async () => {
     const { table_name } = { ...params };

package/lib/WebsocketAPI/onSocketConnected.ts

@@ -2,12 +2,12 @@ import type { AnyObject, SocketFunctionCall } from "prostgles-types";
 import { CHANNELS, getSerialisableError, isObject } from "prostgles-types";
 import { getClientRequestIPsInfo } from "../Auth/AuthHandler";
 import type { AuthResultWithSID, SessionUser } from "../Auth/AuthTypes";
-import type { PRGLIOSocket } from "../DboBuilder/DboBuilderTypes";
+import type { ClientSocketWithCachedData } from "../DboBuilder/DboBuilderTypes";
 import type { DBOFullyTyped } from "../DBSchemaBuilder/DBSchemaBuilder";
 import type { Prostgles, TABLE_METHODS } from "../Prostgles";
 import { runClientMethod, runClientRequest } from "../runClientRequest";
 
-export async function onSocketConnected(this: Prostgles, socket: PRGLIOSocket) {
+export async function onSocketConnected(this: Prostgles, socket: ClientSocketWithCachedData) {
   const isDestroyed = () => this.destroyed;
   if (isDestroyed()) {
     console.log("Socket connected to destroyed instance");

package/lib/WebsocketAPI/pushSocketSchema.ts

@@ -1,9 +1,9 @@
 import { CHANNELS, getSerialisableError, type SQLRequest } from "prostgles-types";
-import type { PRGLIOSocket } from "../DboBuilder/DboBuilderTypes";
+import type { ClientSocketWithCachedData } from "../DboBuilder/DboBuilderTypes";
 import type { Prostgles } from "../Prostgles";
 import { runClientSqlRequest } from "../runClientRequest";
 import { makeSocketError } from "./onSocketConnected";
-export async function pushSocketSchema(this: Prostgles, socket: PRGLIOSocket) {
+export async function pushSocketSchema(this: Prostgles, socket: ClientSocketWithCachedData) {
   const isDestroyed = () => this.destroyed;
   if (isDestroyed()) return;
   try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prostgles-server",
-  "version": "4.2.639",
+  "version": "4.2.640",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",