prostgles-server 4.2.639 → 4.2.641

package/lib/Auth/AuthHandler.ts

@@ -1,6 +1,6 @@
 import type { AnyObject } from "prostgles-types";
 import { CHANNELS, getProperty, isDefined } from "prostgles-types";
-import type { PRGLIOSocket } from "../DboBuilder/DboBuilder";
+import type { ClientSocketWithCachedData } from "../DboBuilder/DboBuilder";
 import type { DBOFullyTyped } from "../DBSchemaBuilder/DBSchemaBuilder";
 import type { Prostgles } from "../Prostgles";
 import type { AuthClientRequest, AuthConfig, BasicSession } from "./AuthTypes";
@@ -226,7 +226,7 @@ export class AuthHandler {
   getSidAndUserFromRequest = getSidAndUserFromRequest.bind(this);
 
   isNonExpiredSocketSession = (
-    socket: PRGLIOSocket,
+    socket: ClientSocketWithCachedData,
     session: BasicSession | undefined,
   ): boolean => {
     const hasExpired = Boolean(session && session.expires <= Date.now());

package/lib/Auth/AuthTypes.ts

@@ -24,7 +24,7 @@ import type {
   UserLike,
 } from "prostgles-types";
 import type { DBOFullyTyped } from "../DBSchemaBuilder/DBSchemaBuilder";
-import type { PRGLIOSocket } from "../DboBuilder/DboBuilderTypes";
+import type { ClientSocketWithCachedData } from "../DboBuilder/DboBuilderTypes";
 import { type CachedSession } from "../DboBuilder/DboBuilderTypes";
 import type { DB } from "../Prostgles";
 import { GET_AUTH_ROUTE } from "./AuthHandler";
@@ -52,7 +52,7 @@ export type BasicSession = {
   onExpiration: "redirect" | "show_error";
 };
 
-type SocketClientRequest = { socket: PRGLIOSocket; httpReq?: undefined };
+type SocketClientRequest = { socket: ClientSocketWithCachedData; httpReq?: undefined };
 type HttpClientRequest = {
   httpReq: ExpressReq;
   res: ExpressRes;

package/lib/Auth/utils/getClientRequestIPsInfo.ts

@@ -1,7 +1,7 @@
-import type { PRGLIOSocket } from "../../DboBuilder/DboBuilderTypes";
+import type { ClientSocketWithCachedData } from "../../DboBuilder/DboBuilderTypes";
 import type { ExpressReq, LoginClientInfo } from "../AuthTypes";
 type ClientReq =
-  | { socket: PRGLIOSocket; httpReq?: undefined }
+  | { socket: ClientSocketWithCachedData; httpReq?: undefined }
   | { httpReq: ExpressReq; socket?: undefined };
 export const getClientRequestIPsInfo = <T extends ClientReq>(req: T): LoginClientInfo => {
   if (req.httpReq) {

package/lib/DBEventsManager.ts

@@ -2,7 +2,7 @@ import type { PrglNotifListener } from "./PostgresNotifListenManager";
 import { PostgresNotifListenManager } from "./PostgresNotifListenManager";
 import type { DB, PGP } from "./Prostgles";
 import { getKeys, CHANNELS } from "prostgles-types";
-import type { PRGLIOSocket } from "./DboBuilder/DboBuilder";
+import type { ClientSocketWithCachedData } from "./DboBuilder/DboBuilder";
 
 export class DBEventsManager {
   notifies: {
@@ -65,7 +65,7 @@ export class DBEventsManager {
 
   async addNotify(
     query: string,
-    socket?: PRGLIOSocket,
+    socket?: ClientSocketWithCachedData,
     func?: any
   ): Promise<{
     socketChannel: string;
@@ -142,7 +142,7 @@ export class DBEventsManager {
     };
   }
 
-  removeNotify(channel?: string, socket?: PRGLIOSocket, func?: any) {
+  removeNotify(channel?: string, socket?: ClientSocketWithCachedData, func?: any) {
     const notifChannel = channel && this.notifies[channel];
     if (notifChannel) {
       if (socket) {
@@ -163,7 +163,7 @@ export class DBEventsManager {
     }
   }
 
-  addNotice(socket: PRGLIOSocket) {
+  addNotice(socket: ClientSocketWithCachedData) {
     if (!socket.id) throw "Expecting a socket obj with id";
 
     if (!this.notice.sockets.find((s) => s.id === socket.id)) {
@@ -180,7 +180,7 @@ export class DBEventsManager {
     return { socketChannel, socketUnsubChannel };
   }
 
-  removeNotice(socket: PRGLIOSocket) {
+  removeNotice(socket: ClientSocketWithCachedData) {
     if (!socket.id) throw "Expecting a socket obj with id";
     this.notice.sockets = this.notice.sockets.filter((s) => s.id !== socket.id);
   }

package/lib/DboBuilder/DboBuilderTypes.ts

@@ -146,7 +146,7 @@ export type CachedSession = {
   __prglCache?: Map<string, CachedSessionData>;
 };
 
-export type PRGLIOSocket = Socket & {
+export type ClientSocketWithCachedData = Socket & {
   readonly id: string;
 
   readonly handshake: {

package/lib/DboBuilder/QueryBuilder/getJoinQuery.ts

@@ -94,7 +94,6 @@ export const getJoinQuery = (viewHandler: ViewHandler, { q1, q2 }: Args): GetJoi
       paths.length === 1 ? targetTableAliasRaw : undefined,
     ).rawAlias,
     _joinFields: firstJoinTableJoinFields,
-    jsonAggSort,
   });
 
   const joinType = q2.isLeftJoin ? "LEFT" : "INNER";
@@ -286,7 +285,6 @@ type GetSelectFieldsArgs = {
   q: NewQueryJoin;
   firstJoinTableAlias: string;
   _joinFields: string[];
-  jsonAggSort: string;
 };
 
 export type SelectItemNested = SelectItemValidated & {
@@ -297,7 +295,6 @@ const getNestedSelectFields = ({
   q,
   firstJoinTableAlias,
   _joinFields,
-  jsonAggSort,
 }: GetSelectFieldsArgs) => {
   const targetTableAlias = q.tableAlias || q.table;
 
@@ -325,7 +322,11 @@ const getNestedSelectFields = ({
 
   const getQuery = (tableAlias?: string) => {
     const partitionBy = `PARTITION BY ${requiredJoinFields.map((f) => asNameAlias(f, tableAlias)).join(", ")}`;
-    return `ROW_NUMBER() OVER(${partitionBy} ${jsonAggSort}) AS ${NESTED_ROWID_FIELD_NAME}`;
+    // Output aliases are not available inside a window expression in this SELECT.
+    const orderBy = prepareOrderByQuery(q.orderByItems, targetTableAlias.raw, (key) =>
+      selectedFields.find((field) => field.alias === key)?.getQuery(targetTableAlias.raw),
+    ).join(", ");
+    return `ROW_NUMBER() OVER(${partitionBy} ${orderBy}) AS ${NESTED_ROWID_FIELD_NAME}`;
   };
 
   if (q.limit) {

package/lib/DboBuilder/dboBuilderUtils.ts

@@ -148,7 +148,11 @@ export const getConstraints = async (
   );
 };
 
-export const prepareOrderByQuery = (items: SortItem[], tableAlias?: string): string[] => {
+export const prepareOrderByQuery = (
+  items: SortItem[],
+  tableAlias?: string,
+  getExpression?: (key: string) => string | undefined,
+): string[] => {
   if (!items.length) return [];
   return [
     "ORDER BY " +
@@ -159,7 +163,8 @@ export const prepareOrderByQuery = (items: SortItem[], tableAlias?: string): str
           if (d.type === "query" && d.nested) {
             return d.fieldQuery;
           }
-          return `${asNameAlias(d.key, tableAlias)} ${orderType} ${nullOrder}`;
+          const expression = getExpression?.(d.key) ?? asNameAlias(d.key, tableAlias);
+          return `${expression} ${orderType} ${nullOrder}`;
         })
         .join(", "),
   ];

package/lib/DboBuilder/runSql/QueryStreamer.ts

@@ -6,14 +6,14 @@ import type { BasicCallback } from "../../PubSubManager/PubSubManager";
 import type { VoidFunction } from "../../SchemaWatch/SchemaWatch";
 import type { DB } from "../../initProstgles";
 import type { DboBuilder } from "../DboBuilder";
-import type { PRGLIOSocket } from "../DboBuilderTypes";
+import type { ClientSocketWithCachedData } from "../DboBuilderTypes";
 import { getErrorAsObject, getSerializedClientErrorFromPGError } from "../dboBuilderUtils";
 import { getDetailedFieldInfo } from "./runSqlUtils";
 import { getConnectionDetails } from "./getAdminClient";
 const Cursor = require("pg-cursor") as typeof CursorType;
 
 type ClientStreamedRequest = {
-  socket: PRGLIOSocket;
+  socket: ClientSocketWithCachedData;
   query: string;
   options: SQLOptions | undefined;
   persistConnection?: boolean;

package/lib/Logging.ts

@@ -43,6 +43,11 @@ export namespace EventTypes {
       channelName: string;
       syncParams: SyncParams;
     } & (
+      | {
+          command: "replicationError";
+          source: "client" | "trigger" | "WAL";
+          error: unknown;
+        }
       | {
           command: "syncData";
           source: "client" | "trigger";

package/lib/Prostgles.ts

@@ -13,7 +13,7 @@ import type { ProstglesInitOptions } from "./ProstglesTypes";
 import { RestApi } from "./RestApi";
 import { TableConfigurator } from "./TableConfig/TableConfigurator";
 
-import type { PRGLIOSocket } from "./DboBuilder/DboBuilder";
+import type { ClientSocketWithCachedData } from "./DboBuilder/DboBuilder";
 import { DBHandlerServer, DboBuilder } from "./DboBuilder/DboBuilder";
 export { DBHandlerServer };
 export type PGP = pgPromise.IMain<{}, pg.IClient>;
@@ -401,7 +401,7 @@ export class Prostgles {
 
   init = initProstgles.bind(this);
 
-  connectedSockets: PRGLIOSocket[] = [];
+  connectedSockets: ClientSocketWithCachedData[] = [];
   setupSocketIO() {
     this.checkDb();
 

package/lib/ProstglesTypes.ts

@@ -16,7 +16,7 @@ import type { TableConfig } from "./TableConfig/TableConfigTypes";
 import type { TableHooks } from "./TableHooks/TableHooks";
 import type { ResolvedAuditConfig, SchemaConfigAudit } from "./Audit/AuditTypes";
 
-import type { PRGLIOSocket } from "./DboBuilder/DboBuilder";
+import type { ClientSocketWithCachedData } from "./DboBuilder/DboBuilder";
 
 import type e from "express";
 import type pgPromise from "pg-promise";
@@ -248,14 +248,14 @@ export type ProstglesInitOptions<
    * Use for connection verification. Will disconnect socket on any errors
    */
   onSocketConnect?: (
-    args: AuthRequestParams<S, SUser> & { socket: PRGLIOSocket },
+    args: AuthRequestParams<S, SUser> & { socket: ClientSocketWithCachedData },
   ) => void | Promise<void>;
 
   /**
    * Called when a socket disconnects
    */
   onSocketDisconnect?: (
-    args: AuthRequestParams<S, SUser> & { socket: PRGLIOSocket },
+    args: AuthRequestParams<S, SUser> & { socket: ClientSocketWithCachedData },
   ) => void | Promise<void>;
 
   /**

package/lib/PubSubManager/PubSubManager.ts

@@ -7,7 +7,7 @@ import type {
   DBHandlerServer,
   DboBuilder,
   LocalParams,
-  PRGLIOSocket,
+  ClientSocketWithCachedData,
 } from "../DboBuilder/DboBuilder";
 import type { PostgresNotifListenManager } from "../PostgresNotifListenManager";
 import type { DB } from "../Prostgles";
@@ -60,12 +60,13 @@ export type SyncParams = {
   throttle?: number;
   lr?: AnyObject;
   is_syncing: boolean;
+  syncQueue?: Promise<void>;
   handlers: ReturnType<(typeof ReplicationProtocol)["getServerHandlers"]>;
   localParams: LocalParams;
 };
 
 export type AddSyncParams = SyncTableInfo & {
-  socket: PRGLIOSocket;
+  socket: ClientSocketWithCachedData;
   table_info: TableSchema;
   table_rules: ParsedTableRule;
   allow_delete?: boolean;
@@ -120,7 +121,7 @@ export type SubscriptionParams = {
   subscribeOptions: SubscribeOptions;
 
   onData?: OnData;
-  socket: PRGLIOSocket | undefined;
+  socket: ClientSocketWithCachedData | undefined;
   localParams?: LocalParams;
 
   lastPushed: number;
@@ -196,7 +197,7 @@ export class PubSubManager {
    * Updated through refreshTriggers()
    */
   _triggers: PubSubManagerTriggers = new Map();
-  sockets: Map<string, PRGLIOSocket> = new Map();
+  sockets: Map<string, ClientSocketWithCachedData> = new Map();
 
   subs: Subscription[] = [];
   syncs: SyncParams[] = [];
@@ -266,7 +267,7 @@ export class PubSubManager {
 
   removeSubscription = (
     channelName: string,
-    subInfo: { type: "local"; onData: OnData } | { type: "ws"; socket: PRGLIOSocket },
+    subInfo: { type: "local"; onData: OnData } | { type: "ws"; socket: ClientSocketWithCachedData },
   ) => {
     const matchingSubIdx = this.subs.findIndex(
       (s) =>
@@ -364,7 +365,7 @@ export class PubSubManager {
 
   pushSubData = pushSubData.bind(this);
 
-  upsertSocket(socket: PRGLIOSocket | undefined) {
+  upsertSocket(socket: ClientSocketWithCachedData | undefined) {
     if (socket && !this.sockets.get(socket.id)) {
       this.sockets.set(socket.id, socket);
       socket.on("disconnect", () => {

package/lib/PubSubManager/SyncReplication/getSyncUtilFunctions.ts

@@ -6,7 +6,7 @@ import {
   type AnyObject,
   type SyncBatchParams,
 } from "prostgles-types";
-import type { LocalParams, PRGLIOSocket } from "../../DboBuilder/DboBuilder";
+import type { LocalParams, ClientSocketWithCachedData } from "../../DboBuilder/DboBuilder";
 import type { TableHandler } from "../../DboBuilder/TableHandler/TableHandler";
 import { getSyncOrderByAndFields } from "./getSyncOrderByAndFields";
 import type { PubSubManager, SyncParams } from "../PubSubManager";
@@ -21,7 +21,7 @@ import { log } from "../PubSubManagerUtils";
 import type { EventTypes } from "../../Logging";
 
 type Args = {
-  socket: PRGLIOSocket;
+  socket: ClientSocketWithCachedData;
   tableHandler: TableHandler;
   sync: SyncParams;
   pubSubManager: PubSubManager;
@@ -389,12 +389,7 @@ export const getSyncUtilFunctions = ({
         return;
       }
       if (sync.lr?.[synced_field] && +sync.lr[synced_field] > +lastRow[synced_field]) {
-        console.error(
-          {
-            syncIssue: "sync.lr[synced_field] is greater than lastRow[synced_field]",
-          },
-          sync.table_name,
-        );
+        return;
       }
       sync.lr = lastRow;
     },

package/lib/PubSubManager/SyncReplication/logSyncError.ts

@@ -0,0 +1,22 @@
+import type { EventTypes } from "../../Logging";
+import type { PubSubManager, SyncParams } from "../PubSubManager";
+
+export const logSyncError = (
+  pubSubManager: PubSubManager,
+  sync: SyncParams,
+  source: Extract<EventTypes.Sync, { command: "replicationError" }>["source"],
+  error: unknown,
+) =>
+  pubSubManager._log({
+    type: "sync",
+    command: "replicationError",
+    source,
+    error,
+    tableName: sync.table_name,
+    channelName: sync.channel_name,
+    socketId: sync.socket_id,
+    sid: sync.sid,
+    connectedSocketIds: pubSubManager.connectedSocketIds,
+    syncParams: sync,
+    duration: -1,
+  });

package/lib/PubSubManager/SyncReplication/syncData.ts

@@ -4,6 +4,7 @@ import { WAL } from "prostgles-types/dist/WAL";
 import type { PubSubManager, SyncParams } from "../PubSubManager";
 import { getSyncUtilFunctions } from "./getSyncUtilFunctions";
 import type { EventTypes } from "../../Logging";
+import { logSyncError } from "./logSyncError";
 
 export type ClientSyncInfo = Partial<{
   c_fr: AnyObject;
@@ -45,7 +46,26 @@ export type ClientExpressData = ClientSyncInfo & {
 /**
  * Server or client requested data sync
  */
-export async function syncData(
+export function syncData(
+  this: PubSubManager,
+  sync: SyncParams,
+  clientData: ClientExpressData | undefined,
+  source: "trigger" | "client",
+) {
+  const run = () =>
+    runSyncData.call(this, sync, clientData, source).catch(async (error) => {
+      if (this.sockets.get(sync.socket_id)?.connected) {
+        await logSyncError(this, sync, source, error);
+      }
+      throw error;
+    });
+  const queued = (sync.syncQueue ?? Promise.resolve()).then(run);
+  // Report failures to the caller while allowing the next request to run.
+  sync.syncQueue = queued.catch(() => undefined);
+  return queued;
+}
+
+async function runSyncData(
   this: PubSubManager,
   sync: SyncParams,
   clientData: ClientExpressData | undefined,
@@ -111,8 +131,12 @@ export async function syncData(
       sync.is_syncing = true;
     },
     onSend: async (data) => {
-      const res = await upsertData(data, "WAL");
-      return res;
+      try {
+        return await upsertData(data, "WAL");
+      } catch (error) {
+        await logSyncError(this, sync, "WAL", error);
+        throw error;
+      }
     },
     onSendEnd: (batch, _, error) => {
       if (error === undefined) {
@@ -164,36 +188,36 @@ export async function syncData(
   }
 
   sync.is_syncing = true;
+  try {
+    let from_synced = null;
+
+    /** Client runs must reconcile the client's snapshot after reattachment. */
+    if (sync.lr && !clientData) {
+      const { s_lr } = await getServerRowInfo();
+
+      /* Make sure trigger is not firing on freshly synced data */
+      if (!rowsFullyMatch(sync.lr, s_lr)) {
+        from_synced = Number(sync.lr[synced_field]);
+        await logSyncData("sync.lr");
+      } else {
+        await logSyncData("rowsFullyMatch");
+      }
+    } else {
+      await logSyncData("getLastSynced(clientData).start");
+      from_synced = await getLastSynced(clientData);
+      await logSyncData("getLastSynced(clientData).end");
+    }
 
-  // from synced does not make sense. It should be sync.lr only!!!
-  let from_synced = null;
-
-  /** Sync was already synced */
-  if (sync.lr) {
-    const { s_lr } = await getServerRowInfo();
-
-    /* Make sure trigger is not firing on freshly synced data */
-    if (!rowsFullyMatch(sync.lr, s_lr)) {
-      from_synced = Number(sync.lr[synced_field]);
-      await logSyncData("sync.lr");
+    if (from_synced !== null) {
+      await logSyncData("syncBatch.start");
+      await syncBatch(from_synced);
+      await logSyncData("syncBatch.end");
     } else {
-      await logSyncData("rowsFullyMatch");
+      await logSyncData("nothingToSync");
     }
-  } else {
-    await logSyncData("getLastSynced(clientData).start");
-    from_synced = await getLastSynced(clientData);
-    await logSyncData("getLastSynced(clientData).end");
-  }
 
-  if (from_synced !== null) {
-    await logSyncData("syncBatch.start");
-    await syncBatch(from_synced);
-    await logSyncData("syncBatch.end");
-  } else {
-    await logSyncData("nothingToSync");
+    await pushData({ state: "synced" });
+  } finally {
+    sync.is_syncing = false;
   }
-
-  await pushData({ state: "synced" });
-
-  sync.is_syncing = false;
 }

package/lib/PubSubManager/addSync.ts

@@ -40,7 +40,6 @@ export async function addSync(
       /* Only a sync per socket per table/condition/select allowed */
       const existing = find(this.syncs, { socket_id: socket.id, channel_name: channelName });
       if (existing) {
-        console.warn("addSync: Client tried to create a duplicate sync", existing.channel_name);
         return existing;
       }
 
@@ -112,18 +111,18 @@ export async function addSync(
         cb(null, { res: "ok" });
       });
 
+      this.syncs.push(newSync);
       return newSync;
     };
 
-    const newSync = upsertSync();
-
     await this.addTrigger(
       { table_name, condition: conditionParsed, tracked_columns: undefined },
       undefined,
       socket,
     );
 
-    this.syncs.push(newSync);
+    // Check and register without yielding so concurrent attachments share one sync.
+    const newSync = upsertSync();
     return { channelName, newSync };
   });
 

package/lib/PubSubManager/addTrigger.ts

@@ -1,6 +1,6 @@
 import * as crypto from "crypto";
 import { asName, pickKeys, tryCatchV2 } from "prostgles-types";
-import type { PRGLIOSocket } from "../DboBuilder/DboBuilderTypes";
+import type { ClientSocketWithCachedData } from "../DboBuilder/DboBuilderTypes";
 import type { TableHandler } from "../DboBuilder/TableHandler/TableHandler";
 import type { ViewSubscriptionOptions } from "./PubSubManager";
 import { type PubSubManager } from "./PubSubManager";
@@ -16,7 +16,7 @@ export async function addTrigger(
   this: PubSubManager,
   params: AddTriggerParams,
   viewOptions: ViewSubscriptionOptions | undefined,
-  socket: PRGLIOSocket | undefined,
+  socket: ClientSocketWithCachedData | undefined,
 ) {
   const addedTrigger = await tryCatchV2(async () => {
     const { table_name } = { ...params };

package/lib/WebsocketAPI/onSocketConnected.ts

@@ -2,12 +2,12 @@ import type { AnyObject, SocketFunctionCall } from "prostgles-types";
 import { CHANNELS, getSerialisableError, isObject } from "prostgles-types";
 import { getClientRequestIPsInfo } from "../Auth/AuthHandler";
 import type { AuthResultWithSID, SessionUser } from "../Auth/AuthTypes";
-import type { PRGLIOSocket } from "../DboBuilder/DboBuilderTypes";
+import type { ClientSocketWithCachedData } from "../DboBuilder/DboBuilderTypes";
 import type { DBOFullyTyped } from "../DBSchemaBuilder/DBSchemaBuilder";
 import type { Prostgles, TABLE_METHODS } from "../Prostgles";
 import { runClientMethod, runClientRequest } from "../runClientRequest";
 
-export async function onSocketConnected(this: Prostgles, socket: PRGLIOSocket) {
+export async function onSocketConnected(this: Prostgles, socket: ClientSocketWithCachedData) {
   const isDestroyed = () => this.destroyed;
   if (isDestroyed()) {
     console.log("Socket connected to destroyed instance");

package/lib/WebsocketAPI/pushSocketSchema.ts

@@ -1,9 +1,9 @@
 import { CHANNELS, getSerialisableError, type SQLRequest } from "prostgles-types";
-import type { PRGLIOSocket } from "../DboBuilder/DboBuilderTypes";
+import type { ClientSocketWithCachedData } from "../DboBuilder/DboBuilderTypes";
 import type { Prostgles } from "../Prostgles";
 import { runClientSqlRequest } from "../runClientRequest";
 import { makeSocketError } from "./onSocketConnected";
-export async function pushSocketSchema(this: Prostgles, socket: PRGLIOSocket) {
+export async function pushSocketSchema(this: Prostgles, socket: ClientSocketWithCachedData) {
   const isDestroyed = () => this.destroyed;
   if (isDestroyed()) return;
   try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prostgles-server",
-  "version": "4.2.639",
+  "version": "4.2.641",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",