prostgles-server 4.2.440 → 4.2.442

package/lib/PubSubManager/PubSubManager.ts

@@ -20,11 +20,11 @@ import { initialiseEventTriggers } from "./initialiseEventTriggers";
 import { refreshTriggers } from "./refreshTriggers";
 
 import type { AnyObject, FieldFilter, SelectParams, WAL } from "prostgles-types";
-import { CHANNELS, type SubscribeOptions } from "prostgles-types";
+import { CHANNELS, getSerialisableError, type SubscribeOptions } from "prostgles-types";
 
 import { find, pickKeys } from "prostgles-types";
-import type { LocalFuncs } from "../DboBuilder/ViewHandler/subscribe";
-import { getOnDataFunc, matchesLocalFuncs } from "../DboBuilder/ViewHandler/subscribe";
+import type { OnData } from "../DboBuilder/ViewHandler/subscribe";
+import { matchesLocalFuncs } from "../DboBuilder/ViewHandler/subscribe";
 import type { EventTypes } from "../Logging";
 import type { ParsedTableRule } from "../PublishParser/PublishParser";
 import { syncData } from "../SyncReplication";
@@ -109,7 +109,7 @@ export type SubscriptionParams = {
   subscribeOptions: SubscribeOptions;
   tracked_columns: string[] | undefined;
 
-  localFuncs?: LocalFuncs;
+  onData?: OnData;
   socket: PRGLIOSocket | undefined;
 
   lastPushed: number;
@@ -125,7 +125,7 @@ export type Subscription = Pick<
   | "lastPushed"
   | "channel_name"
   | "is_ready"
-  | "localFuncs"
+  | "onData"
   | "socket"
   | "socket_id"
   | "table_info"
@@ -136,7 +136,7 @@ export type Subscription = Pick<
   triggers: AddTriggerParams[];
 };
 
-export type PubSubManagerTriggers = Record<string, { condition: string; hash: string }[]>;
+export type PubSubManagerTriggers = Map<string, { condition: string; hash: string }[]>;
 
 /**
  * Used to facilitate table subscribe and sync
@@ -165,7 +165,7 @@ export class PubSubManager {
    * Triggers used for sync/sub that reflect prostgles.app_triggers.
    * Updated through refreshTriggers()
    */
-  _triggers: PubSubManagerTriggers = {};
+  _triggers: PubSubManagerTriggers = new Map();
   sockets: Record<string, PRGLIOSocket> = {};
 
   subs: Subscription[] = [];
@@ -218,13 +218,13 @@ export class PubSubManager {
 
   getClientSubs({
     channel_name,
-    localFuncs,
+    onData,
     socket_id,
-  }: Pick<Subscription, "localFuncs" | "socket_id" | "channel_name">): Subscription[] {
+  }: Pick<Subscription, "onData" | "socket_id" | "channel_name">): Subscription[] {
     return this.subs.filter((s) => {
       return (
         s.channel_name === channel_name &&
-        (matchesLocalFuncs(localFuncs, s.localFuncs) || (socket_id && s.socket_id === socket_id))
+        (matchesLocalFuncs(onData, s.onData) || (socket_id && s.socket_id === socket_id))
       );
     });
   }
@@ -236,30 +236,31 @@ export class PubSubManager {
 
   removeSubscription = (
     channelName: string,
-    subInfo: { type: "local"; localFuncs: LocalFuncs } | { type: "ws"; socket: PRGLIOSocket },
+    subInfo: { type: "local"; onData: OnData } | { type: "ws"; socket: PRGLIOSocket },
   ) => {
     const matchingSubIdx = this.subs.findIndex(
       (s) =>
         s.channel_name === channelName &&
         (subInfo.type === "local" ?
-          getOnDataFunc(subInfo.localFuncs) === getOnDataFunc(s.localFuncs)
+          subInfo.onData === s.onData
         : subInfo.socket.id === s.socket?.id),
     );
-    if (matchingSubIdx > -1) {
-      const tableName = this.subs[matchingSubIdx]!.table_info.name;
-      const oldActiveTriggers = this.getActiveTriggers(tableName);
+    const matchingSub = this.subs[matchingSubIdx];
+    if (matchingSub) {
+      /** Ensure we check and refresh related table/view triggers as well */
+      const oldActiveTriggers = this.getAllActiveTriggers();
       this.subs.splice(matchingSubIdx, 1);
-      const newActiveTriggers = this.getActiveTriggers(tableName);
+      const newActiveTriggers = this.getAllActiveTriggers();
+      const tableNames = new Set(
+        [...oldActiveTriggers, ...newActiveTriggers].map((t) => t.tableName),
+      );
       if (newActiveTriggers.length < oldActiveTriggers.length) {
-        this.deleteOrphanedTriggers(tableName);
+        this.deleteOrphanedTriggers(tableNames);
       }
     } else {
-      console.error(
-        "Could not unsubscribe localFunc. Subscription might not have initialised yet",
-        {
-          channelName,
-        },
-      );
+      console.error("Could not unsubscribe. Subscription might not have initialised yet", {
+        channelName,
+      });
     }
   };
 
@@ -272,11 +273,11 @@ export class PubSubManager {
   notifListener = notifListener.bind(this);
 
   getTriggerInfo = (tableName: string) => {
-    const tableTriggerConditions = this._triggers[tableName]?.map((cond, idx) => ({
+    const tableTriggerConditions = this._triggers.get(tableName)?.map((triggerInfo, idx) => ({
       idx,
-      ...cond,
-      subs: this.getTriggerSubs(tableName, cond.condition),
-      syncs: this.getSyncs(tableName, cond.condition),
+      ...triggerInfo,
+      subs: this.getTriggerSubs(tableName, triggerInfo.condition),
+      syncs: this.getSyncs(tableName, triggerInfo.condition),
     }));
     return tableTriggerConditions;
   };
@@ -287,13 +288,20 @@ export class PubSubManager {
     return activeTriggers;
   };
 
-  getSubData = async (
-    sub: Subscription,
-  ): Promise<{ data: any[]; err?: undefined } | { data?: undefined; err: unknown }> => {
-    const { table_info, filter, selectParams: params, table_rules, socket, localFuncs } = sub; //, subOne = false
+  getAllActiveTriggers = () => {
+    return Array.from(this._triggers.keys()).flatMap((tableName) => {
+      return this.getActiveTriggers(tableName).map((triggerInfo) => ({
+        ...triggerInfo,
+        tableName,
+      }));
+    });
+  };
+
+  getSubData = async (sub: Subscription) => {
+    const { table_info, filter, selectParams: params, table_rules, socket, onData } = sub; //, subOne = false
     const { name: table_name } = table_info;
     const tableHandler = this.dbo[table_name];
-    if (!localFuncs && !socket) {
+    if (!onData && !socket) {
       throw new Error("Subscription must have either localFuncs or socket");
     }
     if (!tableHandler?.find) {
@@ -301,7 +309,7 @@ export class PubSubManager {
     }
 
     try {
-      const data = await tableHandler.find(
+      const data = (await tableHandler.find(
         filter,
         params,
         undefined,
@@ -309,10 +317,10 @@ export class PubSubManager {
         socket && {
           clientReq: { socket },
         },
-      );
+      )) as AnyObject[];
       return { data };
     } catch (err) {
-      return { err };
+      return { err: getSerialisableError(err) || "Unknown error fetching subscription data" };
     }
   };
 
@@ -401,23 +409,23 @@ export class PubSubManager {
   /** Throttle trigger deletes */
   deletingOrphanedTriggers:
     | {
-        tableNames: string[];
+        tableNames: Set<string>;
         timeout: NodeJS.Timeout;
       }
     | undefined;
-  deleteOrphanedTriggers = (latestTableName: string) => {
+  deleteOrphanedTriggers = (latestTableNames: Set<string>) => {
     this.deletingOrphanedTriggers ??= {
-      tableNames: [latestTableName],
+      tableNames: latestTableNames,
       timeout: setTimeout(() => {
         const tableNames = this.deletingOrphanedTriggers!.tableNames;
         this.deletingOrphanedTriggers = undefined;
-        void deleteOrphanedTriggers.bind(this)(tableNames);
+        void deleteOrphanedTriggers.bind(this)(Array.from(tableNames));
       }, 1000),
     };
 
-    if (!this.deletingOrphanedTriggers.tableNames.includes(latestTableName)) {
-      this.deletingOrphanedTriggers.tableNames.push(latestTableName);
-    }
+    latestTableNames.forEach((latestTableName) => {
+      this.deletingOrphanedTriggers?.tableNames.add(latestTableName);
+    });
   };
 
   addingTrigger: any;

package/lib/PubSubManager/addSub.ts

@@ -1,7 +1,12 @@
 import type { SubscriptionChannels } from "prostgles-types";
 import type { VoidFunction } from "../SchemaWatch/SchemaWatch";
 import { tout } from "./init/initPubSubManager";
-import type { BasicCallback, PubSubManager, Subscription, SubscriptionParams } from "./PubSubManager";
+import type {
+  BasicCallback,
+  PubSubManager,
+  Subscription,
+  SubscriptionParams,
+} from "./PubSubManager";
 import { parseCondition } from "./PubSubManagerUtils";
 import type { AddTriggerParams } from "./addTrigger";
 
@@ -17,11 +22,11 @@ type AddSubResult = SubscriptionChannels & {
 /* The distinct list of {table_name, condition} must have a corresponding trigger in the database */
 export async function addSub(
   this: PubSubManager,
-  subscriptionParams: Omit<AddSubscriptionParams, "channel_name" | "parentSubParams">
+  subscriptionParams: Omit<AddSubscriptionParams, "channel_name" | "parentSubParams">,
 ): Promise<AddSubResult> {
   const {
     socket,
-    localFuncs,
+    onData,
     table_rules,
     filter = {},
     selectParams = {},
@@ -33,10 +38,10 @@ export async function addSub(
   } = subscriptionParams;
   const table_name = table_info.name;
 
-  if (!socket && !localFuncs) {
+  if (!socket && !onData) {
     throw "socket AND func missing";
   }
-  if (socket && localFuncs) {
+  if (socket && onData) {
     throw "addSub: cannot have socket AND func";
   }
 
@@ -50,7 +55,7 @@ export async function addSub(
   const newSub: Subscription = {
     channel_name,
     filter,
-    localFuncs,
+    onData,
     selectParams: selectParams,
     lastPushed: 0,
     socket,
@@ -74,7 +79,7 @@ export async function addSub(
   const [matchingSub] = this.getClientSubs(newSub);
   if (matchingSub) {
     console.error(
-      `Trying to add a duplicate ${localFuncs ? "local" : "socket"} sub for: ${channel_name}`
+      `Trying to add a duplicate ${onData ? "local" : "socket"} sub for: ${channel_name}`,
     );
     return result;
   }
@@ -104,7 +109,7 @@ export async function addSub(
     void this.pushSubData(newSub);
   };
 
-  if (localFuncs) {
+  if (onData) {
     /**
      * Must ensure sub will start sending data after all triggers are set up.
      * Socket clients are not affected as they need to confirm they are ready to receive data

package/lib/PubSubManager/init/getDataWatchFunctionQuery.ts

@@ -8,7 +8,7 @@ import { asValue, DELIMITER, NOTIF_CHANNEL, NOTIF_TYPE } from "../PubSubManagerU
  */
 export const udtNamesWithoutEqualityComparison = ["json", "xml"];
 export const getDataWatchFunctionQuery = (debugMode: boolean | undefined) => {
-  return `
+  const dataWatchFunctionQuery = `
   
         CREATE OR REPLACE FUNCTION ${DB_OBJ_NAMES.data_watch_func}() RETURNS TRIGGER 
         AS $$
@@ -161,6 +161,25 @@ export const getDataWatchFunctionQuery = (debugMode: boolean | undefined) => {
         COMMENT ON FUNCTION ${DB_OBJ_NAMES.data_watch_func} IS 'Prostgles internal function used to notify when data in the table changed';
 
   `;
+
+  /** Ensure every execute is followed by EXCEPTION catch to ensure we remove stale schema/faulty triggers */
+  const queryLines = dataWatchFunctionQuery
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l);
+  queryLines.forEach((line, lineIndex) => {
+    const nextLine = queryLines[lineIndex + 1] ?? "";
+    if (
+      line.toUpperCase().startsWith("EXECUTE") &&
+      !nextLine.toUpperCase().startsWith("EXCEPTION")
+    ) {
+      throw new Error(
+        `Every EXECUTE statement in the data watch function must be followed by an EXCEPTION block to catch errors and avoid stale triggers. Problematic line: ${line}`,
+      );
+    }
+  });
+
+  return dataWatchFunctionQuery;
 };
 
 /**
@@ -208,6 +227,14 @@ IF TG_OP = 'UPDATE' THEN
 
       BEGIN
         EXECUTE query INTO changed_columns;
+        EXCEPTION WHEN OTHERS THEN
+          
+          has_errors := TRUE;
+
+          GET STACKED DIAGNOSTICS 
+            err_text = MESSAGE_TEXT,
+            err_detail = PG_EXCEPTION_DETAIL,
+            err_hint = PG_EXCEPTION_HINT;
       END;
 
       /* It is possible to get no changes */

package/lib/PubSubManager/notifListener.ts

@@ -101,21 +101,21 @@ export async function notifListener(this: PubSubManager, data: { payload: string
   } else if (conditionIds?.every((id) => Number.isInteger(id))) {
     state = "ok";
     const firedTableConditions = tableTriggerConditions.filter(({ idx }) =>
-      conditionIds.includes(idx)
+      conditionIds.includes(idx),
     );
     const orphanedTableConditions = conditionIds.filter((condId) => {
       const tc = tableTriggerConditions.at(condId);
       return !tc || (tc.subs.length === 0 && tc.syncs.length === 0);
     });
     if (orphanedTableConditions.length) {
-      void this.deleteOrphanedTriggers.bind(this)(table_name);
+      void this.deleteOrphanedTriggers(new Set(table_name));
     }
 
     firedTableConditions.map(({ subs, syncs }) => {
       log(
         "notifListener",
         subs.map((s) => s.channel_name),
-        syncs.map((s) => s.channel_name)
+        syncs.map((s) => s.channel_name),
       );
 
       syncs.map((s) => {
@@ -128,8 +128,8 @@ export async function notifListener(this: PubSubManager, data: { payload: string
           (trg) =>
             this.dbo[trg.table_name] &&
             sub.is_ready &&
-            ((sub.socket_id && this.sockets[sub.socket_id]) || sub.localFuncs)
-        )
+            ((sub.socket_id && this.sockets[sub.socket_id]) || sub.onData),
+        ),
       );
 
       activeAndReadySubs.forEach((sub) => {
@@ -143,7 +143,7 @@ export async function notifListener(this: PubSubManager, data: { payload: string
           tracked_columns
         ) {
           const subFieldsHaveChanged = changedColumns.some((changedColumn) =>
-            tracked_columns.includes(changedColumn)
+            tracked_columns.includes(changedColumn),
           );
           if (!subFieldsHaveChanged) return;
         }
@@ -181,11 +181,11 @@ export async function notifListener(this: PubSubManager, data: { payload: string
     state,
     op_name,
     condition_ids_str,
-    tableTriggers: this._triggers[table_name],
+    tableTriggers: this._triggers.get(table_name),
     tableSyncs: JSON.stringify(
       this.syncs
         .filter((s) => s.table_name === table_name)
-        .map((s) => pickKeys(s, ["condition", "socket_id"]))
+        .map((s) => pickKeys(s, ["condition", "socket_id"])),
     ),
     connectedSocketIds: this.connectedSocketIds,
   });

package/lib/PubSubManager/pushSubData.ts

@@ -1,12 +1,10 @@
-import { getSerialisableError } from "prostgles-types";
-import { parseLocalFuncs } from "../DboBuilder/ViewHandler/subscribe";
 import type { EventTypes } from "../Logging";
 import type { Subscription } from "./PubSubManager";
 import { type PubSubManager } from "./PubSubManager";
 import { log } from "./PubSubManagerUtils";
 
 export async function pushSubData(this: PubSubManager, sub: Subscription, err?: any) {
-  const { socket_id, channel_name } = sub;
+  const { socket_id, channel_name, onData } = sub;
 
   const onLog = (
     state: Extract<EventTypes.SyncOrSub, { type: "syncOrSub"; command: "pushSubData" }>["state"],
@@ -28,7 +26,6 @@ export async function pushSubData(this: PubSubManager, sub: Subscription, err?:
     onLog("sub_not_found");
     return;
   }
-  const localFuncs = parseLocalFuncs(sub.localFuncs);
 
   if (err) {
     onLog("error");
@@ -39,40 +36,28 @@ export async function pushSubData(this: PubSubManager, sub: Subscription, err?:
   }
 
   sub.lastPushed = Date.now();
-  return new Promise(async (resolve, reject) => {
-    /* TODO: Retire subOne -> it's redundant */
 
-    const { data, err } = await this.getSubData(sub);
+  const { data, err: subDataError } = await this.getSubData(sub);
 
-    if (data) {
-      if (socket_id && this.sockets[socket_id]) {
-        log("Pushed " + data.length + " records to sub");
-        onLog("Emiting to socket");
-        this.sockets[socket_id].emit(channel_name, { data }, () => {
-          resolve(data);
-        });
+  if (subDataError !== undefined) {
+    onLog("fetch data error");
+  }
+  if (socket_id && this.sockets[socket_id]) {
+    log(`Pushing ${data?.length ?? 0} rows to socket`);
+    onLog("Emiting to socket");
+    this.sockets[socket_id].emit(
+      channel_name,
+      subDataError !== undefined ? { err: subDataError } : { data },
+      () => {
         /* TO DO: confirm receiving data or server will unsubscribe
           { data }, (cb)=> { console.log(cb) });
         */
-      } else if (localFuncs) {
-        onLog("pushed to local client");
-        localFuncs.onData(data);
-        resolve(data);
-      } else {
-        onLog("no client to push data to");
-      }
-    } else {
-      onLog("fetch data error");
-      const errObj = getSerialisableError(err) || "Unknown error fetching subscription data";
-      if (socket_id && this.sockets[socket_id]) {
-        this.sockets[socket_id].emit(channel_name, { err: errObj });
-      } else if (localFuncs) {
-        if (!localFuncs.onError) {
-          console.error("Uncaught subscription error", err);
-        }
-        localFuncs.onError?.(errObj);
-      }
-      reject(errObj);
-    }
-  });
+      },
+    );
+  } else if (onData) {
+    onLog("pushed to local client");
+    onData(data ?? [], subDataError);
+  } else {
+    onLog("no client to push data to");
+  }
 }

package/lib/PubSubManager/refreshTriggers.ts

@@ -2,27 +2,28 @@ import type { PubSubManager } from "./PubSubManager";
 
 export async function refreshTriggers(this: PubSubManager) {
   const start = Date.now();
-  const triggers: {
+  const triggers = await this.db.any<{
     table_name: string;
     condition: string;
     condition_hash: string;
-  }[] = await this.db.any(
+  }>(
     `
         SELECT *
         FROM prostgles.v_triggers
         WHERE app_id = $1
         ORDER BY table_name, condition
       `,
-    [this.dboBuilder.prostgles.appId]
+    [this.dboBuilder.prostgles.appId],
   );
 
-  const oldTriggers = { ...this._triggers };
+  const oldTriggers = new Map(this._triggers);
 
-  this._triggers = {};
+  this._triggers = new Map();
   triggers.map((t) => {
-    this._triggers[t.table_name] ??= [];
-    if (!this._triggers[t.table_name]?.map((t) => t.condition).includes(t.condition)) {
-      this._triggers[t.table_name]?.push({ condition: t.condition, hash: t.condition_hash });
+    this._triggers.set(t.table_name, this._triggers.get(t.table_name) ?? []);
+    const tableTriggers = this._triggers.get(t.table_name)!;
+    if (!tableTriggers.map((t) => t.condition).includes(t.condition)) {
+      tableTriggers.push({ condition: t.condition, hash: t.condition_hash });
     }
   });
 

package/lib/PublishParser/PublishParser.ts

@@ -104,37 +104,21 @@ export class PublishParser {
     command,
     clientReq,
   }: DboTableCommand): Promise<ParsedTableRule> {
+    const rules = await this.getParsedTableRule({ tableName, clientReq }, undefined);
     const clientInfo =
       clientReq && (await this.prostgles.authHandler.getSidAndUserFromRequest(clientReq));
     if (clientInfo === "new-session-redirect") {
       throw "new-session-redirect";
     }
-    const rules = await this.getValidatedRequestRule(
-      { tableName, command, clientReq },
-      clientInfo,
-      undefined,
-    );
+    this.validateRequestRule({ tableName, command, clientReq }, rules, undefined);
     return rules;
   }
 
-  async getValidatedRequestRule(
-    { tableName, command, clientReq }: DboTableCommand,
+  async getParsedTableRule(
+    { tableName, clientReq }: Pick<DboTableCommand, "tableName" | "clientReq">,
     clientInfo: AuthResultWithSID | undefined,
-    scope: PermissionScope | undefined,
   ): Promise<ParsedTableRule> {
-    if (!command || !tableName) throw "command OR tableName are missing";
-
-    const rule = RULE_TO_METHODS.find((rtms) => rtms.methods.some((v) => v === command));
-    if (!rule) {
-      throw "Invalid command: " + command;
-    }
-
-    if (scope) {
-      const tableScope = scope.tables;
-      if (!tableScope?.[tableName] || !tableScope[tableName][rule.sqlRule]) {
-        throw `Invalid or disallowed command: ${tableName}.${command}. The PermissionsScope does not allow this command.`;
-      }
-    }
+    if (!tableName) throw "tableName missing";
 
     /* Must be local request -> allow everything */
     if (!clientReq) {
@@ -150,17 +134,51 @@ export class PublishParser {
     /* Must be from socket. Must have a publish */
     if (!this.publish) throw "publish is missing";
 
+    const tableErrors = clientReq.socket?.prostgles?.tableSchemaErrors[tableName];
     /* Get any publish errors for socket */
-    const errorInfo = clientReq.socket?.prostgles?.tableSchemaErrors[tableName]?.[command];
-
-    if (errorInfo) throw errorInfo.error;
+    Object.values(tableErrors ?? {}).forEach((errorInfo) => {
+      throw errorInfo.error;
+    });
 
     const tableRule = await this.getTableRules({ tableName, clientReq }, clientInfo);
-    if (!tableRule)
+
+    if (!tableRule) {
       throw {
         stack: ["getValidatedRequestRule()"],
         message: "Invalid or disallowed table: " + tableName,
       };
+    }
+    return tableRule;
+  }
+
+  validateRequestRule(
+    { tableName, command }: DboTableCommand,
+    tableRule: ParsedTableRule,
+    scope: PermissionScope | undefined,
+  ) {
+    if (!command || !tableName) throw "command OR tableName are missing";
+
+    const rule = RULE_TO_METHODS.find((rtms) => rtms.methods.some((v) => v === command));
+    if (!rule) {
+      throw "Invalid command: " + command;
+    }
+
+    if (scope) {
+      if (scope.sql === "commited") {
+        // Allow all commands
+      } else if (scope.sql === "rolledback") {
+        if (rule.sqlRule === "select") {
+          // Allow select
+        } else {
+          throw `Invalid or disallowed command: ${tableName}.${command}. The PermissionsScope only allows ${scope.sql}`;
+        }
+      } else {
+        const tableScope = scope.tables;
+        if (!tableScope?.[tableName] || !tableScope[tableName][rule.sqlRule]) {
+          throw `Invalid or disallowed command: ${tableName}.${command}. The PermissionsScope does not allow this command.`;
+        }
+      }
+    }
 
     if (command === "upsert") {
       if (!tableRule.update || !tableRule.insert) {
@@ -177,8 +195,6 @@ export class PublishParser {
         message: `Invalid or disallowed command: ${tableName}.${command}`,
       };
     }
-
-    return tableRule;
   }
 
   async getTableRules(