prostgles-server 4.2.159 → 4.2.161

package/lib/PubSubManager/PubSubManager.ts

@@ -0,0 +1,609 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Stefan L. All rights reserved.
+ *  Licensed under the MIT License. See LICENSE in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as crypto from "crypto";
+import { DBHandlerServer, DboBuilder, PRGLIOSocket, TableInfo, TableOrViewInfo } from "../DboBuilder/DboBuilder";
+import { PostgresNotifListenManager } from "../PostgresNotifListenManager";
+import { DB, getIsSuperUser } from "../Prostgles";
+import { addSync } from "./addSync";
+import { initPubSubManager } from "./initPubSubManager";
+
+import * as pgPromise from 'pg-promise';
+import pg from 'pg-promise/typescript/pg-subset';
+
+import { AnyObject, FieldFilter, SelectParams, SubscribeParams, WAL } from "prostgles-types";
+
+import { find, pickKeys, tryCatch } from "prostgles-types/dist/util";
+import { LocalFuncs, getOnDataFunc, matchesLocalFuncs } from "../DboBuilder/ViewHandler/subscribe";
+import { EVENT_TRIGGER_TAGS } from "../Event_Trigger_Tags";
+import { EventTypes } from "../Logging";
+import { TableRule } from "../PublishParser/PublishParser";
+import { syncData } from "../SyncReplication";
+import { addSub } from "./addSub";
+import { DB_OBJ_NAMES } from "./getPubSubManagerInitQuery";
+import { notifListener } from "./notifListener";
+import { DELETE_DISCONNECTED_APPS_QUERY } from "./orphanTriggerCheck";
+import { pushSubData } from "./pushSubData";
+
+type PGP = pgPromise.IMain<{}, pg.IClient>;
+const pgp: PGP = pgPromise({
+});
+export const asValue = (v: any) => pgp.as.format("$1", [v]);
+export const DEFAULT_SYNC_BATCH_SIZE = 50;
+
+export const log = (...args: any[]) => {
+  if (process.env.TEST_TYPE) {
+    console.log(...args)
+  }
+}
+
+export type BasicCallback = (err?: any, res?: any) => void
+
+export type SyncParams = {
+  socket_id: string;
+  sid: string | undefined;
+  channel_name: string;
+  table_name: string;
+  table_rules?: TableRule;
+  synced_field: string;
+  allow_delete: boolean;
+  id_fields: string[];
+  batch_size: number;
+  filter: object;
+  params: {
+    select: FieldFilter
+  };
+  condition: string;
+  wal?: WAL,
+  throttle?: number;
+  lr?: AnyObject;
+  last_synced: number;
+  is_syncing: boolean;
+}
+
+export type AddSyncParams = {
+  socket: PRGLIOSocket;
+  table_info: TableInfo;
+  table_rules: TableRule;
+  synced_field: string;
+  allow_delete?: boolean;
+  id_fields: string[];
+  filter: object;
+  params: {
+    select: FieldFilter
+  };
+  condition: string;
+  throttle?: number;
+}
+
+export type ViewSubscriptionOptions = ({
+  type: "view";
+  viewName: string;
+  definition: string;
+} | {
+  type: "table";
+  viewName?: undefined;
+  definition?: undefined;
+}) & {
+  relatedTables: {
+    tableName: string;
+    tableNameEscaped: string;
+    condition: string;
+  }[];
+}
+
+export type SubscriptionParams = Pick<SubscribeParams, "throttle" | "throttleOpts"> & {
+  socket_id?: string;
+  channel_name: string;
+
+  /** 
+   * If this is a view then an array with all related tables will be  
+   * */
+  viewOptions?: ViewSubscriptionOptions;
+  parentSubParams: Omit<SubscriptionParams, "parentSubParams"> | undefined;
+
+  table_info: TableOrViewInfo;
+  
+  /* Used as input */
+  table_rules?: TableRule;
+  filter: object;
+  params: SelectParams;
+
+  localFuncs?: LocalFuncs;
+  socket: PRGLIOSocket | undefined;
+
+  last_throttled: number;
+  is_throttling?: any;
+  is_ready?: boolean; 
+}
+
+export type PubSubManagerOptions = {
+  dboBuilder: DboBuilder; 
+  wsChannelNamePrefix?: string;
+  pgChannelName?: string;
+}
+
+export type Subscription = Pick<SubscriptionParams, 
+  | "throttle" 
+  | "is_throttling" 
+  | "last_throttled"
+  | "throttleOpts" 
+  | "channel_name" 
+  | "is_ready" 
+  | "localFuncs" 
+  | "socket" 
+  | "socket_id"
+  | "table_info"
+  | "filter"
+  | "params"
+  | "table_rules"
+> & {
+  triggers: {
+    table_name: string;
+    condition: string;
+    is_related: boolean;
+  }[];
+}
+
+/**
+ * Used to facilitate table subscribe and sync
+ */
+export class PubSubManager {
+  static DELIMITER = '|$prstgls$|' as const;
+
+  static EXCLUDE_QUERY_FROM_SCHEMA_WATCH_ID = "prostgles internal query that should be excluded from schema watch " as const;
+
+  public static create = async (options: PubSubManagerOptions) => {
+    const instance = new PubSubManager(options);
+    const result = await initPubSubManager.bind(instance)();
+    return result;
+  }
+
+  appInfoWasInserted = false;
+  get appId() {
+    return this.dboBuilder.prostgles.appId;
+  }
+  get db(): DB  {
+    return this.dboBuilder.db;
+  }
+  get dbo(): DBHandlerServer  {
+    return this.dboBuilder.dbo;
+  }
+  
+  dboBuilder: DboBuilder;
+  _triggers?: Record<string, string[]>;
+  sockets: AnyObject = {};
+  
+  subs: Subscription[] = [];
+  syncs: SyncParams[] = [];
+  socketChannelPreffix: string;
+  postgresNotifListenManager?: PostgresNotifListenManager;
+
+  private constructor(options: PubSubManagerOptions) {
+    const { wsChannelNamePrefix, dboBuilder } = options;
+    if (!dboBuilder.db || !dboBuilder.dbo) {
+      throw 'MISSING: db_pg, db';
+    }
+    
+    this.dboBuilder = dboBuilder;
+  
+    this.socketChannelPreffix = wsChannelNamePrefix || "_psqlWS_";
+
+    log("Created PubSubManager");
+  }
+  appCheckFrequencyMS = 10 * 1000;
+  appCheck?: ReturnType<typeof setInterval>;
+
+  destroyed = false;
+  destroy = () => {
+    this.destroyed = true;
+    if (this.appCheck) {
+      clearInterval(this.appCheck);
+    }
+    this.subs = [];
+    this.syncs = [];
+    this.postgresNotifListenManager?.destroy();
+  }
+
+  getIsDestroyed = () => {
+    if (this.destroyed) {
+      console.trace("Could not start destroyed instance");
+      return false
+    }
+    return true
+  }
+
+  appChecking = false;
+  checkedListenerTableCond?: string[];
+
+  initialiseEventTriggers = async () => {
+    const { watchSchema } = this.dboBuilder.prostgles.opts;
+    if (watchSchema && !(await getIsSuperUser(this.db))) {
+      console.warn("prostgles watchSchema requires superuser db user. Will not watch using event triggers")
+    }
+
+    try {
+      /** We use these names because they include schema where necessary */
+      const allTableNames = Object.keys(this.dbo).filter(k => this.dbo[k]?.tableOrViewInfo);
+      const tableFilterQuery = allTableNames.length ? `OR table_name NOT IN (${allTableNames.map(tblName => asValue(tblName)).join(", ")})` : "";
+      const query = pgp.as.format(`
+        BEGIN;--  ISOLATION LEVEL SERIALIZABLE;
+        
+        /**                                 
+         * ${PubSubManager.EXCLUDE_QUERY_FROM_SCHEMA_WATCH_ID}
+         *  Drop stale triggers
+         * */
+        DO
+        $do$
+          DECLARE trg RECORD;
+            q   TEXT;
+            ev_trg_needed BOOLEAN := FALSE;
+            ev_trg_exists BOOLEAN := FALSE;
+            is_super_user BOOLEAN := FALSE;
+        BEGIN
+
+            /**
+             *  Delete disconnected app records, this will delete related triggers
+             * */
+            ${DELETE_DISCONNECTED_APPS_QUERY};
+
+            DELETE FROM prostgles.app_triggers
+            WHERE app_id NOT IN (SELECT id FROM prostgles.apps)
+            ${tableFilterQuery}
+            ;
+            
+            /** IS THIS STILL NEEDED? Delete existing triggers without locking 
+            */
+              LOCK TABLE prostgles.app_triggers IN ACCESS EXCLUSIVE MODE;
+              EXECUTE format(
+                $q$
+
+                  CREATE TEMP TABLE %1$I AS --ON COMMIT DROP AS
+                  SELECT * FROM prostgles.app_triggers;
+
+                  DELETE FROM prostgles.app_triggers;
+
+                  INSERT INTO prostgles.app_triggers
+                  SELECT * FROM %1$I;
+
+                  DROP TABLE IF EXISTS %1$I;
+                $q$, 
+                ${asValue('triggers_' + this.appId)}
+              );
+            
+            ${SCHEMA_WATCH_EVENT_TRIGGER_QUERY}
+            
+        END
+        $do$; 
+
+
+        COMMIT;
+      `, { EVENT_TRIGGER_TAGS });
+      
+      await this.db.tx(t => t.any(query))
+      .catch(e => {
+        console.error("prepareTriggers failed: ", e);
+        throw e;
+      });
+
+      return true;
+
+    } catch (e) {
+      console.error("prepareTriggers failed: ", e);
+      throw e;
+    }
+  }
+
+  getClientSubs({ channel_name, localFuncs, socket_id }: Pick<Subscription, "localFuncs" | "socket_id" | "channel_name">): Subscription[] { 
+    return this.subs.filter(s => {
+      return s.channel_name === channel_name && 
+        (matchesLocalFuncs(localFuncs, s.localFuncs) || socket_id && s.socket_id === socket_id)
+    });
+  }
+
+  getTriggerSubs(table_name: string, condition: string): Subscription[] {  
+    const subs = this.subs.filter(s => find(s.triggers, { table_name, condition }));
+    return subs;
+  }
+
+  removeLocalSub(channelName: string, localFuncs: LocalFuncs) {
+    const matchingSubIdx = this.subs.findIndex(s => 
+      s.channel_name === channelName && 
+      getOnDataFunc(localFuncs) === getOnDataFunc(s.localFuncs)
+    );
+    if (matchingSubIdx > -1) {
+      this.subs.splice(matchingSubIdx, 1);
+    } else {
+      console.error("Could not unsubscribe. Subscription might not have initialised yet", { channelName })
+    }
+  }
+
+  getSyncs(table_name: string, condition: string) {
+    return (this.syncs || [])
+      .filter((s: SyncParams) => s.table_name === table_name && s.condition === condition);
+  }
+
+  notifListener = notifListener.bind(this);
+
+  getSubData = async (sub: Subscription): Promise<
+    { data: any[]; err?: undefined; } | 
+    { data?: undefined; err: any; }
+  > => {
+    const { table_info, filter, params, table_rules } = sub;  //, subOne = false 
+    const { name: table_name } = table_info;
+    
+    if (!this.dbo?.[table_name]?.find) {
+      throw new Error(`this.dbo.${table_name}.find undefined`);
+    }
+
+    try {
+      const data = await this.dbo?.[table_name]!.find!(filter, params, undefined, table_rules)
+      return { data };
+    } catch(err){
+      return { err };
+    }
+  } 
+
+  pushSubData = pushSubData.bind(this);
+
+  upsertSocket(socket: PRGLIOSocket | undefined) {
+    if (socket && !this.sockets[socket.id]) {
+      this.sockets[socket.id] = socket;
+      socket.on("disconnect", () => {
+
+        this.subs = this.subs.filter(s => {
+          return !(s.socket && s.socket.id === socket.id);
+        });
+
+        this.syncs = this.syncs.filter(s => {
+          return !(s.socket_id && s.socket_id === socket.id);
+        });
+
+        delete this.sockets[socket.id];
+
+        this._log({ 
+          type: "sync",
+          command: "upsertSocket.disconnect",
+          tableName: "",
+          duration: 0,
+          sid: this.dboBuilder.prostgles.authHandler?.getSIDNoError({ socket }),
+          socketId: socket.id,
+          connectedSocketIds: this.connectedSocketIds,
+          remainingSubs: JSON.stringify(this.subs.map(s => ({ tableName: s.table_info.name, triggers: s.triggers }))),
+          remainingSyncs: JSON.stringify(this.syncs.map(s => pickKeys(s, ["table_name", "condition"]))),
+        });
+
+        return "ok";
+      });
+    }
+  }
+
+  get connectedSocketIds() {
+    return this.dboBuilder.prostgles.connectedSockets.map(s => s.id);
+  }
+  _log = (params: EventTypes.Sync) => {
+    return this.dboBuilder.prostgles.opts.onLog?.({ ...params });
+  }
+
+  syncTimeout?: ReturnType<typeof setTimeout>;
+  syncData = syncData.bind(this);
+
+  addSync = addSync.bind(this);
+  
+  addSub = addSub.bind(this);
+
+  getActiveListeners = (): { table_name: string; condition: string }[] => {
+    const activeListeners: { table_name: string; condition: string }[] = [];
+    const upsert = (t: string, c: string) => {
+      if (!activeListeners.find(r => r.table_name === t && r.condition === c)) {
+        activeListeners.push({ table_name: t, condition: c });
+      }
+    }
+    (this.syncs ?? []).map(s => {
+      upsert(s.table_name, s.condition)
+    });
+
+    this.subs.forEach(s => {
+      s.triggers.forEach(trg => {
+        upsert(trg.table_name, trg.condition);
+      });
+    });
+
+    return activeListeners;
+  }
+
+  /**
+   * Sync triggers with database
+   *  */
+  refreshTriggers = async () => {
+
+    const triggers: {
+      table_name: string;
+      condition: string;
+    }[] = await this.db.any(`
+        SELECT *
+        FROM prostgles.v_triggers
+        WHERE app_id = $1
+        ORDER BY table_name, condition
+      `, [this.dboBuilder.prostgles.appId]
+    );
+
+    this._triggers = {};
+    triggers.map(t => {
+      this._triggers ??= {};
+      this._triggers[t.table_name] ??= [];
+      if (!this._triggers[t.table_name]?.includes(t.condition)) {
+        this._triggers[t.table_name]?.push(t.condition)
+      }
+    });
+  }
+
+  addingTrigger: any;
+  addTriggerPool?: Record<string, string[]> = undefined;
+  async addTrigger(params: { table_name: string; condition: string; }, viewOptions: ViewSubscriptionOptions | undefined, socket: PRGLIOSocket | undefined) {
+    
+    const addedTrigger = await tryCatch(async () => {
+
+      const { table_name } = { ...params }
+      let { condition } = { ...params }
+      if (!table_name) throw "MISSING table_name";
+
+      if (!condition || !condition.trim().length) {
+        condition = "TRUE";
+      }
+
+      if (this.dbo[table_name]?.tableOrViewInfo?.isHyperTable) {
+        throw "Triggers do not work on timescaledb hypertables due to bug:\nhttps://github.com/timescale/timescaledb/issues/1084"
+      }
+
+      const trgVals = {
+        tbl: asValue(table_name),
+        cond: asValue(condition),
+        condHash: asValue(
+          crypto.createHash('md5').update(condition).digest('hex')
+        )
+      };
+
+      await this.db.tx(t => t.any(`
+        BEGIN WORK;
+        /* ${PubSubManager.EXCLUDE_QUERY_FROM_SCHEMA_WATCH_ID} */
+        /* why is this lock level needed? */
+        --LOCK TABLE prostgles.app_triggers IN ACCESS EXCLUSIVE MODE;
+
+        /** app_triggers is not refreshed when tables are dropped */
+        DELETE FROM prostgles.app_triggers at
+        WHERE app_id = ${asValue(this.appId)} 
+        AND NOT EXISTS (
+          SELECT 1  
+          FROM pg_catalog.pg_trigger t
+          WHERE tgname like format('prostgles_triggers_%s_', at.table_name) || '%'
+          AND tgenabled = 'O'
+        );
+
+        INSERT INTO prostgles.app_triggers (
+          table_name, 
+          condition, 
+          condition_hash, 
+          app_id, 
+          related_view_name, 
+          related_view_def
+        ) 
+        VALUES (
+          ${trgVals.tbl}, 
+          ${trgVals.cond},
+          ${trgVals.condHash},
+          ${asValue(this.appId)}, 
+          ${asValue(viewOptions?.viewName ?? null)}, 
+          ${asValue(viewOptions?.definition ?? null)}
+        )
+        ON CONFLICT DO NOTHING;
+
+        COMMIT WORK;
+      `));
+
+      /** This might be redundant due to trigger on app_triggers */
+      await this.refreshTriggers();
+
+      return trgVals;
+    });
+
+    await this._log({
+      type: "sync",
+      command: "addTrigger",
+      condition: addedTrigger.cond ?? params.condition,
+      duration: addedTrigger.duration,
+      socketId: socket?.id,
+      state: !addedTrigger.tbl? "fail" : "ok",
+      error: addedTrigger.error,
+      sid: this.dboBuilder.prostgles.authHandler?.getSIDNoError({ socket }),
+      tableName: addedTrigger.tbl ?? params.table_name,
+      connectedSocketIds: this.dboBuilder.prostgles.connectedSockets.map(s => s.id),
+      localParams: { socket }
+    });
+
+    if(addedTrigger.error) throw addedTrigger.error;
+
+    return addedTrigger;
+  }
+}
+
+const SCHEMA_WATCH_EVENT_TRIGGER_QUERY = `
+
+  is_super_user := EXISTS (select 1 from pg_user where usename = CURRENT_USER AND usesuper IS TRUE);
+
+  /* DROP the old buggy schema watch trigger */
+  IF EXISTS (
+    SELECT 1 FROM pg_catalog.pg_event_trigger
+    WHERE evtname = 'prostgles_schema_watch_trigger'
+  ) AND is_super_user IS TRUE 
+  THEN
+    DROP EVENT TRIGGER IF EXISTS prostgles_schema_watch_trigger;
+  END IF;
+
+  ev_trg_needed := EXISTS (
+    SELECT 1 FROM prostgles.apps 
+    WHERE watching_schema_tag_names IS NOT NULL
+  );
+  ev_trg_exists := EXISTS (
+    SELECT 1 FROM pg_catalog.pg_event_trigger
+    WHERE evtname = ${asValue(DB_OBJ_NAMES.schema_watch_trigger)}
+  );
+
+  /* DROP stale event trigger */
+  IF 
+    is_super_user IS TRUE 
+    AND ev_trg_needed IS FALSE 
+    AND ev_trg_exists IS TRUE 
+  THEN
+
+      SELECT format(
+        $$ 
+          DROP EVENT TRIGGER IF EXISTS %I ; 
+          DROP EVENT TRIGGER IF EXISTS %I ; 
+        $$
+        , ${asValue(DB_OBJ_NAMES.schema_watch_trigger)}
+        , ${asValue(DB_OBJ_NAMES.schema_watch_trigger_drop)}
+      )
+      INTO q;
+      EXECUTE q;
+
+  /* CREATE event trigger */
+  ELSIF 
+      is_super_user IS TRUE 
+      AND ev_trg_needed IS TRUE 
+      AND ev_trg_exists IS FALSE 
+  THEN
+
+      DROP EVENT TRIGGER IF EXISTS ${DB_OBJ_NAMES.schema_watch_trigger};
+      CREATE EVENT TRIGGER ${DB_OBJ_NAMES.schema_watch_trigger} 
+      ON ddl_command_end
+      WHEN TAG IN (\${EVENT_TRIGGER_TAGS:csv})
+      EXECUTE PROCEDURE ${DB_OBJ_NAMES.schema_watch_func}();
+
+      DROP EVENT TRIGGER IF EXISTS ${DB_OBJ_NAMES.schema_watch_trigger_drop};
+      CREATE EVENT TRIGGER ${DB_OBJ_NAMES.schema_watch_trigger_drop} 
+      ON sql_drop
+      --WHEN TAG IN (\${EVENT_TRIGGER_TAGS:csv})
+      EXECUTE PROCEDURE ${DB_OBJ_NAMES.schema_watch_func}();
+
+  END IF;
+`
+
+export const NOTIF_TYPE = {
+  data: "data_has_changed",
+  data_trigger_change: "data_watch_triggers_have_changed",
+  schema: "schema_has_changed"
+} as const;
+export const NOTIF_CHANNEL = {
+  preffix: 'prostgles_' as const,
+  getFull: (appID: string | undefined) => {
+    if (!appID) throw "No appID";
+    return NOTIF_CHANNEL.preffix + appID;
+  }
+}
+
+export const parseCondition = (condition: string): string => condition && condition.trim().length ? condition : "TRUE"
+
+export { omitKeys, pickKeys } from "prostgles-types";