prostgles-server 4.2.72 → 4.2.74

package/lib/PubSubManager/PubSubManager.ts

@@ -167,6 +167,10 @@ export class PubSubManager {
     return result;
   }
 
+  appInfoWasInserted = false;
+  get appId() {
+    return this.dboBuilder.prostgles.appId;
+  }
   get db(): DB  {
     return this.dboBuilder.db;
   }
@@ -195,12 +199,6 @@ export class PubSubManager {
 
     log("Created PubSubManager");
   }
-
-  /**
-   * Used facilitate concurrent prostgles connections to the same database
-   */
-  appID?: string;
-
   appCheckFrequencyMS = 10 * 1000;
   appCheck?: ReturnType<typeof setInterval>;
 
@@ -227,8 +225,6 @@ export class PubSubManager {
   checkedListenerTableCond?: string[];
 
   initialiseEventTriggers = async () => {
-    if (!this.appID) throw "prepareTriggers failed: this.appID missing";
-
     const { watchSchema } = this.dboBuilder.prostgles.opts;
     if (watchSchema && !(await getIsSuperUser(this.db))) {
       console.warn("prostgles watchSchema requires superuser db user. Will not watch using event triggers")
@@ -268,7 +264,7 @@ export class PubSubManager {
 
                 DROP TABLE IF EXISTS %1$I;
               $q$, 
-              ${asValue('triggers_' + this.appID)}
+              ${asValue('triggers_' + this.appId)}
             );
 
             is_super_user := EXISTS (select 1 from pg_user where usename = CURRENT_USER AND usesuper IS TRUE);
@@ -402,15 +398,14 @@ export class PubSubManager {
   
   addSub = addSub.bind(this);
 
-
   getActiveListeners = (): { table_name: string; condition: string }[] => {
-    const result: { table_name: string; condition: string }[] = [];
+    const activeListeners: { table_name: string; condition: string }[] = [];
     const upsert = (t: string, c: string) => {
-      if (!result.find(r => r.table_name === t && r.condition === c)) {
-        result.push({ table_name: t, condition: c });
+      if (!activeListeners.find(r => r.table_name === t && r.condition === c)) {
+        activeListeners.push({ table_name: t, condition: c });
       }
     }
-    (this.syncs || []).map(s => {
+    (this.syncs ?? []).map(s => {
       upsert(s.table_name, s.condition)
     });
 
@@ -420,7 +415,7 @@ export class PubSubManager {
       });
     });
 
-    return result;
+    return activeListeners;
   }
 
   /**
@@ -436,7 +431,7 @@ export class PubSubManager {
         FROM prostgles.v_triggers
         WHERE app_id = $1
         ORDER BY table_name, condition
-      `, [this.appID]
+      `, [this.dboBuilder.prostgles.appId]
     );
 
     this._triggers = {};
@@ -459,7 +454,6 @@ export class PubSubManager {
       const { table_name } = { ...params }
       let { condition } = { ...params }
       if (!table_name) throw "MISSING table_name";
-      if (!this.appID) throw "MISSING appID";
 
       if (!condition || !condition.trim().length) {
         condition = "TRUE";
@@ -479,8 +473,20 @@ export class PubSubManager {
         /* ${ PubSubManager.EXCLUDE_QUERY_FROM_SCHEMA_WATCH_ID} */
         LOCK TABLE prostgles.app_triggers IN ACCESS EXCLUSIVE MODE;
 
-        INSERT INTO prostgles.app_triggers (table_name, condition, app_id, related_view_name, related_view_def) 
-          VALUES (${trgVals.tbl}, ${trgVals.cond}, ${asValue(this.appID)}, ${asValue(viewOptions?.viewName ?? null)}, ${asValue(viewOptions?.definition ?? null)})
+        INSERT INTO prostgles.app_triggers (
+          table_name, 
+          condition, 
+          app_id, 
+          related_view_name, 
+          related_view_def
+        ) 
+        VALUES (
+          ${trgVals.tbl}, 
+          ${trgVals.cond}, 
+          ${asValue(this.appId)}, 
+          ${asValue(viewOptions?.viewName ?? null)}, 
+          ${asValue(viewOptions?.definition ?? null)}
+        )
         ON CONFLICT DO NOTHING;
               
         COMMIT WORK;

package/lib/PubSubManager/getInitQuery.ts

@@ -1,6 +1,7 @@
 
 import { asValue, NOTIF_CHANNEL, NOTIF_TYPE, PubSubManager } from "./PubSubManager";
 const { version } = require("../../package.json");
+import { getAppCheckQuery } from "./orphanTriggerCheck";
 
 export const DB_OBJ_NAMES = {
   trigger_add_remove_func: "prostgles.trigger_add_remove_func",
@@ -279,7 +280,6 @@ BEGIN
                         err_detail = PG_EXCEPTION_DETAIL,
                         err_hint = PG_EXCEPTION_HINT;
 
-
                     END;
 
                     --RAISE NOTICE 'has_errors: % ', has_errors;
@@ -307,7 +307,7 @@ BEGIN
                                   THEN concat_ws('; ', 'error', err_text, err_detail, err_hint, 'query: ' || query ) 
                                   ELSE COALESCE(v_trigger.cids, '') 
                                 END
-                                ${this.dboBuilder.prostgles.opts.DEBUG_MODE? (", COALESCE(current_query(), 'current_query ??'), (select json_agg(t)::TEXT FROM (SELECT * from old_table) t), query") : ""}
+                                ${this.dboBuilder.prostgles.opts.DEBUG_MODE? (", COALESCE(current_query(), 'current_query ??'), ' ', query") : ""}
                               ), 7999/4) -- Some chars are 2bytes -> 'Ω'
                             );
                         END LOOP;
@@ -323,6 +323,7 @@ BEGIN
                     END IF;
                 END IF;
 
+                ${getAppCheckQuery()}
 
                 RETURN NULL;
                
@@ -574,6 +575,8 @@ BEGIN
                       );
                     END LOOP;
 
+                    ${getAppCheckQuery()}
+
                 END IF;
 
             END;

package/lib/PubSubManager/initPubSubManager.ts

@@ -1,7 +1,7 @@
 import { PostgresNotifListenManager } from "../PostgresNotifListenManager";
 import { getWatchSchemaTagList } from "../SchemaWatch/getWatchSchemaTagList";
 import { asValue, log, NOTIF_CHANNEL, PubSubManager } from "./PubSubManager";
-const REALTIME_TRIGGER_CHECK_QUERY = "prostgles-server internal query used to manage realtime triggers" as const;  
+export const REALTIME_TRIGGER_CHECK_QUERY = "prostgles-server internal query used to manage realtime triggers" as const;  
 import { getInitQuery } from "./getInitQuery";
 
 export async function initPubSubManager(this: PubSubManager): Promise<PubSubManager | undefined> {
@@ -15,25 +15,37 @@ export async function initPubSubManager(this: PubSubManager): Promise<PubSubMana
     if (!this.getIsDestroyed()) return;
 
     /* Prepare App id */
-    if (!this.appID) {
+    if (!this.appInfoWasInserted) {
+      this.appInfoWasInserted = true;
       const check_frequency_ms = this.appCheckFrequencyMS;
       const watching_schema_tag_names = this.dboBuilder.prostgles.schemaWatch?.type.watchType !== "NONE" ? getWatchSchemaTagList(this.dboBuilder.prostgles.opts.watchSchema) : null;
-      const raw = await this.db.one(
-        "INSERT INTO prostgles.apps (check_frequency_ms, watching_schema_tag_names, application_name) \
-        VALUES($1, $2, current_setting('application_name')) \
+      await this.db.one(
+        "INSERT INTO prostgles.apps (id, check_frequency_ms, watching_schema_tag_names, application_name) \
+        VALUES($1, $2, $3, current_setting('application_name')) \
         RETURNING *; "
         , [
+          this.appId,
           check_frequency_ms,
           watching_schema_tag_names
         ]
       );
-      this.appID = raw.id;
 
-      if (!this.appCheck) {
+      const appRecord = await this.db.one("SELECT * FROM prostgles.apps WHERE id = $1", [this.appId]);
+      if (!appRecord || !appRecord.application_name?.includes(this.appId)) {
+        throw `initPubSubManager error: App record with application_name containing appId (${this.appId}) not found`;
+      }
+
+      await this.db.any(`
+        DELETE FROM prostgles.app_triggers
+        WHERE app_id = ${asValue(this.appId)}
+      `);
+      
+      console.log("REMOVE app check disabled");
+      if (!this.appCheck && Math.random() < 12) {
 
         this.appCheck = setInterval(async () => {
           let checkForStaleTriggers = "";
-          try {   //  drop owned by api
+          try {
 
             this.appChecking = true;
  
@@ -43,7 +55,7 @@ export async function initPubSubManager(this: PubSubManager): Promise<PubSubMana
               SET last_used = CASE WHEN (table_name, condition) IN (
                 ${listeners.map(l => ` ( ${asValue(l.table_name)}, ${asValue(l.condition)} ) `).join(", ")}
               ) THEN NOW() ELSE last_used END
-              WHERE app_id = ${asValue(this.appID)};
+              WHERE app_id = ${asValue(this.appId)};
             `;
 
             const checkedListenerTableCond = listeners.map(l => `${l.table_name}.${l.condition}`);
@@ -62,10 +74,11 @@ export async function initPubSubManager(this: PubSubManager): Promise<PubSubMana
                     /* UPDATE currently used triggers */
                     ${updateCurrentlyUsedTriggersQuery}
 
-                    /* DELETE stale triggers for current app. Other triggers will be deleted on app startup */
-                    DELETE FROM prostgles.app_triggers
-                    WHERE app_id = ${asValue(this.appID)}
-                    AND last_used < NOW() - 4 * ${asValue(this.appCheckFrequencyMS)} * interval '1 millisecond'; -- 10 seconds at the moment
+                    /* DELETE stale triggers for current app. Other triggers will be deleted on app startup 
+                      DELETE FROM prostgles.app_triggers
+                      WHERE app_id = ${asValue(this.appId)}
+                      AND last_used < NOW() - 4 * ${asValue(this.appCheckFrequencyMS)} * interval '1 millisecond'; -- 10 seconds at the moment
+                    */
 
                 END IF;
               
@@ -94,7 +107,7 @@ export async function initPubSubManager(this: PubSubManager): Promise<PubSubMana
                     /* Last check used to remove disconnected apps */
                     UPDATE prostgles.apps 
                     SET last_check = NOW()
-                    WHERE id = ${asValue(this.appID)};
+                    WHERE id = ${asValue(this.appId)};
 
                     ${dataTriggerCheckQuery}
                 END IF;
@@ -106,7 +119,7 @@ export async function initPubSubManager(this: PubSubManager): Promise<PubSubMana
               this.db.any(`    
               /* 
                 ${queryIdentifier}
-                ${REALTIME_TRIGGER_CHECK_QUERY} 
+                ${REALTIME_TRIGGER_CHECK_QUERY}
                 ${PubSubManager.EXCLUDE_QUERY_FROM_SCHEMA_WATCH_ID}
               */
               DO $$ 
@@ -129,8 +142,10 @@ export async function initPubSubManager(this: PubSubManager): Promise<PubSubMana
 
             /** In some cases a query idles and blocks everything else. Terminate all similar queries */
             this.db.any(
-              "SELECT state, pg_terminate_backend(pid) from pg_stat_activity WHERE query ilike ${qid} and pid <>  pg_backend_pid();", 
-              { qid: "%" + REALTIME_TRIGGER_CHECK_QUERY + "%" }
+              "SELECT state, pg_terminate_backend(pid) from pg_stat_activity \
+               WHERE query ilike ${qid} \
+               AND pid <> pg_backend_pid();", 
+              { qid: `%${REALTIME_TRIGGER_CHECK_QUERY}%` }
             );
 
             /** If no tries left
@@ -150,7 +165,7 @@ export async function initPubSubManager(this: PubSubManager): Promise<PubSubMana
       }
     }
 
-    this.postgresNotifListenManager = new PostgresNotifListenManager(this.db, this.notifListener, NOTIF_CHANNEL.getFull(this.appID));
+    this.postgresNotifListenManager = new PostgresNotifListenManager(this.db, this.notifListener, NOTIF_CHANNEL.getFull(this.appId));
 
     await this.initialiseEventTriggers();
 

package/lib/PubSubManager/notifListener.ts

@@ -80,6 +80,21 @@ export async function notifListener(this: PubSubManager, data: { payload: string
 
     state = "ok";
     const conditions = tableTriggers.filter((c, i) => condition_ids.includes(i));
+    const orphanedConditions = condition_ids.filter((condId) => typeof tableTriggers.at(condId) !== "string" );
+    if(orphanedConditions.length){
+      this.db
+        .any(" \
+          DELETE FROM prostgles.app_triggers \
+          WHERE table_name = $1 \
+          AND id IN ($2:csv) \
+          AND app_id = $3 \
+          ", 
+          [table_name, orphanedConditions, this.appId]
+        )
+        .catch(e => {
+          console.error("Error deleting orphaned triggers", e);
+        });
+    }
 
     conditions.map(condition => {
 

package/lib/PubSubManager/orphanTriggerCheck.ts

@@ -0,0 +1,79 @@
+import { PubSubManager } from './PubSubManager';
+import { REALTIME_TRIGGER_CHECK_QUERY } from "./initPubSubManager";
+
+/**
+ * Schema and Data watch triggers (DB_OBJ_NAMES.schema_watch_func, DB_OBJ_NAMES.data_watch_func) 
+ * survive and continue to user resources even after the client disconnects.
+ * We must therefore .
+ */
+
+const queryIdentifier = "prostgles query used to keep track of which prgl backend clients are still connected";
+const connectedApplicationNamesQuery = `
+  SELECT DISTINCT application_name
+  FROM prostgles.apps 
+  WHERE application_name IN (
+    SELECT application_name 
+    FROM pg_catalog.pg_stat_activity
+    WHERE pid = pg_backend_pid()
+  )
+`;
+
+const LAST_CHECKED_SETTING_NAME = 'prostgles.last_checked';
+
+/** It is a function to prevent undefined EXCLUDE_QUERY_FROM_SCHEMA_WATCH_ID */
+export const getAppCheckQuery = () => `
+  /* 
+    ${queryIdentifier}
+    ${REALTIME_TRIGGER_CHECK_QUERY} 
+    ${PubSubManager.EXCLUDE_QUERY_FROM_SCHEMA_WATCH_ID}
+  */
+  IF
+    /* prostgles schema must exist */
+    EXISTS (
+      SELECT 1 
+      FROM information_schema.tables 
+      WHERE  table_schema = 'prostgles'
+      AND    table_name   = 'apps'
+    )
+    /* Ensure we don't check too often */
+    AND (
+      NULLIF(current_setting('${LAST_CHECKED_SETTING_NAME}', true), '') IS NULL
+      OR current_setting('${LAST_CHECKED_SETTING_NAME}', true)::timestamp < NOW() - INTERVAL '1 minute'
+    )
+  THEN
+    EXECUTE format('SET LOCAL ${LAST_CHECKED_SETTING_NAME} TO %L', now());
+    --USED FOR DEBUG REMOVE
+    PERFORM pg_notify('prostgles', current_setting('${LAST_CHECKED_SETTING_NAME}', true));
+
+    IF EXISTS (
+      ${connectedApplicationNamesQuery}
+    ) THEN
+
+      /* Remove disconnected apps */
+      WITH deleted_apps AS (
+        DELETE FROM prostgles.apps a
+        WHERE NOT EXISTS (
+          SELECT 1
+          FROM pg_catalog.pg_stat_activity s
+          WHERE s.application_name = a.application_name
+        )
+        RETURNING a.id
+      )
+      DELETE FROM prostgles.app_triggers
+      WHERE app_id IN (
+        SELECT id 
+        FROM deleted_apps
+      );
+ 
+    END IF;
+  END IF;
+`;
+
+
+    // /* Ensure we don't check in paralel */
+    // AND NOT EXISTS (
+    //   SELECT 1
+    //   FROM pg_catalog.pg_stat_activity
+    //   WHERE query ilike '%${queryIdentifier}%'
+    //   AND state = 'active'
+    // )

package/lib/SchemaWatch/SchemaWatch.ts

@@ -1,6 +1,5 @@
 import type { DboBuilder } from "../DboBuilder/DboBuilder";
 import { EVENT_TRIGGER_TAGS } from "../Event_Trigger_Tags";
-import { OnSchemaChangeCallback } from "../Prostgles";
 import { PubSubManager, log } from "../PubSubManager/PubSubManager";
 import { ValidatedWatchSchemaType, getValidatedWatchSchemaType } from "./getValidatedWatchSchemaType";
 const COMMAND_FIRST_KEYWORDS = EVENT_TRIGGER_TAGS
@@ -11,6 +10,8 @@ const DB_FALLBACK_COMMANDS = Array.from(new Set(COMMAND_FIRST_KEYWORDS));
 
 export type VoidFunction = () => void;
 
+export type OnSchemaChangeCallback = ((event: { command: string; query: string }) => void);
+
 export class SchemaWatch {
 
   dboBuilder: DboBuilder;

package/lib/SchemaWatch/getValidatedWatchSchemaType.ts

@@ -1,5 +1,5 @@
 import type { DboBuilder } from "../DboBuilder/DboBuilder";
-import { OnSchemaChangeCallback } from "../Prostgles";
+import { OnSchemaChangeCallback } from "./SchemaWatch";
 
 export type ValidatedWatchSchemaType = 
 | { watchType: "NONE" }

package/lib/SchemaWatch/getWatchSchemaTagList.ts

@@ -1,6 +1,6 @@
 import { getKeys, isObject } from "prostgles-types";
 import { EVENT_TRIGGER_TAGS } from "../Event_Trigger_Tags";
-import { ProstglesInitOptions } from "../Prostgles";
+import { ProstglesInitOptions } from "../ProstglesTypes";
 
 export const getWatchSchemaTagList = (watchSchema: ProstglesInitOptions["watchSchema"]) => {
   if(!watchSchema) return undefined;

package/lib/index.ts

@@ -1,5 +1,6 @@
 import { SessionUser } from "./AuthHandler";
-import { Prostgles, ProstglesInitOptions } from "./Prostgles";
+import { Prostgles } from "./Prostgles";
+import { ProstglesInitOptions } from "./ProstglesTypes";
 
 function prostgles<S = void, SUser extends SessionUser = SessionUser>(params: ProstglesInitOptions<S, SUser>){
 

package/lib/initProstgles.ts

@@ -5,10 +5,11 @@ import { isEmpty, pickKeys } from "prostgles-types";
 import { AuthHandler } from "./AuthHandler";
 import { DBEventsManager } from "./DBEventsManager";
 import { DBOFullyTyped } from "./DBSchemaBuilder";
-import { DBHandlerServer, Prostgles, ProstglesInitOptions, getIsSuperUser } from "./Prostgles";
+import { DBHandlerServer, Prostgles, getIsSuperUser } from "./Prostgles";
 import { DbTableInfo, PublishParser } from "./PublishParser/PublishParser";
 import { sleep } from "./utils";
 import { SchemaWatch } from "./SchemaWatch/SchemaWatch";
+import { ProstglesInitOptions } from "./ProstglesTypes";
 
 export type DbConnection = string | pg.IConnectionParameters<pg.IClient>;
 export type DbConnectionOpts = pg.IDefaults;
@@ -58,9 +59,31 @@ export type InitResult = {
 export const initProstgles = async function(this: Prostgles, onReady: OnReadyCallbackBasic, reason: OnInitReason): Promise<InitResult> {
   this.loaded = false;
 
-  /* 1. Connect to db */
   if (!this.db) {
-    const { db, pgp } = getDbConnection(this.opts.dbConnection, this.opts.dbOptions, this.opts.DEBUG_MODE,
+    let existingAppName = "";
+    let connString = "";
+    if(typeof this.opts.dbConnection === "string"){
+      connString = this.opts.dbConnection;
+    } else if(this.opts.dbConnection.connectionString){
+      connString = this.opts.dbConnection.connectionString;
+    } else {
+      existingAppName = this.opts.dbConnection.application_name ?? "";
+    }
+
+    if(connString){
+      try {
+        const url = new URL(connString);
+        existingAppName = url.searchParams.get("application_name") ?? url.searchParams.get("ApplicationName") ?? "";
+      } catch (e) {
+
+      }
+    }
+
+    const conObj = typeof this.opts.dbConnection === "string" ? { connectionString: this.opts.dbConnection } : this.opts.dbConnection 
+    const application_name = `prostgles ${this.appId} ${existingAppName}`;
+
+    /* 1. Connect to db */
+    const { db, pgp } = getDbConnection({ ...conObj, application_name }, this.opts.dbOptions, this.opts.DEBUG_MODE,
       notice => {
         if (this.opts.onNotice) this.opts.onNotice(notice);
         if (this.dbEventsManager) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prostgles-server",
-  "version": "4.2.72",
+  "version": "4.2.74",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -53,10 +53,10 @@
     "@types/pg": "^8.11.5",
     "@types/pg-cursor": "^2.7.2",
     "@types/sharp": "^0.30.4",
-    "@types/socket.io": "^3.0.2",
     "@typescript-eslint/eslint-plugin": "^5.62.0",
     "@typescript-eslint/parser": "^5.62.0",
     "eslint": "^8.51.0",
+    "socket.io": "^4.7.5",
     "typescript": "^5.3.3"
   }
 }

package/tests/isomorphicQueries.spec.ts

@@ -487,11 +487,11 @@ export const isomorphicQueries = async (db: DBOFullyTyped | DBHandlerClient, log
         ) as { tgname: string; enabled: boolean; }[];
       }
       await sub.unsubscribe();
-      let validTriggers = await getTableTriggers(table_name)
-      assert.equal(validTriggers.filter(t => t.enabled).length, 3);
+      let validTriggers = await getTableTriggers(table_name);
+      assert.equal(validTriggers.filter(t => t.enabled).length, 3, '3 Triggers should be enabled');
       await db.sql?.(`DELETE FROM prostgles.app_triggers`, []); //  WHERE table_name = $1
-      validTriggers = await getTableTriggers(table_name)
-      assert.equal(validTriggers.length, 3);
+      validTriggers = await getTableTriggers(table_name);
+      assert.equal(validTriggers.length, 3, '3 Triggers should exist but be disabled');
       assert.equal(validTriggers.filter(t => t.enabled).length, 0);
     }
 

package/tests/server/index.ts

@@ -124,6 +124,7 @@ function dd(){
 			expressApp: app,
 			tableName: "files",
 		},
+		// DEBUG_MODE: true,
 		restApi: {
 			expressApp: app,
 			routePrefix: "/api"