prostgles-server 4.2.159 → 4.2.161

package/lib/initProstgles.ts

@@ -0,0 +1,319 @@
+import * as pgPromise from "pg-promise";
+import pg from "pg-promise/typescript/pg-subset";
+import { getKeys, isEmpty } from "prostgles-types";
+import { AuthHandler } from "./Auth/AuthHandler";
+import { DBEventsManager } from "./DBEventsManager";
+import { DBOFullyTyped } from "./DBSchemaBuilder";
+import { DBHandlerServer, Prostgles, getIsSuperUser } from "./Prostgles";
+import { ProstglesInitOptions } from "./ProstglesTypes";
+import { DbTableInfo, PublishParser } from "./PublishParser/PublishParser";
+import { SchemaWatch } from "./SchemaWatch/SchemaWatch";
+import { sleep } from "./utils";
+
+export type DbConnection = string | pg.IConnectionParameters<pg.IClient>;
+export type DbConnectionOpts = pg.IDefaults;
+
+export type PGP = pgPromise.IMain<{}, pg.IClient>;
+export type DB = pgPromise.IDatabase<{}, pg.IClient>;
+
+export type UpdateableOptions = Pick<ProstglesInitOptions, "fileTable" | "restApi" | "tableConfig" | "schema" | "auth">;
+export type OnInitReason = 
+  | { 
+    type: "schema change"; 
+    query: string; 
+    command: string; 
+  } 
+  | {
+    type: "prgl.update";
+    newOpts: Omit<UpdateableOptions, typeof clientOnlyUpdateKeys[number]>;
+  }
+  | { 
+    type: "init" | "prgl.restart" | "TableConfig"
+  };
+
+type OnReadyParamsCommon = {
+  db: DB;
+  tables: DbTableInfo[];
+  reason: OnInitReason;
+}
+export type OnReadyParamsBasic = OnReadyParamsCommon & {
+  dbo: DBHandlerServer; 
+}
+export type OnReadyParams<S> = OnReadyParamsCommon & {
+  dbo: DBOFullyTyped<S>; 
+}
+
+export type OnReadyCallback<S = void> = (params: OnReadyParams<S>) => any;
+export type OnReadyCallbackBasic = (params: OnReadyParamsBasic) => any;
+
+export type InitResult = {
+  db: DBOFullyTyped;
+  _db: DB;
+  pgp: PGP;
+  io?: any;
+  destroy: () => Promise<boolean>;
+  /**
+   * Generated database public schema TS types for all tables and views
+   */
+  getTSSchema: () => string;
+  update: (newOpts: UpdateableOptions) => Promise<void>;
+  restart: () => Promise<InitResult>; 
+  options: ProstglesInitOptions;
+}
+
+const clientOnlyUpdateKeys = ["auth"] as const satisfies (keyof UpdateableOptions)[];
+
+export const initProstgles = async function(this: Prostgles, onReady: OnReadyCallbackBasic, reason: OnInitReason): Promise<InitResult> {
+  this.loaded = false;
+
+  if (!this.db) {
+    let existingAppName = "";
+    let connString = "";
+    if(typeof this.opts.dbConnection === "string"){
+      connString = this.opts.dbConnection;
+    } else if(this.opts.dbConnection.connectionString){
+      connString = this.opts.dbConnection.connectionString;
+    } else {
+      existingAppName = this.opts.dbConnection.application_name ?? "";
+    }
+
+    if(connString){
+      try {
+        const url = new URL(connString);
+        existingAppName = url.searchParams.get("application_name") ?? url.searchParams.get("ApplicationName") ?? "";
+      } catch (e) {
+
+      }
+    }
+
+    const conObj = typeof this.opts.dbConnection === "string" ? { connectionString: this.opts.dbConnection } : this.opts.dbConnection 
+    const application_name = `prostgles ${this.appId} ${existingAppName}`;
+
+    /* 1. Connect to db */
+    const { db, pgp } = getDbConnection({
+      ...this.opts,
+      dbConnection: { ...conObj, application_name }, 
+      onNotice: notice => {
+        if (this.opts.onNotice) this.opts.onNotice(notice);
+        if (this.dbEventsManager) {
+          this.dbEventsManager.onNotice(notice)
+        }
+      }
+    });
+    this.db = db;
+    this.pgp = pgp;
+    this.isSuperUser = await getIsSuperUser(db);
+  }
+  this.checkDb();
+
+  const db = this.db!;
+  const pgp = this.pgp!;
+
+  /* 2. Execute any SQL file if provided */
+  if (this.opts.sqlFilePath) {
+    await this.runSQLFile(this.opts.sqlFilePath);
+  }
+
+  try {
+
+    await this.refreshDBO();
+    await this.initTableConfig(reason);
+    await this.initFileTable();
+    await this.initRestApi();
+
+    this.schemaWatch = await SchemaWatch.create(this.dboBuilder);
+
+    if (this.opts.publish) {
+
+      if (!this.opts.io) {
+        console.warn("IO missing. Publish has no effect without io");
+      }
+
+      /* 3.9 Check auth config */
+      await this.initAuthHandler();
+
+      this.publishParser = new PublishParser(this.opts.publish, this.opts.publishMethods as any, this.opts.publishRawSQL, this.dbo!, this.db, this as any);
+      this.dboBuilder.publishParser = this.publishParser;
+
+      /* 4. Set publish and auth listeners */
+      await this.setSocketEvents();
+
+    } else if (this.opts.auth) {
+      throw "Auth config does not work without publish";
+    }
+
+    this.dbEventsManager = new DBEventsManager(db, pgp);
+
+
+    this.writeDBSchema();
+
+    /* 5. Finish init and provide DBO object */
+    try {
+      if (this.destroyed) {
+        console.trace(1)
+      }
+      onReady({
+        dbo: this.dbo as any, 
+        db: this.db, 
+        tables: this.dboBuilder.tables,
+        reason
+      });
+    } catch (err) {
+      console.error("Prostgles: Error within onReady: \n", err)
+    }
+
+    this.loaded = true;
+    return {
+      db: this.dbo! as any,
+      _db: db,
+      pgp,
+      io: this.opts.io,
+      getTSSchema: this.getTSFileContent,
+      options: this.opts,
+      update: async (newOpts) => {
+
+        getKeys(newOpts).forEach(k => {
+          //@ts-ignore
+          this.opts[k] = newOpts[k];
+        });
+
+
+        if("fileTable" in newOpts){
+          await this.initFileTable();
+        }
+        if("restApi" in newOpts){
+          await this.initRestApi();
+        }
+        if("tableConfig" in newOpts){
+          await this.initTableConfig({ type: "prgl.update", newOpts });
+          await this.refreshDBO();
+        }
+        if("schema" in newOpts){
+          await this.initTableConfig({ type: "prgl.update", newOpts });
+          await this.refreshDBO();
+        }
+        if("auth" in newOpts){
+          await this.initAuthHandler();
+        }
+
+        if(isEmpty(newOpts)) return;
+
+        /** 
+         * Some of these changes require clients to reconnect 
+         * While others also affect the server and onReady should be called
+        */
+        if(getKeys(newOpts).every(updatedKey => clientOnlyUpdateKeys.includes(updatedKey as any))){
+          await this.setSocketEvents();
+        } else {
+          await this.init(onReady, { type: "prgl.update", newOpts });
+        }
+      },
+      restart: () => this.init(onReady, { type: "prgl.restart" }),
+      destroy: async () => {
+        console.log("destroying prgl instance")
+        this.destroyed = true;
+        if (this.opts.io) {
+          this.opts.io.on("connection", () => {
+            console.log("Socket connected to destroyed instance")
+          });
+
+          /** Try to close IO without stopping http server */
+          if(this.opts.io.sockets.constructor.name === "Namespace"){
+            for (const socket of this.opts.io.sockets.sockets.values()) {
+              socket._onclose("server shutting down");
+            }
+          }
+          if(this.opts.io.engine.constructor.name === 'Server'){
+            this.opts.io.engine.close();
+          } 
+        }
+        this.fileManager?.destroy();
+        this.dboBuilder?.destroy();
+        this.authHandler?.destroy();
+        await this.tableConfigurator?.destroy();
+        this.dbo = undefined;
+        this.db = undefined;
+        await db.$pool.end();
+        await sleep(1000);
+        return true;
+      }
+    };
+  } catch (e: any) {
+    console.trace(e)
+    throw "init issues: " + e.toString();
+  }
+}
+
+type GetDbConnectionArgs = Pick<ProstglesInitOptions, "DEBUG_MODE" | "onQuery" | "dbConnection" | "dbOptions" | "onNotice">;
+const getDbConnection = function({ dbConnection, onQuery, DEBUG_MODE, dbOptions, onNotice  }: GetDbConnectionArgs): { db: DB, pgp: PGP } {
+  
+  const onQueryOrError: undefined | ((error: any, ctx: pgPromise.IEventContext<pg.IClient>) => void) = !onQuery && !DEBUG_MODE? undefined : (error, ctx) => {
+    if (onQuery) {
+      onQuery(error, ctx);
+    } else if (DEBUG_MODE) {
+      if(error){
+        console.error(error, ctx);
+      } else {
+        console.log(ctx)
+      }
+    }
+  };
+  
+  const pgp: PGP = pgPromise({
+    ...(onQueryOrError ? {
+      query: ctx => onQueryOrError(undefined, ctx),
+      error: onQueryOrError
+    } : {}),
+    ...((onNotice || DEBUG_MODE) ? {
+      connect: function ({ client, useCount }) {
+        const isFresh = !useCount;
+        if (isFresh && !client.listeners('notice').length) {
+          client.on('notice', function (msg) {
+            if (onNotice) {
+              onNotice(msg, msg?.message);
+            } else {
+              console.log("notice: %j", msg?.message);
+            }
+          });
+        }
+        if (isFresh && !client.listeners('error').length) {
+          client.on('error', function (msg) {
+            if (onNotice) {
+              onNotice(msg, msg?.message);
+            } else {
+              console.log("error: %j", msg?.message);
+            }
+          });
+        }
+      },
+    } : {})
+  });
+  // pgp.pg.defaults.max = 70;
+
+  // /* Casts count/sum/max to bigint. Needs rework to remove casting "+count" and other issues; */
+  // pgp.pg.types.setTypeParser(20, BigInt);
+
+  /**
+   * Prevent timestamp casting to ensure we don't lose the microseconds.
+   * This is needed to ensure the filters work as expected for a given row
+   * 
+  register(1114, parseTimestamp) // timestamp without time zone
+  register(1184, parseTimestampTz) // timestamp with time zone
+   */
+  // pgp.pg.types.setTypeParser(1114, v => v); // timestamp without time zone
+  // pgp.pg.types.setTypeParser(1184, v => v); // timestamp with time zone
+  // pgp.pg.types.setTypeParser(1182, v => v); // date
+  pgp.pg.types.setTypeParser(pgp.pg.types.builtins.TIMESTAMP, v => v); // timestamp without time zone
+  pgp.pg.types.setTypeParser(pgp.pg.types.builtins.TIMESTAMPTZ, v => v); // timestamp with time zone
+  pgp.pg.types.setTypeParser(pgp.pg.types.builtins.DATE, v => v); // date
+  
+
+  if (dbOptions) {
+    Object.assign(pgp.pg.defaults, dbOptions);
+  }
+
+  return {
+    db: pgp(dbConnection),
+    pgp
+  };
+}

package/lib/onSocketConnected.ts

@@ -0,0 +1,102 @@
+import { AnyObject, CHANNELS } from "prostgles-types";
+import type { Prostgles, TABLE_METHODS } from "./Prostgles";
+import { PRGLIOSocket } from "./DboBuilder/DboBuilderTypes";
+import { runClientMethod, runClientRequest } from "./runClientRequest";
+import { getErrorAsObject } from "./DboBuilder/dboBuilderUtils";
+
+export async function onSocketConnected(this: Prostgles, socket: PRGLIOSocket) {
+  if (this.destroyed) {
+    console.log("Socket connected to destroyed instance");
+    socket.disconnect();
+    return
+  }
+  this.connectedSockets.push(socket);
+
+  try {
+    await this.opts.onLog?.({
+      type: "connect", 
+      sid: this.authHandler?.getSID({ socket }),
+      socketId: socket.id,
+      connectedSocketIds: this.connectedSockets.map(s => s.id)
+    });
+
+    if (!this.db || !this.dbo) throw new Error("db/dbo missing");
+    const { dbo, db } = this;
+
+    if (this.opts.onSocketConnect) {
+      try {
+        const getUser = async () => { return await this.authHandler?.getClientInfo({ socket }); }
+        await this.opts.onSocketConnect({ socket, dbo: dbo as any, db, getUser });
+      } catch(error) {
+        const connectionError = error instanceof Error? error.message : typeof error === "string"? error : JSON.stringify(error);
+        socket.emit(CHANNELS.CONNECTION, { connectionError });
+        socket.disconnect();
+        return;
+      }
+    }
+
+    socket.removeAllListeners(CHANNELS.DEFAULT)
+    socket.on(CHANNELS.DEFAULT, async (args: SocketRequestParams, cb = (..._callback: any[]) => { /* Empty */}) => {
+      runClientRequest.bind(this)({  ...args, type: "socket", socket })
+        .then(res => {
+          cb(null, res)
+        }).catch(err => {
+          cb(err);
+        });
+    });
+
+    socket.on("disconnect", () => {
+
+      this.dbEventsManager?.removeNotice(socket);
+      this.dbEventsManager?.removeNotify(undefined, socket);
+      this.connectedSockets = this.connectedSockets.filter(s => s.id !== socket.id);
+      this.dboBuilder.queryStreamer.onDisconnect(socket.id);
+      this.opts.onLog?.({ 
+        type: "disconnect", 
+        sid: this.authHandler?.getSID({ socket }),
+        socketId: socket.id,
+        connectedSocketIds: this.connectedSockets.map(s => s.id)
+      });
+
+      if (this.opts.onSocketDisconnect) {
+        const getUser = async () => { return await this.authHandler?.getClientInfo({ socket }); }
+        this.opts.onSocketDisconnect({ socket, dbo: dbo as any, db, getUser });
+      }
+    });
+
+    socket.removeAllListeners(CHANNELS.METHOD)
+    socket.on(CHANNELS.METHOD, async ({ method, params }: SocketMethodRequest, cb = (..._callback: any) => { /* Empty */ }) => {
+      runClientMethod.bind(this)({
+        type: "socket",
+        socket,
+        method,
+        params
+      }).then(res => {
+        cb(null, res)
+      }).catch(err => {
+        makeSocketError(cb, err)
+      });
+    });
+
+    this.pushSocketSchema(socket);
+  } catch (e) {
+    console.trace("setSocketEvents: ", e)
+  }
+}
+
+
+export function makeSocketError(cb: (err: AnyObject) => void, err: any) {
+  cb(getErrorAsObject(err));
+}
+
+type SocketRequestParams = {
+  tableName: string;
+  command: typeof TABLE_METHODS[number];
+  param1: any;
+  param2: any;
+  param3: any;
+}
+type SocketMethodRequest = {
+  method: string;
+  params: any;
+}

package/lib/runClientRequest.ts

@@ -0,0 +1,129 @@
+import { AnyObject, TableHandler, UserLike, getKeys, pickKeys } from "prostgles-types";
+import { ExpressReq } from "./Auth/AuthTypes";
+import { LocalParams, PRGLIOSocket } from "./DboBuilder/DboBuilder";
+import { parseFieldFilter } from "./DboBuilder/ViewHandler/parseFieldFilter";
+import { canRunSQL } from "./DboBuilder/runSQL";
+import { Prostgles } from "./Prostgles";
+ 
+type ReqInfo = {
+  type: "socket";
+  socket: PRGLIOSocket;
+  httpReq?: undefined;
+} | {
+  type: "http";
+  httpReq: ExpressReq;
+  socket?: undefined;
+}
+type ReqInfoClient = {
+  socket: PRGLIOSocket;
+} | {
+  httpReq: ExpressReq;
+}
+type Args = ReqInfo & {
+  tableName: string; 
+  command: string;
+  param1: any;
+  param2: any;
+  param3: any;
+};
+const SOCKET_ONLY_COMMANDS = ["subscribe", "subscribeOne", "sync"];
+
+const getReqInfoClient = (reqInfo: ReqInfo): ReqInfoClient => {
+  if(reqInfo.type === "socket"){
+    return { socket: reqInfo.socket };
+  }
+  return { httpReq: reqInfo.httpReq };
+}
+export const runClientRequest = async function(this: Prostgles, args: Args){
+  /* Channel name will only include client-sent params so we ignore table_rules enforced params */
+  if ((args.type === "socket" && !args.socket) || (args.type === "http" && !args.httpReq) || !this.authHandler || !this.publishParser || !this.dbo) {
+    throw "socket/httpReq or authhandler missing";
+  }
+
+  const { tableName, command, param1, param2, param3 } = args;
+
+  if(args.type !== "socket" && SOCKET_ONLY_COMMANDS.includes(command)){
+    throw "The following commands cannot be completed over a non-websocket connection: " + SOCKET_ONLY_COMMANDS;
+  }
+  const reqInfo = getReqInfoClient(args);
+  const clientInfo = await this.authHandler.getClientInfo(args);
+  const valid_table_command_rules = await this.publishParser.getValidatedRequestRule({ tableName, command, localParams: reqInfo }, clientInfo);
+  if (valid_table_command_rules) {
+    const sessionUser: UserLike | undefined = !clientInfo?.user? undefined : {
+      ...parseFieldFilter(clientInfo.sessionFields ?? [] as any, false, Object.keys(clientInfo.user)),
+      ...pickKeys(clientInfo.user, ["id", "type"]) as UserLike,
+    }
+    const localParams: LocalParams = { ...reqInfo, isRemoteRequest: { user: sessionUser } }
+    if(param3 && (param3 as LocalParams).returnQuery){
+      const isAllowed = await canRunSQL(this, localParams);
+      if(isAllowed){
+        localParams.returnQuery = (param3 as LocalParams).returnQuery;
+      } else {
+        throw "Must be allowed to run sql to use returnQuery";
+      }
+    }
+    const tableHandler = this.dbo[tableName];
+    if(!tableHandler || !tableHandler.column_names) throw `Invalid tableName ${tableName} provided`;
+    const method = tableHandler[command as keyof TableHandler];
+    if(!method) throw `Invalid command ${command} provided`;
+    //@ts-ignore
+    return this.dbo[tableName][command](param1, param2, param3, valid_table_command_rules, localParams);
+  } else {
+    throw `Invalid OR disallowed request: ${tableName}.${command} `;
+  }
+}
+
+export const clientCanRunSqlRequest = async function(this: Prostgles, args: ReqInfo){
+  const reqInfo = getReqInfoClient(args);
+  if(!this.opts.publishRawSQL || typeof this.opts.publishRawSQL !== "function"){
+    return { allowed: false, reqInfo }
+  } 
+  const canRunSQL = async () => {
+    if(!this.authHandler){
+      throw "authHandler missing";
+    } 
+    const publishParams = await this.publishParser?.getPublishParams(reqInfo);
+    const res = await this.opts.publishRawSQL?.(publishParams as any);
+    return Boolean(res && typeof res === "boolean" || res === "*");
+  }
+
+  const allowed = await canRunSQL();
+  return { allowed, reqInfo };
+}
+
+type ArgsSql = ReqInfo & {
+  query: string;
+  args?: AnyObject | any[];
+  options?: any;
+}
+export const runClientSqlRequest = async function(this: Prostgles, params: ArgsSql){
+  const { allowed, reqInfo } = await clientCanRunSqlRequest.bind(this)(params);
+  if(!allowed){
+    throw "Not allowed to execute sql";
+  }
+  if (!this.dbo?.sql) throw "Internal error: sql handler missing";
+  const { query, args, options } = params;
+  return this.dbo.sql(query, args, options, reqInfo);
+}
+
+
+
+type ArgsMethod = ReqInfo & {
+  method: string;
+  params?: any[]
+}
+export const runClientMethod = async function(this: Prostgles, reqArgs: ArgsMethod){
+
+  const reqInfo = getReqInfoClient(reqArgs);
+  const { method, params = [] } = reqArgs;
+  const methods = await this.publishParser?.getAllowedMethods(reqInfo);
+
+  if (!methods || !methods[method]) {
+    throw ("Disallowed/missing method " + JSON.stringify(method));
+  } 
+
+  const methodDef = methods[method]!;
+  const onRun = (typeof methodDef === "function" || typeof (methodDef as any).then === "function")? (methodDef as (...args: any) => Promise<void>) : methodDef.run;
+  const res = await onRun(...params);
+  return res;  
+}

package/lib/shortestPath.ts

@@ -0,0 +1,122 @@
+import { AnyObject } from "prostgles-types";
+
+const shortestDistanceNode = (distances: AnyObject, visited: AnyObject) => {
+	let shortest = null;
+
+	for (const node in distances) {
+		const currentIsShortest =
+			shortest === null || distances[node] < distances[shortest];
+		if (currentIsShortest && !visited.includes(node)) {
+			shortest = node;
+		}
+	}
+	return shortest;
+};
+export type Graph = {
+	[key: string]: { [key: string]: number }
+};
+
+export const findShortestPath = (graph: Graph, startNode: string, endNode: string): { distance: number, path: string[] } => {
+	// establish object for recording distances from the start node
+	let distances: AnyObject = {};
+	distances[endNode] = "Infinity";
+	distances = Object.assign(distances, graph[startNode]);
+
+	// track paths
+	const parents: AnyObject = { endNode: null };
+	for (const child in graph[startNode]) {
+		parents[child] = startNode;
+	}
+
+	// track nodes that have already been visited
+	const visited: AnyObject = [];
+
+	// find the nearest node
+	let node = shortestDistanceNode(distances, visited);
+
+	// for that node
+	while (node) {
+		// find its distance from the start node & its child nodes
+		const distance = distances[node];
+		const children = graph[node];
+		// for each of those child nodes
+		for (const child in children) {
+			// make sure each child node is not the start node
+			if (String(child) === String(startNode)) {
+				continue;
+			} else {
+				// save the distance from the start node to the child node
+				const newdistance = distance + children[child];
+				// if there's no recorded distance from the start node to the child node in the distances object
+				// or if the recorded distance is shorter than the previously stored distance from the start node to the child node
+				// save the distance to the object
+				// record the path
+				if (!distances[child] || distances[child] > newdistance) {
+					distances[child] = newdistance;
+					parents[child] = node;
+				}
+			}
+		}
+		// move the node to the visited set
+		visited.push(node);
+		// move to the nearest neighbor node
+		node = shortestDistanceNode(distances, visited);
+	}
+
+	// using the stored paths from start node to end node
+	// record the shortest path
+	const shortestPath = [endNode];
+	let parent = parents[endNode];
+	while (parent) {
+		shortestPath.push(parent);
+		parent = parents[parent];
+	}
+	shortestPath.reverse();
+
+	// return the shortest path from start node to end node & its distance
+	const results = {
+		distance: distances[endNode],
+		path: shortestPath,
+	};
+
+	return results;
+};
+
+/* Usage: 
+
+const graph = {
+	start: { A: 5, B: 2 },
+	A: { start: 1, C: 4, D: 2 },
+	B: { A: 8, D: 7 },
+	C: { D: 6, end: 3 },
+	D: { end: 1 },
+	end: {},
+};
+
+findShortestPath(graph, 'start', 'end');
+
+{
+  "distance": 8,
+  "path": [
+    "start",
+    "A",
+    "D",
+    "end"
+  ]
+}
+
+
+
+
+// The graph is unidirectional
+
+const graph = {
+	start: { A: 5 },
+	end: { A: 1 },
+	A: { start: 1, end: 1 },
+};
+
+findShortestPath(graph, 'start', 'end');
+
+
+*/