prostgles-server 4.2.159 → 4.2.161

package/lib/DboBuilder/prepareShortestJoinPaths.ts

@@ -0,0 +1,186 @@
+import { DboBuilder } from "../DboBuilder/DboBuilder";
+import { JOIN_TYPES, Join } from "../ProstglesTypes";
+import { Graph, findShortestPath } from "../shortestPath";
+import { TableSchema } from "./DboBuilderTypes";
+import { JoinPaths } from "./ViewHandler/ViewHandler";
+
+type Result = {
+  joinGraph?: Graph | undefined;
+  joins: Join[];
+  shortestJoinPaths: JoinPaths;
+}
+export async function prepareShortestJoinPaths(dboBuilder: DboBuilder): Promise<Result> {
+
+  if (dboBuilder.prostgles.opts.joins) {
+
+    let joinConfig = await dboBuilder.prostgles.opts.joins;
+    if (!dboBuilder.tablesOrViews) {
+      throw new Error("Could not create join config. this.tablesOrViews missing");
+    }
+
+    const inferredJoins = await getInferredJoins2(dboBuilder.tablesOrViews);
+    if (joinConfig === "inferred") {
+      joinConfig = inferredJoins
+      /* If joins are specified then include inferred joins except the explicit tables */
+    } else if (Array.isArray(joinConfig)) {
+      const joinTables = joinConfig.map(j => j.tables).flat();
+      joinConfig = joinConfig.concat(inferredJoins.filter(j => !j.tables.find(t => joinTables.includes(t))))
+    } else if (joinConfig) {
+      throw new Error("Unexpected joins init param. Expecting 'inferred' OR joinConfig but got: " + JSON.stringify(joinConfig))
+    }
+    const joins = JSON.parse(JSON.stringify(joinConfig)) as Join[];
+
+    // Validate joins
+    try {
+      const tovNames = dboBuilder.tablesOrViews!.map(t => t.name);
+
+      // 2 find incorrect tables
+      const missing = joins.flatMap(j => j.tables).find(t => !tovNames.includes(t));
+      if (missing) {
+        throw "Table not found: " + missing;
+      }
+
+      // 3 find incorrect fields
+      joins.map(({ tables, on }) => {
+        const t1 = tables[0],
+          t2 = tables[1];
+        on.map(cond => {
+
+          const f1s = Object.keys(cond),
+            f2s = Object.values(cond);
+          [[t1, f1s], [t2, f2s]].map(v => {
+            const t = <string>v[0],
+              f = <string[]>v[1];
+
+            const tov = dboBuilder.tablesOrViews!.find(_t => _t.name === t);
+            if (!tov) throw "Table not found: " + t;
+            const m1 = f.filter(k => !tov!.columns.map(c => c.name).includes(k))
+            if (m1 && m1.length) {
+              throw `Table ${t}(${tov.columns.map(c => c.name).join()}) has no fields named: ${m1.join()}`;
+            }
+          });
+        })
+      });
+
+      // 4 find incorrect/missing join types
+      const expected_types = " \n\n-> Expecting: " + JOIN_TYPES.map(t => JSON.stringify(t)).join(` | `)
+      const mt = joins.find(j => !j.type);
+      if (mt) throw "Join type missing for: " + JSON.stringify(mt, null, 2) + expected_types;
+
+      const it = joins.find(j => !JOIN_TYPES.includes(j.type));
+      if (it) throw "Incorrect join type for: " + JSON.stringify(it, null, 2) + expected_types;
+
+    } catch (e) {
+      const errMsg = ((joinConfig as any) === "inferred"? "INFERRED " : "") + "JOINS VALIDATION ERROR \n-> " + e;
+      throw errMsg;
+    }
+
+    // Make joins graph
+    const joinGraph: Graph = {};
+    joins.forEach(({ tables }) => {
+      const _t = tables.slice().sort(),
+        t1 = _t[0]!,
+        t2 = _t[1]!;
+
+      if (t1 === t2) return;
+
+      joinGraph![t1] ??= {};
+      joinGraph![t1]![t2] = 1;
+
+      joinGraph![t2] ??= {};
+      joinGraph![t2]![t1] = 1;
+    });
+    const tables = Array.from(new Set(joins.flatMap(t => t.tables)));
+    const shortestJoinPaths: JoinPaths = [];
+    tables.forEach((t1, i1) => {
+      tables.forEach((t2, i2) => {
+
+        /** Prevent recursion */
+        if (
+          t1 === t2 ||
+          shortestJoinPaths.some(jp => {
+            if (arrayValuesMatch([jp.t1, jp.t2], [t1, t2])) {
+              const spath = findShortestPath(joinGraph, t1, t2);
+              if (spath && arrayValuesMatch(spath.path, jp.path)) {
+                return true;
+              }
+            }
+          })
+        ) {
+          return;
+        }
+
+        const spath = findShortestPath(joinGraph, t1, t2);
+        if (!(spath && spath.distance < Infinity)) return;
+
+        const existing1 = shortestJoinPaths.find(j => j.t1 === t1 && j.t2 === t2)
+        if (!existing1) {
+          shortestJoinPaths.push({ t1, t2, path: spath.path.slice() });
+        }
+
+        const existing2 = shortestJoinPaths.find(j => j.t2 === t1 && j.t1 === t2);
+        if (!existing2) {
+          shortestJoinPaths.push({ t1: t2, t2: t1, path: spath.path.slice().reverse() });
+        }
+      });
+    });
+    return {
+      joins, shortestJoinPaths, joinGraph
+    }
+  }
+
+  return {
+    joins: [],
+    shortestJoinPaths: [],
+  };
+}
+
+
+const arrayValuesMatch = <T>(arr1: T[], arr2: T[]): boolean => {
+  return arr1.slice().sort().join() === arr2.slice().sort().join()
+}
+
+async function getInferredJoins2(schema: TableSchema[]): Promise<Join[]> {
+  const joins: Join[] = [];
+  const upsertJoin = (t1: string, t2: string, cols: { col1: string; col2: string }[], type: Join["type"]) => {
+    const existingIdx = joins.findIndex(j => arrayValuesMatch(j.tables.slice(0), [t1, t2]));
+    const existing = joins[existingIdx];
+    const normalCond = cols.reduce((a, v) => ({ ...a, [v.col1]: v.col2 }), {});
+    const revertedCond = cols.reduce((a, v) => ({ ...a, [v.col2]: v.col1 }), {});
+    if (existing) {
+      const isLTR = existing.tables[0] === t1
+      const cond = isLTR ? normalCond : revertedCond;
+
+      /** At some point we should add relationship type to EACH JOIN CONDITION GROUP */
+      // const fixedType = isLTR? type : type.split("").reverse().join("") as Join["type"];
+
+      /** Avoid duplicates */
+      if (!existing.on.some(_cond => JSON.stringify(_cond) === JSON.stringify(cond))) {
+        existing.on.push(cond);
+        joins[existingIdx] = existing;
+      }
+    } else {
+      joins.push({
+        tables: [t1, t2],
+        on: [normalCond],
+        type
+      })
+    }
+  }
+  schema.map(tov => {
+    tov.columns.map(col => {
+      if (col.references) {
+        col.references.forEach(r => {
+          const joinCols = r.cols.map((c, i) => ({ col1: c, col2: r.fcols[i]! }));
+          let type: Join["type"] = "one-many";
+          const ftablePkeys = schema.find(_tov => _tov.name === r.ftable)?.columns.filter(fcol => fcol.is_pkey);
+          if (ftablePkeys?.length && ftablePkeys.every(fkey => r.fcols.includes(fkey.name))) {
+            type = "one-one";
+          }
+          upsertJoin(tov.name, r.ftable, joinCols, type)
+        })
+      }
+    })
+  })
+  return joins;
+}

package/lib/DboBuilder/runSQL.ts

@@ -0,0 +1,182 @@
+import pgPromise, { ParameterizedQuery as PQ, ParameterizedQuery } from 'pg-promise';
+import pg from "pg-promise/typescript/pg-subset";
+import { AnyObject, SQLOptions, SQLResult, SQLResultInfo } from "prostgles-types";
+import { DB, Prostgles } from "../Prostgles";
+import { DboBuilder, LocalParams, pgp, postgresToTsType } from "./DboBuilder";
+
+
+export async function runSQL(this: DboBuilder, queryWithoutRLS: string, args: undefined | AnyObject | any[], options: SQLOptions | undefined, localParams?: LocalParams) {
+  const queryWithRLS = queryWithoutRLS;
+  if(queryWithRLS?.replace(/\s\s+/g, ' ').toLowerCase().includes("create extension pg_stat_statements")){
+    const { shared_preload_libraries } = await this.db.oneOrNone('SHOW shared_preload_libraries');
+    if(!(shared_preload_libraries || "").includes("pg_stat_statements")){
+      throw "This query will crash the server (pg_stat_statements must be loaded via shared_preload_libraries). Need to: \n ALTER SYSTEM SET shared_preload_libraries = 'pg_stat_statements' \n" +
+      " AND restart server: \n    (linux) sudo service postgresql restart\n   (mac) brew services restart postgres\n "
+    }
+  }
+
+  await this.cacheDBTypes();
+
+  if (!(await canRunSQL(this.prostgles, localParams))) {
+    throw "Not allowed to run SQL";
+  }
+
+  const { returnType, allowListen, hasParams = true }: SQLOptions = options || ({} as SQLOptions);
+  const { socket } = localParams || {};
+
+
+  const db = localParams?.tx?.t || this.db;
+  if (returnType === "stream") {
+    if(localParams?.tx) throw "Cannot use stream with localParams transaction";
+    if (!socket) throw "Only allowed with client socket";
+    const streamInfo = await this.queryStreamer.create({ socket, query: pgp.as.format(queryWithRLS, args), options });
+    return streamInfo;
+
+  } else if (returnType === "noticeSubscription") {
+    if (!socket) throw "Only allowed with client socket"
+    return await this.prostgles.dbEventsManager?.addNotice(socket);
+
+  } else if (returnType === "statement") {
+    try {
+      return pgp.as.format(queryWithoutRLS, args);
+    } catch (err) {
+      throw (err as any).toString();
+    }
+    
+  }
+  
+  if (!db) {
+    throw "db is missing"
+  }
+
+  let finalQuery: string | ParameterizedQuery = queryWithRLS + "";
+  const isNotListenOrNotify = (returnType === "arrayMode") && !["listen ", "notify "].find(c => queryWithoutRLS.toLowerCase().trim().startsWith(c))
+  if (isNotListenOrNotify) {
+    finalQuery = new PQ({ 
+      rowMode: "array",
+      text: hasParams ? pgp.as.format(queryWithRLS, args) : queryWithRLS, 
+    });
+  }
+
+  const params = hasParams ? args : undefined;
+  let queryResult: pgPromise.IResultExt<AnyObject> | undefined;
+  
+  if(returnType === "default-with-rollback"){
+    const ROLLBACK = "Rollback";
+    await db.tx(async t => {
+      queryResult = await t.result<AnyObject>(finalQuery, params);
+      /** Rollback */
+      return Promise.reject(new Error(ROLLBACK));
+    }).catch(e => {
+      if(!(e instanceof Error && e.message === ROLLBACK)) throw e;
+    });
+  }  else {
+    queryResult = await db.result<AnyObject>(finalQuery, params);
+  }
+  if(!queryResult) throw "No query result";
+  const { fields, rows } = queryResult;
+
+  const listenHandlers = await onSQLResult.bind(this)(queryWithoutRLS, queryResult, allowListen, localParams);
+  if(listenHandlers) {
+    return listenHandlers;
+  }
+
+  if (returnType === "rows") {
+    return rows;
+
+  } else if (returnType === "row") {
+    return rows[0];
+
+  } else if (returnType === "value") {
+    return Object.values(rows?.[0] ?? {})?.[0];
+
+  } else if (returnType === "values") {
+    return rows.map(r => Object.values(r ?? {})[0]);
+
+  } else {
+
+    const qres: SQLResult<typeof returnType> = {
+      duration: 0,
+      ...queryResult,
+      fields: getDetailedFieldInfo.bind(this)(fields),
+    };
+    return qres;
+  }
+}
+
+const onSQLResult = async function(this: DboBuilder, queryWithoutRLS: string, { command }: Omit<SQLResultInfo, "duration">, allowListen: boolean | undefined, localParams?: LocalParams) {
+
+  this.prostgles.schemaWatch?.onSchemaChangeFallback?.({ command, query: queryWithoutRLS });
+
+  if (command === "LISTEN") {
+    const { socket } = localParams || {};
+    if (!allowListen) throw new Error(`Your query contains a LISTEN command. Set { allowListen: true } to get subscription hooks. Or ignore this message`)
+    if (!socket) throw "Only allowed with client socket"
+    return await this.prostgles.dbEventsManager?.addNotify(queryWithoutRLS, socket);
+  }
+}
+
+export async function cacheDBTypes(this: DboBuilder, force = false) {
+  if(force){
+    this.DATA_TYPES = undefined;
+    this.USER_TABLES = undefined;
+    this.USER_TABLE_COLUMNS = undefined;
+  }
+  this.DATA_TYPES ??= await this.db.any("SELECT oid, typname FROM pg_type") ?? [];
+  this.USER_TABLES ??= await this.db.any(`
+    SELECT 
+      relid, 
+      relname, 
+      schemaname, 
+      array_to_json(array_agg(c.column_name) FILTER (WHERE c.column_name IS NOT NULL)) as pkey_columns
+    FROM pg_catalog.pg_statio_user_tables t
+    LEFT JOIN (
+      SELECT a.attname as column_name, i.indrelid as table_oid
+      FROM   pg_index i
+      JOIN   pg_attribute a ON a.attrelid = i.indrelid
+        AND a.attnum = ANY(i.indkey)
+      WHERE i.indisprimary
+    ) c
+    ON t.relid = c.table_oid
+    GROUP BY relid, relname, schemaname
+  `) ?? [];
+  this.USER_TABLE_COLUMNS ??= await this.db.any(`
+    SELECT t.relid, t.schemaname,t.relname, c.column_name, c.udt_name, c.ordinal_position
+    FROM information_schema.columns c
+    INNER JOIN pg_catalog.pg_statio_user_tables t
+    ON  c.table_schema = t.schemaname AND c.table_name = t.relname 
+  `);
+}
+
+export function getDetailedFieldInfo(this: DboBuilder, fields: pg.IColumn[]) {
+  return fields?.map(f => {
+    const dataType = this.DATA_TYPES!.find(dt => +dt.oid === +f.dataTypeID)?.typname ?? "text",
+      table = this.USER_TABLES!.find(t => +t.relid === +f.tableID),
+      column = this.USER_TABLE_COLUMNS!.find(c => +c.relid === +f.tableID && c.ordinal_position === f.columnID),
+      tsDataType = postgresToTsType(dataType);
+
+    return {
+      ...f,
+      tsDataType,
+      dataType,
+      udt_name: dataType,
+      tableName: table?.relname,
+      tableSchema: table?.schemaname,
+      columnName: column?.column_name
+    }
+  }) ?? [];
+}
+
+export const canRunSQL = async (prostgles: Prostgles, localParams?: LocalParams): Promise<boolean> => {
+  if (!localParams?.socket || !localParams?.httpReq) return true;
+
+  const { socket } = localParams;
+  const publishParams = await prostgles.publishParser!.getPublishParams({ socket });
+  //@ts-ignore
+  const res = await prostgles.opts.publishRawSQL?.(publishParams);
+  return Boolean(res && typeof res === "boolean" || res === "*");
+}
+
+export const canCreateTables = async (db: DB): Promise<boolean> => {
+  return db.any(`SELECT has_database_privilege(current_database(), 'create') as yes`).then(rows => rows?.[0].yes === true)
+}

package/lib/DboBuilder/runTransaction.ts

@@ -0,0 +1,50 @@
+import * as pg from "pg";
+const { Client } = pg;
+
+export type PGTX = ((query: pg.Client["query"]) => Promise<void>);
+export type RunTransactionOpts = {
+  begin?: boolean;
+  onSuccess: "COMMIT" | "ROLLBACK" | undefined;
+}
+export const runClientTransaction = async (handler: PGTX, { onSuccess, begin = true }: RunTransactionOpts, dbConn: pg.ClientConfig) => {
+  const client = new Client(dbConn)
+  try {
+    if(begin){
+      await client.query('BEGIN')
+    }
+    await handler(client.query);
+    if(onSuccess){
+      await client.query(onSuccess);
+    }
+  } catch (e) {
+    await client.query('ROLLBACK');
+    throw e
+  } finally {
+    client.end();
+  }
+}
+
+export const runClientTransactionStatement = async (statement: string, dbConn: pg.ClientConfig) => {
+  return runClientTransaction(async (query) => {
+    await query(statement);
+  }, { onSuccess: undefined, begin: false }, dbConn);
+}
+
+// let pool: pg.Pool | undefined;
+// export const runPoolTransaction = async (handler: PGTX, { onSuccess, begin = true }: RunTransactionOpts, dbConn: pg.ClientConfig) => {
+//   pool ??= new Pool(dbConn);
+//   const client = await pool.connect()
+
+//   try {
+//     await client.query('BEGIN')
+//     await handler(client.query);
+//     if(onSuccess){
+//       await client.query(onSuccess);
+//     }
+//   } catch (e) {
+//     await client.query('ROLLBACK');
+//     throw e
+//   } finally {
+//     client.release();
+//   }
+// }