prostgles-server 4.2.160 → 4.2.162

package/lib/DboBuilder/QueryBuilder/QueryBuilder.ts

@@ -0,0 +1,288 @@
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Stefan L. All rights reserved.
+ *  Licensed under the MIT License. See LICENSE in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { asName, ColumnInfo, getKeys, isEmpty, isObject, JoinSelect, PG_COLUMN_UDT_DATA_TYPE, Select, ValidatedColumnInfo } from "prostgles-types";
+import { isPlainObject, postgresToTsType, SortItem } from "../DboBuilder";
+
+import { ParsedJoinPath } from "../ViewHandler/parseJoinPath";
+import { ViewHandler } from "../ViewHandler/ViewHandler";
+import { COMPUTED_FIELDS, FieldSpec, FunctionSpec, parseFunction } from "./Functions";
+
+export type SelectItem = {
+  getFields: (args?: any[]) => string[] | "*";
+  getQuery: (tableAlias?: string) => string;
+  columnPGDataType?: string;
+  column_udt_type?: PG_COLUMN_UDT_DATA_TYPE;
+  tsDataType?: ValidatedColumnInfo["tsDataType"]
+  alias: string;
+  selected: boolean;
+} & ({
+  type: "column";
+  columnName: string;
+} | {
+  type: "function" | "aggregation" | "joinedColumn" | "computed";
+  columnName?: undefined;
+});
+export type SelectItemValidated = SelectItem & { fields: string[]; }
+export type WhereOptions = Awaited<ReturnType<ViewHandler["prepareWhere"]>>;
+export type NewQueryRoot = {
+  /**
+   * All fields from the table will be in nested SELECT and GROUP BY to allow order/filter by fields not in select 
+   */
+  allFields: string[];
+
+  /**
+   * Contains user selection and all the allowed columns. Allowed columns not selected are marked with  selected: false
+   */
+  select: SelectItem[];
+
+  table: string;
+  where: string;
+  whereOpts: WhereOptions;
+  orderByItems: SortItem[];
+  having: string;
+  limit: number | null;
+  offset: number;
+  isLeftJoin: boolean;
+  tableAlias?: string;
+};
+
+export type NewQueryJoin = (NewQuery & {
+  joinPath: ParsedJoinPath[];
+  joinAlias: string;
+});
+export type NewQuery = NewQueryRoot & {
+  joins?: NewQueryJoin[];
+}
+
+export const asNameAlias = (field: string, tableAlias?: string) => {
+  const result = asName(field);
+  if(tableAlias) return asName(tableAlias) + "." + result;
+  return result;
+}
+
+export const parseFunctionObject = (funcData: any): { funcName: string; args: any[] } => {
+  const makeErr = (msg: string) => `Function not specified correctly. Expecting { $funcName: ["columnName" | <value>, ...args] } object but got: ${JSON.stringify(funcData)} \n ${msg}`
+  if(!isObject(funcData)) throw makeErr("");
+  const keys = getKeys(funcData);
+  if(keys.length !== 1) throw makeErr("");
+  const funcName = keys[0]!;
+  const args = funcData[funcName];
+  if(!args || !Array.isArray(args)){
+    throw makeErr("Arguments missing or invalid");
+  }
+
+  return { funcName, args };
+}
+
+
+export class SelectItemBuilder {
+
+  select: SelectItemValidated[] = [];
+  private allFields: string[];
+
+  private allowedFields: string[];
+  private allowedOrderByFields: string[];
+  private computedFields: FieldSpec[];
+  private functions: FunctionSpec[];
+  private allowedFieldsIncludingComputed: string[];
+  private isView: boolean;
+  private columns: ColumnInfo[];
+
+  constructor(params: { allowedFields: string[]; allowedOrderByFields: string[]; computedFields: FieldSpec[]; functions: FunctionSpec[]; allFields: string[]; isView: boolean; columns: ColumnInfo[]; }){
+    this.allFields = params.allFields;
+    this.allowedFields = params.allowedFields;
+    this.allowedOrderByFields = params.allowedOrderByFields;
+    this.computedFields = params.computedFields;
+    this.isView = params.isView;
+    this.functions = params.functions;
+    this.columns = params.columns;
+    this.allowedFieldsIncludingComputed = this.allowedFields.concat(this.computedFields? this.computedFields.map(cf => cf.name) : []);
+    if(!this.allowedFields.length){
+      if(!this.columns.length){
+        throw "This view/table has no columns. Cannot select anything";
+      }
+      throw "allowedFields empty/missing";
+    }
+
+    /* Check for conflicting computed column names */
+    const conflictingCol = this.allFields.find(fieldName => this.computedFields.find(cf => cf.name === fieldName));
+    if(conflictingCol){
+      throw "INTERNAL ERROR: Cannot have duplicate column names ( " + conflictingCol + " ). One or more computed column names are colliding with table columns ones";
+    }
+  }
+
+  private checkField = (f: string, isSelected: boolean) => {
+    const allowedSelectedFields = this.allowedFieldsIncludingComputed;
+    const allowedNonSelectedFields = [...this.allowedFieldsIncludingComputed, ...this.allowedOrderByFields];
+
+    /** Not selected items can be part of the orderBy fields */
+    const allowedFields = isSelected? allowedSelectedFields : allowedNonSelectedFields;
+    if(!allowedFields.includes(f)){
+      throw "Field " + f + " is invalid or dissallowed. \nAllowed fields: " + allowedFields.join(", ");
+    }
+    return f;
+  }
+
+  private addItem = (item: SelectItem) => {
+    let fields = item.getFields();
+    // console.trace(fields)
+    if(fields === "*") fields = this.allowedFields.slice(0);
+    fields.map(f => this.checkField(f, item.selected));
+
+    if(this.select.find(s => s.alias === item.alias)){ 
+      throw `Cannot specify duplicate columns ( ${item.alias} ). Perhaps you're using "*" with column names?`;
+    }
+    this.select.push({ ...item, fields });
+  }
+
+  private addFunction = (func: FunctionSpec | string, args: any[], alias: string) => {
+    const funcDef = parseFunction({
+      func, args, functions: this.functions,
+      allowedFields: this.allowedFieldsIncludingComputed,
+    });
+
+    this.addItem({
+      type: funcDef.type,
+      alias,
+      getFields: () => funcDef.getFields(args),
+      getQuery: (tableAlias?: string) => funcDef.getQuery({ allColumns: this.columns, allowedFields: this.allowedFields, args, tableAlias, 
+        ctidField: undefined,
+
+        /* CTID not available in AFTER trigger */
+        // ctidField: this.isView? undefined : "ctid" 
+      }),
+      selected: true
+    });
+  }
+
+  addColumn = (fieldName: string, selected: boolean) => {
+  
+    /* Check if computed col */
+    if(selected){
+      const compCol = COMPUTED_FIELDS.find(cf => cf.name === fieldName);
+      if(compCol && !this.select.find(s => s.alias === fieldName)){
+        const cf: FunctionSpec = { 
+          ...compCol,
+          type: "computed",
+          numArgs: 0,
+          singleColArg: false,
+          getFields: (_args: any[]) => [] 
+        }
+        this.addFunction(cf, [], compCol.name)
+        return;
+      }
+    }
+
+    const colDef = this.columns.find(c => c.name === fieldName);
+    const alias = selected? fieldName : ("not_selected_" + fieldName);
+    this.addItem({
+      type: "column",
+      columnName: fieldName,
+      columnPGDataType: colDef?.data_type,
+      column_udt_type: colDef?.udt_name,
+      tsDataType: colDef && postgresToTsType(colDef.udt_name),
+      alias,
+      getQuery: (tableAlias) => asNameAlias(fieldName, tableAlias),
+      getFields: () => [fieldName],
+      selected
+    });
+  }
+
+  parseUserSelect = async (userSelect: Select, joinParse?: (key: string, val: JoinSelect, throwErr: (msg: string) => any) => any) => {
+
+    /* [col1, col2, col3] */
+    if(Array.isArray(userSelect)){
+      if(userSelect.find(key => typeof key !== "string")) throw "Invalid array select. Expecting an array of strings";
+  
+      userSelect.map(key => this.addColumn(key, true))
+  
+    /* Empty select */
+    } else if(userSelect === ""){ 
+      return [];
+      
+    } else if(userSelect === "*"){
+      this.allowedFields.map(key => this.addColumn(key, true) );
+
+    } else if(isPlainObject(userSelect) && !isEmpty(userSelect)){
+      const selectKeys = Object.keys(userSelect),
+        selectValues = Object.values(userSelect);
+  
+      /* Cannot include and exclude at the same time */
+      if(
+        selectValues.filter(v => [0, false].includes(v)).length 
+      ){
+        if(selectValues.filter(v => ![0, false].includes(v)).length ){
+          throw "\nCannot include and exclude fields at the same time";
+        }
+  
+        /* Exclude only */
+        this.allowedFields.filter(f => !selectKeys.includes(f)).map(key => this.addColumn(key, true) )
+          
+      } else {
+        await Promise.all(selectKeys.map(async key => {
+          const val: any = userSelect[key as keyof typeof userSelect],
+            throwErr = (extraErr = "") => {
+              console.trace(extraErr)
+              throw "Unexpected select -> " + JSON.stringify({ [key]: val }) + "\n" + extraErr;
+            };
+        
+          /* Included fields */
+          if([1, true].includes(val)){
+            if(key === "*"){
+              this.allowedFields.map(key => this.addColumn(key, true) )
+            } else {
+              this.addColumn(key, true);
+            }
+  
+          /* Aggs and functions */
+          } else if(typeof val === "string" || isObject(val)) {
+  
+            /* Function shorthand notation
+                { id: "$max" } === { id: { $max: ["id"] } } === SELECT MAX(id) AS id 
+            */  
+            if(
+              (typeof val === "string" && val !== "*") ||
+              isPlainObject(val) && Object.keys(val).length === 1 && Array.isArray(Object.values(val)[0])
+            ){
+
+              let funcName: string | undefined, args: any[] | undefined;
+              if(typeof val === "string") {
+                /* Shorthand notation -> it is expected that the key is the column name used as the only argument */
+                try {
+                  this.checkField(key, true)
+                } catch (err){
+                  throwErr(` Shorthand function notation error: the specifield column ( ${key} ) is invalid or dissallowed. \n Use correct column name or full aliased function notation, e.g.: -> { alias: { $func_name: ["column_name"] } } `)
+                }
+                funcName = val;
+                args = [key];
+
+              /** Function full notation { $funcName: ["colName", ...args] } */
+              } else {
+                ({ funcName, args } = parseFunctionObject(val));
+              }
+              
+              this.addFunction(funcName, args, key);
+  
+            /* Join */
+            } else {
+
+              if(!joinParse) {
+                throw "Joins dissalowed";
+              }
+              await joinParse(key, val as JoinSelect, throwErr);
+              
+            }
+  
+          } else throwErr();
+  
+        }));
+      }
+    } else throw "Unexpected select -> " + JSON.stringify(userSelect);
+  
+  }
+
+}

package/lib/DboBuilder/QueryBuilder/getJoinQuery.ts

@@ -0,0 +1,263 @@
+import { isDefined, asName } from "prostgles-types";
+import { ParsedJoinPath, parseJoinPath } from "../ViewHandler/parseJoinPath";
+import { NewQuery, NewQueryJoin, SelectItem, asNameAlias } from "./QueryBuilder";
+import { ROOT_TABLE_ALIAS, ROOT_TABLE_ROW_NUM_ID, indentLines } from "./getSelectQuery";
+import { ViewHandler } from "../ViewHandler/ViewHandler";
+import { getJoinOnCondition } from "../ViewHandler/getTableJoinQuery";
+import { prepareOrderByQuery } from "../DboBuilder";
+
+type Args = {
+  q1: NewQuery;
+  q2: NewQueryJoin;
+  selectParamsGroupBy: boolean;
+}
+
+/**
+ * Rename all join columns to prevent name clash
+ */
+export const getJoinCol = (colName: string) => {
+  const alias = asName("prgl_join_col__" + colName);
+  return {
+    alias,
+    rootSelect: `${asName(colName)} AS ${alias}`,
+  }
+}
+
+export const JSON_AGG_FIELD_NAME = "prostgles_json_agg_result_field";
+/**
+ * Used for LIMIT and for sorting
+ */
+export const NESTED_ROWID_FIELD_NAME = "prostgles_rowid_field";
+
+const getJoinTable = (tableName: string, pathIndex: number, isLastTableAlias: string | undefined) => {
+  const rawAlias = isLastTableAlias ?? `p${pathIndex}  ${tableName}`; 
+  return {
+    // name: asName(tableName), /** table names are already escaped */
+    name: tableName,
+    alias: asName(rawAlias),
+    rawAlias,
+  }
+}
+
+type GetJoinQueryResult = {
+  resultAlias: string;
+  // queryLines: string[]; 
+  firstJoinTableJoinFields: string[];
+  isOrJoin: boolean;
+  type: "cte";
+  joinLines: string[];
+  cteLines: string[];
+}
+
+/**
+  Returns join query. All inner join tables will be prefixed with path index unless it's the final target table which is aliased using the q2 tableAlias
+
+  LEFT JOIN (
+    SELECT [target table select + join fields]
+    FROM first_join/target_table
+    JOIN ..next_joins ON ...
+    JOIN target_table
+  ) target_table
+  ON ...condition
+ */
+export const getJoinQuery = (viewHandler: ViewHandler, { q1, q2 }: Args): GetJoinQueryResult => {
+  const paths = parseJoinPath({ 
+    rootTable: q1.table, 
+    rawPath: q2.joinPath, 
+    viewHandler: viewHandler, 
+    allowMultiOrJoin: true, 
+    addShortestJoinIfMissing: true, 
+  });
+
+  const targetTableAliasRaw = q2.tableAlias || q2.table;
+  const targetTableAlias = asName(targetTableAliasRaw);
+  
+  const firstJoinTablePath = paths[0]!;
+  const firstJoinTableJoinFields = firstJoinTablePath.on.flatMap(condObj => Object.entries(condObj).map(([source, target]) => target));
+  const { rootSelectItems, jsonAggLimit } = getNestedSelectFields({ 
+    q: q2, 
+    firstJoinTableAlias: getJoinTable(firstJoinTablePath.table, 0, paths.length === 1? targetTableAliasRaw : undefined).rawAlias, 
+    _joinFields: firstJoinTableJoinFields 
+  });
+
+  const joinType = q2.isLeftJoin? "LEFT" : "INNER";
+
+  const isOrJoin = firstJoinTablePath.on.length > 1;
+  const joinCondition = getJoinOnCondition({ 
+    on: firstJoinTablePath.on, 
+    leftAlias: asName(q1.tableAlias || q1.table), 
+    rightAlias: targetTableAlias, 
+    getRightColName: (col) => getJoinCol(col).alias
+  });
+
+  const joinFields = rootSelectItems.filter(s => s.isJoinCol).map(s => s.alias);
+  const selectedFields = rootSelectItems.filter(s => s.selected).map(s => asNameAlias(s.alias, targetTableAliasRaw));
+  const rootNestedSort = q1.orderByItems.filter(d => d.nested?.joinAlias === q2.joinAlias);
+  const jsonAggSort = prepareOrderByQuery(q2.orderByItems, targetTableAliasRaw).join(", ");
+  const jsonAgg = `json_agg((SELECT x FROM (SELECT ${selectedFields}) as x )${jsonAggSort}) ${jsonAggLimit} as ${JSON_AGG_FIELD_NAME}`;
+  
+  const { innerQuery } = getInnerJoinQuery({ paths, q1, q2, rootSelectItems, targetTableAliasRaw });
+
+  const requiredJoinFields = joinFields.map(field => getJoinCol(field).alias);
+  /**
+   * Used to prevent duplicates in case of OR filters
+   */
+  const rootTableIdField = `${ROOT_TABLE_ALIAS}.${ROOT_TABLE_ROW_NUM_ID}`;
+  const wrappingQuery = [
+    `SELECT `,
+      ...indentLines([
+        ...(isOrJoin? [rootTableIdField]: requiredJoinFields), 
+        jsonAgg, 
+        ...rootNestedSort.map(d => d.nested!.wrapperQuerySortItem)
+      ], { appendCommas: true }),
+    `FROM (`,
+    ...indentLines(innerQuery),
+    `) ${targetTableAlias}`,
+    ...(isOrJoin? [
+    `LEFT JOIN ${q1.table} ${ROOT_TABLE_ALIAS}`,
+    `ON ${joinCondition}`
+    ] : []),
+    `GROUP BY ${isOrJoin? rootTableIdField : requiredJoinFields}`,
+  ]; 
+  
+  /**
+   * This is done to prevent join cte names clashing with actual table names
+   */
+  const targetTableAliasTempRename = asName(`${targetTableAlias}_prostgles_join_temp_rename`)
+  const cteLines = [
+    `${targetTableAliasTempRename} AS (`,
+    ...indentLines(wrappingQuery),
+    `)`
+  ];
+
+  const joinLines = [
+    `${joinType} JOIN ( SELECT * FROM ${targetTableAliasTempRename} ) as ${targetTableAlias}`,
+    isOrJoin? 
+      `ON ${targetTableAlias}.${ROOT_TABLE_ROW_NUM_ID} = ${rootTableIdField}` :
+      `ON ${joinCondition}`
+  ];
+
+  return {
+    type: "cte",
+    resultAlias: JSON_AGG_FIELD_NAME,
+    // queryLines,
+    joinLines,
+    cteLines,
+    isOrJoin,
+    firstJoinTableJoinFields,
+  }
+}
+
+
+/**
+ * prepares the 
+ */
+const getInnerJoinQuery = ({ paths, q1, q2, targetTableAliasRaw, rootSelectItems }: {
+  paths: ParsedJoinPath[];
+  q1: NewQuery;
+  q2: NewQueryJoin;
+  targetTableAliasRaw: string;
+  rootSelectItems: SelectItemNested[];
+}) => {
+
+  const innerQuery = paths.flatMap((path, i) => {
+    
+    const isLast = i === paths.length - 1;
+    const targetQueryExtraQueries: string[] = [];
+
+    const prevTable = getJoinTable(!i? (q1.tableAlias? asName(q1.tableAlias) : q1.table) : paths[i-1]!.table, i-1, undefined);
+    
+    const table = getJoinTable(path.table, i, isLast? targetTableAliasRaw : undefined);
+
+    if(isLast){
+      if(q2.where){
+        targetQueryExtraQueries.push(q2.where);
+      }
+
+      /* If aggs exist need to set groupBy add joinFields into select */
+      const aggs = q2.select.filter(s => s.type === "aggregation")
+      if (aggs.length) {
+        const groupByFields = rootSelectItems.map((c, i) => (c.isJoinCol || c.selected && c.type !== "aggregation")? `${i+1}` : undefined ).filter(isDefined);
+        if(groupByFields.length){
+          targetQueryExtraQueries.push(`GROUP BY ${groupByFields}`)
+        }
+        if(q2.having){
+          targetQueryExtraQueries.push(`HAVING ${q2.having}`)
+        }
+      }
+    }
+
+    const isFirst = !i;
+    if(isFirst){
+      return [
+        `SELECT `,
+        `  /* Join fields + select */`, 
+        ...indentLines(rootSelectItems.map(s => s.query), { appendCommas: true }),
+        `FROM ${table.name} ${table.alias}`,
+        ...targetQueryExtraQueries
+      ]
+    }
+
+    return [
+      `INNER JOIN ${table.name} ${table.alias}`,
+      `ON ${getJoinOnCondition({ 
+        on: path.on, 
+        leftAlias: prevTable.alias, 
+        rightAlias: table.alias,
+      })}`,
+      ...targetQueryExtraQueries
+    ]
+  });
+
+  return { innerQuery }
+}
+
+
+type GetSelectFieldsArgs = {
+  q: NewQueryJoin;
+  firstJoinTableAlias: string;
+  _joinFields: string[];
+}
+
+export type SelectItemNested = SelectItem & { query: string; isJoinCol: boolean; };
+const getNestedSelectFields = ({ q, firstJoinTableAlias, _joinFields }: GetSelectFieldsArgs) => {
+  const targetTableAlias = (q.tableAlias || q.table);
+  
+  const requiredJoinFields = Array.from(new Set(_joinFields))
+  const selectedFields = q.select.filter(s => s.selected);
+  const rootSelectItems: SelectItemNested[] = selectedFields 
+    .map(s => ({ 
+      ...s,
+      isJoinCol: false,
+      query: s.getQuery(targetTableAlias) + " AS " + asName(s.alias)
+    }))
+    .concat(requiredJoinFields.map(f => ({
+      type: "column",
+      columnName: f,
+      alias: f,
+      getFields: () => [f],
+      getQuery: (tableAlias) => asNameAlias(f, tableAlias),
+      selected: false,
+      isJoinCol: true,
+      query: `${asName(firstJoinTableAlias)}.${getJoinCol(f).rootSelect}`,
+    })));
+  
+    const getQuery = (tableAlias?: string) => {
+      const partitionBy = `PARTITION BY ${requiredJoinFields.map(f => asNameAlias(f, tableAlias))}`;
+      return `ROW_NUMBER() OVER(${partitionBy}) AS ${NESTED_ROWID_FIELD_NAME}`
+    };
+    rootSelectItems.push({
+      type: "computed",
+      selected: false,
+      alias: NESTED_ROWID_FIELD_NAME,
+      getFields: () => [],
+      getQuery,
+      query: getQuery(firstJoinTableAlias),
+      isJoinCol: false,
+    })
+  
+  return { 
+    rootSelectItems, 
+    jsonAggLimit: q.limit? `FILTER (WHERE ${NESTED_ROWID_FIELD_NAME} <= ${q.limit})` : ""
+  };
+}