prostgles-server 4.2.159 → 4.2.161

package/lib/DboBuilder/TableHandler/delete.ts

@@ -0,0 +1,115 @@
+import pgPromise from "pg-promise";
+import { AnyObject, DeleteParams, FieldFilter } from "prostgles-types";
+import { DeleteRule, TableRule } from "../../PublishParser/PublishParser";
+import { Filter, LocalParams, getErrorAsObject, getSerializedClientErrorFromPGError, withUserRLS } from "../DboBuilder";
+import { runQueryReturnType } from "../ViewHandler/find";
+import { TableHandler } from "./TableHandler";
+import { onDeleteFromFileTable } from "./onDeleteFromFileTable";
+
+export async function _delete(this: TableHandler, filter?: Filter, params?: DeleteParams, param3_unused?: undefined, tableRules?: TableRule, localParams?: LocalParams): Promise<any> {
+  const start = Date.now();
+  try {
+    const { returning } = params || {};
+    filter = filter || {};
+    this.checkFilter(filter);
+
+    let forcedFilter: AnyObject | undefined = {},
+      filterFields: FieldFilter | undefined = "*",
+      returningFields: FieldFilter | undefined = "*",
+      validate: DeleteRule["validate"];
+
+    const { testRule = false, returnQuery = false } = localParams || {};
+    if (tableRules) {
+      if (!tableRules.delete) throw "delete rules missing";
+      forcedFilter = tableRules.delete.forcedFilter;
+      filterFields = tableRules.delete.filterFields;
+      returningFields = tableRules.delete.returningFields;
+      validate = tableRules.delete.validate;
+
+      if (!returningFields) returningFields = tableRules?.select?.fields;
+      if (!returningFields) returningFields = tableRules?.delete?.filterFields;
+
+      if (!filterFields) throw ` Invalid delete rule for ${this.name}. filterFields missing `;
+
+      /* Safely test publish rules */
+      if (testRule) {
+        await this.validateViewRules({ filterFields, returningFields, forcedFilter, rule: "delete" });
+        return true;
+      }
+    }
+
+    if (params) {
+      const good_paramsObj: Record<keyof DeleteParams, 1> = { returning: 1, returnType: 1 };
+      const good_params = Object.keys(good_paramsObj);
+      const bad_params = Object.keys(params).filter(k => !good_params.includes(k));
+      if (bad_params && bad_params.length) throw "Invalid params: " + bad_params.join(", ") + " \n Expecting: " + good_params.join(", ");
+    }
+
+    let queryType: keyof pgPromise.ITask<{}> = 'none';
+    let queryWithoutRLS = `DELETE FROM ${this.escapedName} `;
+    const filterOpts = (await this.prepareWhere({
+      select: undefined,
+      filter,
+      forcedFilter,
+      filterFields,
+      localParams,
+      tableRule: tableRules
+    }))
+    queryWithoutRLS += filterOpts.where;
+    if (validate) {
+      const _filter = filterOpts.filter;
+      await validate(_filter);
+    }
+
+    let returningQuery = "";
+    if (returning) {
+      queryType = "any";
+      if (!returningFields) {
+        throw "Returning dissallowed";
+      }
+      returningQuery = this.makeReturnQuery(await this.prepareReturning(returning, this.parseFieldFilter(returningFields)));
+      queryWithoutRLS += returningQuery
+    }
+
+    // TODO - delete orphaned files
+    // if(this.dboBuilder.prostgles.opts.fileTable?.referencedTables?.[this.name]?.referenceColumns){
+    //   if(!this.getFinalDBtx(localParams)){
+    //     const ACTION = "delete";
+    //     return this.dboBuilder.getTX(_dbtx => _dbtx[this.name]?.[ACTION]?.(filter, params, param3_unused, tableRules, localParams))
+    //   }
+    // }
+
+    const queryWithRLS = withUserRLS(localParams, queryWithoutRLS);
+    if (returnQuery) return queryWithRLS;
+
+    /**
+     * Delete file
+     */
+    if (this.is_media) {
+      const result = await onDeleteFromFileTable.bind(this)({ 
+        localParams, 
+        queryType, 
+        returningQuery: returnQuery? returnQuery : undefined,
+        filterOpts,
+      });
+      await this._log({ command: "delete", localParams, data: { filter, params }, duration: Date.now() - start });
+      return result;
+    }
+
+    const result = await runQueryReturnType({ 
+      queryWithoutRLS,
+      queryWithRLS,
+      newQuery: undefined, 
+      returnType: params?.returnType, 
+      handler: this, 
+      localParams
+    });
+    await this._log({ command: "delete", localParams, data: { filter, params }, duration: Date.now() - start });
+    return result;
+
+  } catch (e) {
+    await this._log({ command: "delete", localParams, data: { filter, params }, duration: Date.now() - start, error: getErrorAsObject(e) });
+    throw getSerializedClientErrorFromPGError(e, { type: "tableMethod", localParams, view: this });
+  }
+} 
+

package/lib/DboBuilder/TableHandler/insert.ts

@@ -0,0 +1,183 @@
+import { AnyObject, InsertParams, asName, isObject } from "prostgles-types";
+import { TableRule, ValidateRowBasic } from "../../PublishParser/PublishParser";
+import { LocalParams, getErrorAsObject, getSerializedClientErrorFromPGError, withUserRLS } from "../DboBuilder";
+import { insertNestedRecords } from "../insertNestedRecords";
+import { prepareNewData } from "./DataValidator";
+import { TableHandler } from "./TableHandler";
+import { insertTest } from "./insertTest";
+import { runInsertUpdateQuery } from "./runInsertUpdateQuery";
+
+export async function insert(this: TableHandler, rowOrRows: AnyObject | AnyObject[] = {}, insertParams?: InsertParams, param3_unused?: undefined, tableRules?: TableRule, localParams?: LocalParams): Promise<any | any[] | boolean> {
+  
+  const ACTION = "insert";
+  const start = Date.now();
+  try {
+
+    const { fixIssues = false } = insertParams || {};
+    const { returnQuery = false, nestedInsert } = localParams || {};
+ 
+    const finalDBtx = this.getFinalDBtx(localParams);
+    const rule = tableRules?.[ACTION];
+    const { postValidate, checkFilter, validate, allowedNestedInserts } = rule ?? {};
+
+    /** Post validate and checkFilter require a transaction dbo handler because they happen after the insert */
+    if(postValidate || checkFilter){
+      if(!finalDBtx){
+        return this.dboBuilder.getTX(_dbtx => _dbtx[this.name]?.[ACTION]?.(rowOrRows, insertParams, param3_unused, tableRules, localParams))
+      }
+    }
+
+    const { testOnly, fields, forcedData, returningFields } = await insertTest.bind(this)({ tableRules, localParams });
+    if(testOnly){
+      return true;
+    }
+
+    if(allowedNestedInserts){
+      if(!nestedInsert || !allowedNestedInserts.some(ai => ai.table === nestedInsert?.previousTable && ai.column === nestedInsert.referencingColumn)){
+        throw `Direct inserts not allowed. Only nested inserts from these tables: ${JSON.stringify(allowedNestedInserts)} `
+      }
+    }
+    validateInsertParams(insertParams);
+
+    const isMultiInsert = Array.isArray(rowOrRows);
+    const preValidatedRows = await Promise.all((isMultiInsert? rowOrRows : [rowOrRows]).map(async nonValidated => {
+      const { preValidate, validate } = tableRules?.insert ?? {};
+      const { tableConfigurator } = this.dboBuilder.prostgles;
+      if(!tableConfigurator) throw "tableConfigurator missing";
+      let row = await tableConfigurator.getPreInsertRow(this, { dbx: this.getFinalDbo(localParams), validate, localParams, row: nonValidated })
+      if (preValidate) {
+        if(!localParams) throw "localParams missing for insert preValidate";
+        row = await preValidate({ row, dbx: (this.tx?.dbTX || this.dboBuilder.dbo) as any, localParams });
+      }
+
+      return row;
+    }));
+    const preValidatedrowOrRows = isMultiInsert? preValidatedRows : preValidatedRows[0]!;
+
+    /**
+     * If media it will: upload file and continue insert
+     * If nested insert it will: make separate inserts and not continue main insert
+     */
+    const mediaOrNestedInsert = await insertNestedRecords.bind(this)({ data: preValidatedrowOrRows, param2: insertParams, tableRules, localParams });
+    const { data, insertResult } = mediaOrNestedInsert;
+    if ("insertResult" in mediaOrNestedInsert) {
+      return insertResult;
+    }
+
+    const pkeyNames = this.columns.filter(c => c.is_pkey).map(c => c.name);
+    const getInsertQuery = async (_rows: AnyObject[]) => {
+      const validatedData = await Promise.all(_rows.map(async _row => {
+
+        const row = { ..._row };
+
+        if (!isObject(row)) {
+          throw "\nInvalid insert data provided. Expected an object but received: " + JSON.stringify(row);
+        }
+  
+        const { data: validatedRow, allowedCols } = await prepareNewData({ 
+          row, 
+          forcedData, 
+          allowedFields: fields, 
+          tableRules, 
+          fixIssues, 
+          tableConfigurator: this.dboBuilder.prostgles.tableConfigurator, 
+          tableHandler: this, 
+        });
+        return { validatedRow, allowedCols };
+      }));
+      const validatedRows = validatedData.map(d => d.validatedRow);
+      const allowedCols = Array.from(new Set(validatedData.flatMap(d => d.allowedCols)));
+      const dbTx = finalDBtx || this.dboBuilder.dbo;
+      const validationOptions = { validate: validate as ValidateRowBasic, localParams };
+      // const query = await this.colSet.getInsertQuery(validatedRows, allowedCols, dbTx, validate, localParams);
+      const query = (await this.dataValidator.parse({ command: "insert", rows: validatedRows, allowedCols, dbTx, validationOptions })).getQuery();
+      const { onConflict } = insertParams ?? {};
+      let conflict_query = "";
+      if (onConflict === "DoNothing") {
+        conflict_query = " ON CONFLICT DO NOTHING ";
+      } else if(onConflict === "DoUpdate"){
+        if(!pkeyNames.length) {
+          throw "Cannot do DoUpdate on a table without a primary key";
+        }
+        const nonPkeyCols = allowedCols.filter(c => !pkeyNames.includes(c)).map(v => asName(v));
+        if(!nonPkeyCols.length){
+          throw "Cannot on conflict DoUpdate on a table with only primary key columns";
+        }
+        conflict_query = ` ON CONFLICT (${pkeyNames.join(", ")}) DO UPDATE SET ${nonPkeyCols.map(k => `${k} = EXCLUDED.${k}`).join(", ")}`;
+      }
+      return query + conflict_query;
+    };
+    
+    let query = ""; 
+    if (Array.isArray(data)) {
+
+      if(!data.length){
+        throw "Empty insert. Provide data";
+      }
+
+      query = await getInsertQuery(data); 
+    } else {
+      query = await getInsertQuery([data ?? {}]); 
+    }
+
+    const queryWithoutUserRLS = query;
+    const queryWithRLS = withUserRLS(localParams, query);
+    if (returnQuery) return queryWithRLS;
+
+    if (this.dboBuilder.prostgles.opts.DEBUG_MODE) {
+      console.log(this.tx?.t.ctx?.start, "insert in " + this.name, data);
+    }
+
+    const result = await runInsertUpdateQuery({
+      rule, 
+      localParams, 
+      queryWithoutUserRLS, 
+      tableHandler: this, 
+      returningFields, 
+      data: preValidatedrowOrRows,
+      fields,
+      params: insertParams,
+      type: "insert",
+      isMultiInsert,
+    });
+    await this._log({ command: "insert", localParams, data: { rowOrRows, param2: insertParams }, duration: Date.now() - start });
+    return result;
+  } catch (e) {
+    await this._log({ command: "insert", localParams, data: { rowOrRows, param2: insertParams }, duration: Date.now() - start, error: getErrorAsObject(e) });
+    throw getSerializedClientErrorFromPGError(e, { type: "tableMethod", localParams, view: this });
+  }
+} 
+
+const validateInsertParams = (params: InsertParams | undefined) => {
+
+  const { onConflict, returnType, returning } = params ?? {};
+  if(![undefined, "DoNothing", "DoUpdate"].includes(onConflict)){
+    throw `Invalid onConflict: ${onConflict}. Expecting one of: DoNothing, DoUpdate`;
+  }
+
+  const allowedReturnTypes: InsertParams["returnType"][] = ["row", "value", "values", "statement", undefined]
+  if(!allowedReturnTypes.includes(returnType)){
+    throw `Invalid return type ${returnType}. Expecting one of: ${allowedReturnTypes}`
+  }
+
+  if(returnType && returnType !== "statement" && !returning){
+    throw `Must specify returning when using a non statement returnType: ${returnType}`;
+  }
+
+  if (params) {
+    const good_paramsObj: Record<keyof InsertParams, 1> = { returning: 1, returnType: 1, fixIssues: 1, onConflict: 1 };
+    const good_params = Object.keys(good_paramsObj);
+    const bad_params = Object.keys(params).filter(k => !good_params.includes(k));
+    if (bad_params && bad_params.length) throw "Invalid params: " + bad_params.join(", ") + " \n Expecting: " + good_params.join(", ");
+  }
+}
+
+// const removeBuffers = (o: any) => {
+//   if(isPlainObject(o)){
+//     return JSON.stringify(getKeys(o).reduce((a, k) => {
+//       const value = o[k]
+//       return { ...a, [k]: Buffer.isBuffer(value)? `Buffer[${value.byteLength}][...REMOVED]` : value 
+//     }
+//   }, {}));
+//   }
+// }

package/lib/DboBuilder/TableHandler/insertTest.ts

@@ -0,0 +1,78 @@
+import { AnyObject, asName, FieldFilter, FullFilter } from "prostgles-types";
+import { LocalParams, pgp } from "../DboBuilder";
+import { TableRule } from "../../PublishParser/PublishParser";
+import { asValue } from "../../PubSubManager/PubSubManager";
+import { TableHandler } from "./TableHandler";
+
+type InsertTestArgs = {
+  tableRules: TableRule | undefined;
+  localParams: LocalParams | undefined;
+}
+export async function insertTest(this: TableHandler, { localParams, tableRules }: InsertTestArgs) {
+  const { testRule } = localParams || {};
+  
+  const ACTION = "insert";
+  
+  let returningFields: FieldFilter | undefined;
+  let forcedData: AnyObject | undefined;
+  let fields: FieldFilter | undefined;
+  let checkFilter: FullFilter<AnyObject, void> | undefined;
+  let testOnly = false;
+
+  if (tableRules) {
+    if (!tableRules[ACTION]) throw `${ACTION} rules missing for ${this.name}`;
+    returningFields = tableRules[ACTION].returningFields;
+    forcedData = tableRules[ACTION].forcedData;
+    checkFilter = tableRules[ACTION].checkFilter;
+    fields = tableRules[ACTION].fields;
+
+    /* If no returning fields specified then take select fields as returning or the allowed insert fields */
+    if (!returningFields) returningFields = tableRules?.select?.fields || tableRules?.insert.fields;
+
+    if (!fields) throw ` invalid insert rule for ${this.name} -> fields missing `;
+
+    /* Safely test publish rules */
+    if (testRule) {
+      await this.validateViewRules({ fields, returningFields, forcedFilter: forcedData, rule: "insert" });
+      if(checkFilter){
+        try {
+          await this.find(checkFilter, { limit: 0 });
+        } catch (e){
+          throw `Invalid checkFilter provided for ${this.name}. Error: ${JSON.stringify(e)}`
+        }
+      }
+      if (forcedData) {
+        const keys = Object.keys(forcedData);
+        if (keys.length) {
+          const dataCols = keys.filter(k => this.column_names.includes(k));
+          const nestedInsertCols = keys.filter(k => !this.column_names.includes(k) && this.dboBuilder.dbo[k]?.insert);
+          if(nestedInsertCols.length){
+            throw `Nested insert not supported for forcedData rule: ${nestedInsertCols}`;
+          }
+          const badCols = keys.filter(k => !dataCols.includes(k) && !nestedInsertCols.includes(k));
+          if(badCols.length){
+            throw `Invalid columns found in forced filter: ${badCols.join(", ")}`;
+          }
+          try {
+            const values = "(" + dataCols.map(k => asValue(forcedData![k]) + "::" + this.columns.find(c => c.name === k)!.udt_name).join(", ") + ")",
+              colNames = dataCols.map(k => asName(k)).join(",");
+            const query = pgp.as.format("EXPLAIN INSERT INTO " + this.escapedName + " (${colNames:raw}) SELECT * FROM ( VALUES ${values:raw} ) t WHERE FALSE;", { colNames, values })
+            await this.db.any(query);
+          } catch (e) {
+            throw "\nissue with forcedData: \nVALUE: " + JSON.stringify(forcedData, null, 2) + "\nERROR: " + e;
+          }
+        }
+      }
+
+      testOnly = true;
+    }
+  }
+
+  return {
+    returningFields,
+    fields,
+    forcedData,
+    checkFilter,
+    testOnly
+  }
+} 

package/lib/DboBuilder/TableHandler/onDeleteFromFileTable.ts

@@ -0,0 +1,62 @@
+import { AnyObject, asName } from "prostgles-types";
+import { LocalParams } from "../DboBuilder";
+import { pickKeys } from "../../PubSubManager/PubSubManager";
+import { TableHandler } from "./TableHandler";
+
+type OnDeleteFromFileTableArgs = {
+  localParams: LocalParams | undefined;
+  queryType: "one" | "none" | "many" | "any";
+  returningQuery: undefined | string;
+  filterOpts: {
+    where: string;
+    filter: AnyObject;
+  }
+}
+export async function onDeleteFromFileTable(this: TableHandler, { localParams, queryType, returningQuery, filterOpts }: OnDeleteFromFileTableArgs){
+
+  if (!this.dboBuilder.prostgles.fileManager) throw new Error("fileManager missing")
+  if (this.dboBuilder.prostgles.opts.fileTable?.delayedDelete) {
+    return this.dbHandler[queryType](`UPDATE ${asName(this.name)} SET deleted = now() ${filterOpts.where} ${returningQuery ?? ""};`)
+  } else {
+
+    const txDelete = async (tbl: TableHandler) => {
+      if (!tbl.tx) throw new Error("Missing transaction object tx");
+      let files: { id: string; name: string }[] = [];
+      const totalFiles = await tbl.count(filterOpts.filter);
+      do {
+        const batch = await tbl.find(filterOpts.filter, { limit: 100, offset: files.length });
+        files = files.concat(batch);
+      } while(files.length < totalFiles)
+      
+      const fileManager = tbl.dboBuilder.prostgles.fileManager
+      if (!fileManager) throw new Error("fileManager missing");
+
+      for await (const file of files) {
+        await tbl.tx.t.any(`DELETE FROM ${asName(this.name)} WHERE id = \${id}`, file);
+      }
+      /** If any table delete fails then do not delete files */
+      for await (const file of files) {
+        await fileManager.deleteFile(file.name);
+        /** TODO: Keep track of deleted files in case of failure */
+        // await tbl.t?.any(`UPDATE ${asName(this.name)} SET deleted = NOW(), deleted_from_storage = NOW()  WHERE id = ` + "${id}", file);
+      }
+
+      if (returningQuery) {
+        return files.map(f => pickKeys(f, ["id", "name"]));
+      }
+
+      return undefined;
+    }
+
+    if (localParams?.tx?.dbTX) {
+      return txDelete(localParams.tx.dbTX[this.name] as TableHandler)
+    } else if (this.tx) {
+      return txDelete(this)
+    } else {
+
+      return this.dboBuilder.getTX(tx => {
+        return txDelete(tx[this.name] as TableHandler)
+      })
+    }
+  }
+}

package/lib/DboBuilder/TableHandler/runInsertUpdateQuery.ts

@@ -0,0 +1,134 @@
+import { AnyObject, asName, FieldFilter, InsertParams, UpdateParams } from "prostgles-types";
+import { InsertRule, UpdateRule } from "../../PublishParser/PublishParser";
+import { getClientErrorFromPGError, LocalParams, withUserRLS } from "../DboBuilder";
+import { getSelectItemQuery, TableHandler } from "./TableHandler";
+
+type RunInsertUpdateQueryArgs = {
+  tableHandler: TableHandler;
+  queryWithoutUserRLS: string; 
+  localParams: LocalParams | undefined;
+  fields: FieldFilter | undefined;
+  returningFields: FieldFilter | undefined;
+} & ({
+  type: "insert";
+  params: InsertParams | undefined
+  rule: InsertRule | undefined;
+  data: AnyObject | AnyObject[];
+  isMultiInsert: boolean;
+  nestedInsertsResultsObj?: undefined;
+} | {
+  type: "update";
+  nestedInsertsResultsObj: Record<string, any>;
+  params: UpdateParams | undefined
+  rule: UpdateRule | undefined;
+  data: undefined;
+});
+
+export const runInsertUpdateQuery = async (args: RunInsertUpdateQueryArgs) => {
+  const { tableHandler, queryWithoutUserRLS, rule, localParams, fields, returningFields, params, nestedInsertsResultsObj } = args;
+  const { name } = tableHandler;
+
+  const returningSelectItems = await tableHandler.prepareReturning(params?.returning, tableHandler.parseFieldFilter(returningFields))
+  const { checkFilter, postValidate } = rule ?? {};
+  let checkCondition = "WHERE FALSE";
+  if(checkFilter){
+    const checkCond = await tableHandler.prepareWhere({
+      select: undefined,
+      localParams: undefined,
+      tableRule: undefined,
+      filter: checkFilter,
+      addWhere: false,
+    });
+    checkCondition = `WHERE NOT (${checkCond.where})`;
+  }
+  const hasReturning = !!returningSelectItems.length;
+  const userRLS = withUserRLS(localParams, "");
+  const escapedTableName = asName(name)
+  const query = ` 
+    ${userRLS}
+    WITH ${escapedTableName} AS (
+      ${queryWithoutUserRLS}
+      RETURNING *
+    )
+    SELECT 
+      count(*) as row_count,
+      (
+        SELECT json_agg(item)
+        FROM (
+          SELECT *
+          FROM ${escapedTableName}
+        ) item
+      ) as modified,
+      (
+        SELECT json_agg(item)
+        FROM (
+          SELECT ${!hasReturning? "1" : getSelectItemQuery(returningSelectItems)}
+          FROM ${escapedTableName}
+          WHERE ${hasReturning? "TRUE" : "FALSE"}
+        ) item
+      ) as modified_returning,
+      ( 
+        SELECT json_agg(item)
+        FROM (
+          SELECT *
+          FROM ${escapedTableName}
+          ${checkCondition}
+          LIMIT 5
+        ) item
+      ) AS failed_check
+    FROM ${escapedTableName}
+  `;
+
+  const allowedFieldKeys = tableHandler.parseFieldFilter(fields);
+  let result: { 
+    row_count: number | null; 
+    modified: AnyObject[] | null; 
+    failed_check: AnyObject[] | null;
+    modified_returning: AnyObject[] | null;
+  };
+  
+  const queryType = "one";
+
+  const tx = localParams?.tx?.t || tableHandler.tx?.t;
+  if (tx) {
+    result = await tx[queryType](query).catch((err: any) => getClientErrorFromPGError(err, { type: "tableMethod", localParams, view: tableHandler, allowedKeys: allowedFieldKeys }));
+  } else {
+    result = await tableHandler.db.tx(t => (t as any)[queryType](query)).catch(err => getClientErrorFromPGError(err, { type: "tableMethod", localParams, view: tableHandler, allowedKeys: allowedFieldKeys }));
+  }
+
+  if(checkFilter && result.failed_check?.length){
+    throw new Error(`Insert ${name} records failed the check condition: ${JSON.stringify(checkFilter, null, 2)}`);
+  }
+
+  const finalDBtx = tableHandler.getFinalDBtx(localParams);
+  if(postValidate){
+    if(!finalDBtx) throw new Error("Unexpected: no dbTX for postValidate");
+    if(!localParams) throw new Error("Unexpected: no localParams for postValidate");
+
+    const rows = result.modified ?? [];
+    for await (const row of rows){
+      await postValidate({ row: row ?? {}, dbx: finalDBtx as any, localParams })
+    }
+  }
+
+  let returnMany = false;
+  if(args.type === "update"){
+    const { multi = true } = args.params || {};
+    if(!multi && result.row_count && +result.row_count > 1){
+      throw `More than 1 row modified: ${result.row_count} rows affected`;
+    }
+
+    if(hasReturning){
+      returnMany = multi;
+    }
+
+  } else {
+    returnMany = args.isMultiInsert
+  }
+
+  if(!hasReturning) return undefined;
+
+  const modified_returning = result.modified_returning?.map(d => ({ ...d, ...nestedInsertsResultsObj }))
+
+  return returnMany? modified_returning : modified_returning?.[0];
+}