prostgles-server 4.2.4 → 4.2.6

package/lib/DboBuilder/TableHandler/insert.ts

@@ -1,10 +1,12 @@
 import { AnyObject, InsertParams, isObject } from "prostgles-types";
 import { LocalParams, parseError, withUserRLS } from "../DboBuilder";
-import { TableRule } from "../../PublishParser/PublishParser";
-import { insertDataParse } from "../insertDataParse";
+import { TableRule, ValidateRowBasic } from "../../PublishParser/PublishParser";
+import { insertNestedRecords } from "../insertNestedRecords";
 import { insertTest } from "./insertTest";
 import { TableHandler } from "./TableHandler";
 import { runInsertUpdateQuery } from "./runInsertUpdateQuery";
+import { prepareNewData } from "./DataValidator";
+import { DBOFullyTyped } from "../../DBSchemaBuilder";
 
 export async function insert(this: TableHandler, rowOrRows: AnyObject | AnyObject[] = {}, insertParams?: InsertParams, param3_unused?: undefined, tableRules?: TableRule, localParams?: LocalParams): Promise<any | any[] | boolean> {
   
@@ -38,11 +40,26 @@ export async function insert(this: TableHandler, rowOrRows: AnyObject | AnyObjec
     }
     validateInsertParams(insertParams);
 
+    const isMultiInsert = Array.isArray(rowOrRows);
+    const preValidatedRows = await Promise.all((isMultiInsert? rowOrRows : [rowOrRows]).map(async nonValidated => {
+      const { preValidate, validate } = tableRules?.insert ?? {};
+      const { tableConfigurator } = this.dboBuilder.prostgles;
+      if(!tableConfigurator) throw "tableConfigurator missing";
+      let row = await tableConfigurator.getPreInsertRow(this, { dbx: this.getFinalDbo(localParams), validate, localParams, row: nonValidated })
+      if (preValidate) {
+        if(!localParams) throw "localParams missing for insert preValidate";
+        row = await preValidate({ row, dbx: (this.tx?.dbTX || this.dboBuilder.dbo) as any, localParams });
+      }
+
+      return row;
+    }));
+    const preValidatedrowOrRows = isMultiInsert? preValidatedRows : preValidatedRows[0]!;
+
     /**
      * If media it will: upload file and continue insert
      * If nested insert it will: make separate inserts and not continue main insert
      */
-    const mediaOrNestedInsert = await insertDataParse.bind(this)(rowOrRows, insertParams, param3_unused, tableRules, localParams);
+    const mediaOrNestedInsert = await insertNestedRecords.bind(this)({ data: preValidatedrowOrRows, param2: insertParams, tableRules, localParams });
     const { data, insertResult } = mediaOrNestedInsert;
     if ("insertResult" in mediaOrNestedInsert) {
       return insertResult;
@@ -55,25 +72,38 @@ export async function insert(this: TableHandler, rowOrRows: AnyObject | AnyObjec
         const row = { ..._row };
 
         if (!isObject(row)) {
-          console.trace(row)
-          throw "\ninvalid insert data provided -> " + JSON.stringify(row);
+          throw "\nInvalid insert data provided. Expected an object but received: " + JSON.stringify(row);
         }
   
-        const { data: validatedRow, allowedCols } = this.validateNewData({ row, forcedData, allowedFields: fields, tableRules, fixIssues });
+        const { data: validatedRow, allowedCols } = await prepareNewData({ 
+          row, 
+          forcedData, 
+          allowedFields: fields, 
+          tableRules, 
+          fixIssues, 
+          tableConfigurator: this.dboBuilder.prostgles.tableConfigurator, 
+          tableHandler: this, 
+        });
         return { validatedRow, allowedCols };
       }));
       const validatedRows = validatedData.map(d => d.validatedRow);
-      const allowedCols = Array.from( new Set(validatedData.flatMap(d => d.allowedCols)));
-      const dbTx = finalDBtx || this.dboBuilder.dbo
-      const query = await this.colSet.getInsertQuery(validatedRows, allowedCols, dbTx, validate, localParams);
-
+      const allowedCols = Array.from(new Set(validatedData.flatMap(d => d.allowedCols)));
+      const dbTx = finalDBtx || this.dboBuilder.dbo;
+      const validationOptions = { validate: validate as ValidateRowBasic, localParams };
+      // const query = await this.colSet.getInsertQuery(validatedRows, allowedCols, dbTx, validate, localParams);
+      const query = (await this.dataValidator.parse({ command: "insert", rows: validatedRows, allowedCols, dbTx, validationOptions })).getQuery();
       const { onConflict } = insertParams ?? {};
       let conflict_query = "";
       if (onConflict === "DoNothing") {
         conflict_query = " ON CONFLICT DO NOTHING ";
       } else if(onConflict === "DoUpdate"){
-        if(!pkeyNames.length) throw "Cannot do DoUpdate on a table without a primary key";
+        if(!pkeyNames.length) {
+          throw "Cannot do DoUpdate on a table without a primary key";
+        }
         const nonPkeyCols = allowedCols.filter(c => !pkeyNames.includes(c));
+        if(!nonPkeyCols.length){
+          throw "Cannot on conflict DoUpdate on a table with only primary key columns";
+        }
         conflict_query = ` ON CONFLICT (${pkeyNames.join(", ")}) DO UPDATE SET ${nonPkeyCols.map(k => `${k} = EXCLUDED.${k}`).join(", ")}`;
       }
       return query + conflict_query;
@@ -100,14 +130,16 @@ export async function insert(this: TableHandler, rowOrRows: AnyObject | AnyObjec
     }
 
     return runInsertUpdateQuery({
-      rule, localParams, 
+      rule, 
+      localParams, 
       queryWithoutUserRLS, 
       tableHandler: this, 
       returningFields, 
-      data: rowOrRows,
+      data: preValidatedrowOrRows,
       fields,
       params: insertParams,
       type: "insert",
+      isMultiInsert,
     });
     
   } catch (e) {

package/lib/DboBuilder/TableHandler/runInsertUpdateQuery.ts

@@ -2,6 +2,7 @@ import { AnyObject, asName, FieldFilter, InsertParams, UpdateParams } from "pros
 import { LocalParams, getClientErrorFromPGError, withUserRLS } from "../DboBuilder";
 import { InsertRule, UpdateRule } from "../../PublishParser/PublishParser";
 import { getSelectItemQuery, TableHandler } from "./TableHandler";
+import { DBOFullyTyped } from "../../DBSchemaBuilder";
 
 type RunInsertUpdateQueryArgs = {
   tableHandler: TableHandler;
@@ -14,6 +15,7 @@ type RunInsertUpdateQueryArgs = {
   params: InsertParams | undefined
   rule: InsertRule | undefined;
   data: AnyObject | AnyObject[];
+  isMultiInsert: boolean;
   nestedInsertsResultsObj?: undefined;
 } | {
   type: "update";
@@ -23,7 +25,8 @@ type RunInsertUpdateQueryArgs = {
   data: undefined;
 });
 
-export const runInsertUpdateQuery = async ({ tableHandler, queryWithoutUserRLS, rule, localParams, fields, returningFields, params, data, type, nestedInsertsResultsObj }: RunInsertUpdateQueryArgs) => {
+export const runInsertUpdateQuery = async (args: RunInsertUpdateQueryArgs) => {
+  const { tableHandler, queryWithoutUserRLS, rule, localParams, fields, returningFields, params, nestedInsertsResultsObj } = args;
   const { name } = tableHandler;
 
   const returningSelectItems = await tableHandler.prepareReturning(params?.returning, tableHandler.parseFieldFilter(returningFields))
@@ -104,13 +107,13 @@ export const runInsertUpdateQuery = async ({ tableHandler, queryWithoutUserRLS,
 
     const rows = result.modified ?? [];
     for await (const row of rows){
-      await postValidate({ row: row ?? {}, dbx: finalDBtx, localParams })
+      await postValidate({ row: row ?? {}, dbx: finalDBtx as any, localParams })
     }
   }
 
   let returnMany = false;
-  if(type === "update"){
-    const { multi = true } = params || {};
+  if(args.type === "update"){
+    const { multi = true } = args.params || {};
     if(!multi && result.row_count && +result.row_count > 1){
       throw `More than 1 row modified: ${result.row_count} rows affected`;
     }
@@ -120,7 +123,7 @@ export const runInsertUpdateQuery = async ({ tableHandler, queryWithoutUserRLS,
     }
 
   } else {
-    returnMany = Array.isArray(data)
+    returnMany = args.isMultiInsert
   }
 
   if(!hasReturning) return undefined;

package/lib/DboBuilder/TableHandler/update.ts

@@ -1,10 +1,11 @@
-import { AnyObject, unpatchText, UpdateParams } from "prostgles-types";
-import { Filter, isPlainObject, LocalParams, parseError, withUserRLS } from "../DboBuilder";
+import { AnyObject, UpdateParams } from "prostgles-types";
 import { TableRule } from "../../PublishParser/PublishParser";
-import { getInsertTableRules, getReferenceColumnInserts } from "../insertDataParse";
+import { Filter, LocalParams, parseError, withUserRLS } from "../DboBuilder";
+import { getInsertTableRules, getReferenceColumnInserts } from "../insertNestedRecords";
 import { runInsertUpdateQuery } from "./runInsertUpdateQuery";
 import { TableHandler } from "./TableHandler";
 import { updateFile } from "./updateFile";
+import { prepareNewData } from "./DataValidator";
 
 export async function update(this: TableHandler, filter: Filter, _newData: AnyObject, params?: UpdateParams, tableRules?: TableRule, localParams?: LocalParams): Promise<AnyObject | void> {
   const ACTION = "update";
@@ -23,11 +24,15 @@ export async function update(this: TableHandler, filter: Filter, _newData: AnyOb
       ({ newData } = await updateFile.bind(this)({ newData, filter, localParams, tableRules }));
     }
 
-    const parsedRules = await this.parseUpdateRules(filter, newData, params, tableRules, localParams)
+    const parsedRules = await this.parseUpdateRules(filter, params, tableRules, localParams)
     if (localParams?.testRule) {
       return parsedRules;
     }
 
+    if (!newData || !Object.keys(newData).length) {
+      throw "no update data provided\nEXPECTING db.table.update(filter, updateData, options)";
+    }
+
     const { fields, validateRow, forcedData,  returningFields, forcedFilter, filterFields } = parsedRules;
     const { onConflict, fixIssues = false } = params || {};
     const { returnQuery = false } = localParams ?? {};
@@ -39,38 +44,16 @@ export async function update(this: TableHandler, filter: Filter, _newData: AnyOb
       if (bad_params && bad_params.length) throw "Invalid params: " + bad_params.join(", ") + " \n Expecting: " + good_params.join(", ");
     }
 
-    const { data, allowedCols } = this.validateNewData({ row: newData, forcedData, allowedFields: fields, tableRules, fixIssues });
-
-    /* Patch data */
-    const patchedTextData: {
-      fieldName: string;
-      from: number;
-      to: number;
-      text: string;
-      md5: string
-    }[] = [];
-    this.columns.map(c => {
-      const d = data[c.name];
-      if (c.data_type === "text" && d && isPlainObject(d) && !["from", "to"].find(key => typeof d[key] !== "number")) {
-        const unrecProps = Object.keys(d).filter(k => !["from", "to", "text", "md5"].includes(k));
-        if (unrecProps.length) throw "Unrecognised params in textPatch field: " + unrecProps.join(", ");
-        patchedTextData.push({ ...d, fieldName: c.name } as (typeof patchedTextData)[number]);
-      }
+    const { data, allowedCols } = await prepareNewData({ 
+      row: newData, 
+      forcedData, 
+      allowedFields: fields, 
+      tableRules, 
+      fixIssues,
+      tableConfigurator: this.dboBuilder.prostgles.tableConfigurator,
+      tableHandler: this, 
     });
 
-    if (patchedTextData?.length) {
-      if (tableRules && !tableRules.select) throw "Select needs to be permitted to patch data";
-      const rows = await this.find(filter, { select: patchedTextData.reduce((a, v) => ({ ...a, [v.fieldName]: 1 }), {}) }, undefined, tableRules);
-
-      if (rows.length !== 1) {
-        throw "Cannot patch data within a filter that affects more/less than 1 row";
-      }
-      patchedTextData.map(p => {
-        data[p.fieldName] = unpatchText(rows[0][p.fieldName], p);
-      });
-    }
-
-    const nData = { ...data };
     const updateFilter = await this.prepareWhere({
       filter,
       forcedFilter,
@@ -78,8 +61,12 @@ export async function update(this: TableHandler, filter: Filter, _newData: AnyOb
       localParams,
       tableRule: tableRules
     })
-
-    const nestedInserts = getReferenceColumnInserts.bind(this)(nData, true);
+    
+    /**
+     * Nested inserts
+     */
+    const nData = { ...data };
+    const nestedInserts = getReferenceColumnInserts(this, nData, true);
     const nestedInsertsResultsObj: Record<string, any> = {};
     if(nestedInserts.length){
       const updateCount = await this.count(updateFilter.filter);
@@ -105,7 +92,8 @@ export async function update(this: TableHandler, filter: Filter, _newData: AnyOb
       }));
     }
 
-    let query = await this.colSet.getUpdateQuery(nData, allowedCols, this.getFinalDbo(localParams), validateRow, localParams)
+    // let query = await this.colSet.getUpdateQuery(nData, allowedCols, this.getFinalDbo(localParams), validateRow, localParams)
+    let query = (await this.dataValidator.parse({ command: "update", rows: [nData], allowedCols, dbTx: this.getFinalDbo(localParams), validationOptions: { validate: validateRow, localParams }})).getQuery()
     query += "\n" + updateFilter.where;
     if (onConflict === "DoNothing") query += " ON CONFLICT DO NOTHING ";
     if(onConflict === "DoUpdate"){

package/lib/DboBuilder/TableHandler/updateBatch.ts

@@ -12,18 +12,18 @@ export async function updateBatch(this: TableHandler, updates: [Filter, AnyObjec
       throw `updateBatch not allowed for tables with checkFilter or postValidate rules`
     }
     const updateQueries: string[] = await Promise.all(
-      updates.map(async ([filter, data]) => {
+      updates.map(async ([filter, data]) => { 
         const query = (await this.update(
           filter,
           data,
-          { ...(params || {}), returning: undefined },
+          { ...(params ?? {}), returning: undefined },
           tableRules,
-          { ...(localParams || {}), returnQuery: "noRLS" }
+          { ...(localParams ?? {}), returnQuery: "noRLS" }
         )) as unknown as string;
 
         return query;
       })
-    ); 
+    );
     const queries = [
       withUserRLS(localParams, ""),
       ...updateQueries

package/lib/DboBuilder/TableHandler/updateFile.ts

@@ -1,9 +1,10 @@
 import { AnyObject, getKeys, isObject } from "prostgles-types";
 import { LocalParams, Media } from "../DboBuilder";
-import { TableRule, ValidateRow } from "../../PublishParser/PublishParser";
+import { TableRule, ValidateRow, ValidateRowBasic } from "../../PublishParser/PublishParser";
 import { omitKeys } from "../../PubSubManager/PubSubManager";
 import { isFile, uploadFile } from "../uploadFile";
 import { TableHandler } from "./TableHandler";
+import { DBOFullyTyped } from "../../DBSchemaBuilder";
 
 type Args = {
   newData: AnyObject; 
@@ -32,8 +33,8 @@ export const updateFile = async function(this: TableHandler, { filter, newData,
   const fileManager = this.dboBuilder.prostgles.fileManager
   if(!fileManager) throw new Error("fileManager missing");
   if(rule?.validate && !localParams) throw new Error("localParams missing");
-  const validate: ValidateRow | undefined = rule?.validate? async (row) => {
-    return rule.validate!({ update: row, filter, dbx:  this.tx?.dbTX || this.dboBuilder.dbo, localParams: localParams! })
+  const validate: ValidateRowBasic | undefined = rule?.validate? async (row) => {
+    return rule.validate!({ update: row, filter, dbx:  (this.tx?.dbTX || this.dboBuilder.dbo) as any, localParams: localParams! })
   } : undefined;
 
   const existingFile: Media | undefined = await (localParams?.tx?.dbTX?.[this.name] as TableHandler || this).findOne({ id: existingMediaId });

package/lib/DboBuilder/ViewHandler/ViewHandler.ts

@@ -25,7 +25,6 @@ import { asNameAlias } from "../QueryBuilder/QueryBuilder";
 import { find } from "../find";
 import { getColumns } from "../getColumns";
 import { LocalFuncs, subscribe } from "../subscribe";
-import { ColSet } from "./ColSet";
 import { getInfo } from "./getInfo";
 import { parseFieldFilter } from "./parseFieldFilter";
 import { prepareWhere } from "./prepareWhere";
@@ -46,7 +45,6 @@ export class ViewHandler {
   columnsForTypes: ColumnInfo[];
   column_names: string[];
   tableOrViewInfo: TableSchema;// TableOrViewInfo;
-  colSet: ColSet;
   tsColumnDefs: string[] = [];
   joins: Join[];
   joinGraph?: Graph;
@@ -74,16 +72,13 @@ export class ViewHandler {
     this.name = tableOrViewInfo.escaped_identifier;
     this.escapedName = tableOrViewInfo.escaped_identifier;
     this.columns = tableOrViewInfo.columns;
-
     /* cols are sorted by name to reduce .d.ts schema rewrites */
     this.columnsForTypes = tableOrViewInfo.columns.slice(0).sort((a, b) => a.name.localeCompare(b.name));
 
     this.column_names = tableOrViewInfo.columns.map(c => c.name);
  
     this.dboBuilder = dboBuilder;
-    this.joins = this.dboBuilder.joins ?? [];
- 
-    this.colSet = new ColSet(this.columns, this.name); 
+    this.joins = this.dboBuilder.joins ?? []; 
     this.columnsForTypes.map(({ name, udt_name, is_nullable }) => {
       this.tsColumnDefs.push(`${escapeTSNames(name)}?: ${postgresToTsType(udt_name) as string} ${is_nullable ? " | null " : ""};`);
     });
@@ -456,54 +451,8 @@ export class ViewHandler {
     return result;
   }
 
-  /** 
-  * Prepare and validate field object:   
-  * @example ({ item_id: 1 }, { user_id: 32 }) => { item_id: 1, user_id: 32 }
-  * OR
-  * ({ a: 1 }, { b: 32 }, ["c", "d"]) => throw "a field is not allowed"
-  * @param {Object} obj - initial data
-  * @param {Object} forcedData - set/override property
-  * @param {string[]} allowed_cols - allowed columns (excluding forcedData) from table rules
-  */
-  prepareFieldValues(obj: AnyObject = {}, forcedData: AnyObject = {}, allowed_cols: FieldFilter | undefined, removeDisallowedColumns = false): AnyObject {
-    const column_names = this.column_names.slice(0);
-    if (!column_names?.length) {
-      throw "table column_names mising";
-    }
-    let _allowed_cols = column_names.slice(0);
-    const _obj = { ...obj };
-
-    if (allowed_cols) {
-      _allowed_cols = this.parseFieldFilter(allowed_cols, false);
-    }
-    let final_filter = { ..._obj };
-    const filter_keys: Array<keyof typeof final_filter> = Object.keys(final_filter);
-
-    if (removeDisallowedColumns && filter_keys.length) {
-      final_filter = {};
-      filter_keys
-        .filter(col => _allowed_cols.includes(col))
-        .map(col => {
-          final_filter[col] = _obj[col];
-        });
-    }
-
-    /* If has keys check against allowed_cols */
-    if (final_filter && Object.keys(final_filter).length && _allowed_cols) {
-      validateObj(final_filter, _allowed_cols)
-    }
-
-    if (forcedData && Object.keys(forcedData).length) {
-      final_filter = { ...final_filter, ...forcedData };
-    }
-
-    validateObj(final_filter, column_names.slice(0));
-    return final_filter;
-  }
-
-
   parseFieldFilter(fieldParams: FieldFilter = "*", allow_empty = true, allowed_cols?: string[]): string[] {
-    return parseFieldFilter(fieldParams, allow_empty, allowed_cols || this.column_names.slice(0))
+    return parseFieldFilter(fieldParams, allow_empty, allowed_cols ?? this.column_names.slice(0))
   }
 
 }
@@ -512,7 +461,7 @@ export class ViewHandler {
 /** 
 * Throw error if illegal keys found in object
 */
-function validateObj<T extends Record<string, any>>(obj: T, allowedKeys: string[]): T {
+export const validateObj = <T extends Record<string, any>>(obj: T, allowedKeys: string[]): T => {
   if (obj && Object.keys(obj).length) {
     const invalid_keys = Object.keys(obj).filter(k => !allowedKeys.includes(k));
     if (invalid_keys.length) {

package/lib/DboBuilder/dboBuilderUtils.ts

@@ -154,7 +154,10 @@ export const getConstraints = async (db: DB, schema: ProstglesInitOptions["schem
   `, { schemaNames });
 }
 
-
+/**
+ * @deprecated
+ * use isObject
+ */
 export function isPlainObject(o: any): o is Record<string, any> {
   return Object(o) === o && Object.getPrototypeOf(o) === Object.prototype;
 }

package/lib/DboBuilder/getColumns.ts

@@ -1,6 +1,6 @@
 import { 
   AnyObject, PG_COLUMN_UDT_DATA_TYPE, 
-  ValidatedColumnInfo, _PG_geometric 
+  ValidatedColumnInfo, _PG_geometric, isObject 
 } from "prostgles-types";
 import { isPlainObject, LocalParams, parseError, postgresToTsType } from "./DboBuilder";
 import { TableRule } from "../PublishParser/PublishParser";
@@ -29,19 +29,18 @@ export async function getColumns(
 
     if (params && tableRules && isTableHandler(this)) {
       if (
-        !isPlainObject(params) || 
-        (params.data && !isPlainObject(params.data)) || 
-        !isPlainObject(params.filter) || 
+        !isObject(params) || 
+        !isObject(params.filter) || 
         params.rule !== "update"
       ) {
-        throw "params must be { rule: 'update', filter: object, data?: object } but got: " + JSON.stringify(params);
+        throw "params must be { rule: 'update', filter: object } but received: " + JSON.stringify(params);
       }
 
       if (!tableRules?.update) {
         dynamicUpdateFields = [];
       } else {
-        const { data, filter } = params;
-        const updateRules = await this.parseUpdateRules(filter, data, undefined, tableRules, localParams);
+        const { filter } = params;
+        const updateRules = await this.parseUpdateRules(filter, undefined, tableRules, localParams);
         dynamicUpdateFields = updateRules.fields;
       }
     }

package/lib/DboBuilder/{insertDataParse.ts → insertNestedRecords.ts}

@@ -1,36 +1,43 @@
-import { AnyObject, getKeys, InsertParams, isDefined, isObject, PG_COLUMN_UDT_DATA_TYPE } from "prostgles-types";
+import { AnyObject, getKeys, InsertParams, isDefined, isObject } from "prostgles-types";
 import { LocalParams, TableHandlers } from "./DboBuilder";
 import { TableRule } from "../PublishParser/PublishParser";
 import { omitKeys } from "../PubSubManager/PubSubManager";
 import { TableHandler } from "./TableHandler/TableHandler";
 
+type InsertNestedRecordsArgs = {
+  data: (AnyObject | AnyObject[]);
+  param2?: InsertParams;
+  tableRules?: TableRule;
+  localParams?: LocalParams;
+};
+
 /**
  * Referenced inserts within a single transaction
  */
-export async function insertDataParse(
+export async function insertNestedRecords(
   this: TableHandler,
-  data: (AnyObject | AnyObject[]),
-  param2?: InsertParams,
-  param3_unused?: undefined,
-  tableRules?: TableRule,
-  _localParams?: LocalParams
+  {
+    data,
+    param2, 
+    tableRules,
+    localParams = {},
+  }: InsertNestedRecordsArgs
 ): Promise<{
   data?: AnyObject | AnyObject[];
   insertResult?: AnyObject | AnyObject[];
-}> {
-  const localParams = _localParams || {};
+}> { 
   const MEDIA_COL_NAMES = ["data", "name"];
 
-  const isMultiInsert = Array.isArray(data);
-  const getExtraKeys = (d: AnyObject) => getKeys(d).filter(k => {
+  const getExtraKeys = (row: AnyObject) => getKeys(row).filter(fieldName => {
     /* If media then use file insert columns */
     if (this.is_media) {
-      return !this.column_names.concat(MEDIA_COL_NAMES).includes(k)
-    } else if (!this.columns.find(c => c.name === k)) {
-      if (!isObject(d[k]) && !Array.isArray(d[k])) {
-        throw new Error("Invalid/Dissalowed field in data: " + k)
-      } else if (!this.dboBuilder.dbo[k]) {
-        throw new Error("Invalid/Dissalowed nested insert table name in data: " + k)
+      return !this.column_names.concat(MEDIA_COL_NAMES).includes(fieldName)
+    } else if (!this.columns.find(c => c.name === fieldName)) {
+      if (!isObject(row[fieldName]) && !Array.isArray(row[fieldName])) {
+        throw new Error("Invalid/Dissalowed field in data: " + fieldName)
+      } else if (!this.dboBuilder.dbo[fieldName]) {
+        return false;
+        // throw new Error("Invalid/Dissalowed nested insert table name in data: " + fieldName)
       }
       return true;
     }
@@ -46,7 +53,8 @@ export async function insertDataParse(
    * If true then will do the full insert within this function
    * Nested insert is not allowed for the file table 
    * */
-  const hasNestedInserts = this.is_media ? false : (isMultiInsert ? data : [data]).some(d => getExtraKeys(d).length || getReferenceColumnInserts.bind(this)(d).length);
+  const isMultiInsert = Array.isArray(data);
+  const hasNestedInserts = this.is_media ? false : (isMultiInsert ? data : [data]).some(d => getExtraKeys(d).length || getReferenceColumnInserts(this, d).length);
 
   /**
    * Make sure nested insert uses a transaction
@@ -59,36 +67,34 @@ export async function insertDataParse(
         (dbTX[this.name] as TableHandler).insert(
           data,
           param2,
-          param3_unused,
+          undefined,
           tableRules,
           { tx: { dbTX, t: _t }, ...localParams }
         )
       )
     }
   }
+ 
 
-  const { preValidate, validate } = tableRules?.insert ?? {};
-
-  const _data = await Promise.all((isMultiInsert ? data : [data]).map(async _row => {
-    const { tableConfigurator } = this.dboBuilder.prostgles;
-    if(!tableConfigurator) throw "tableConfigurator missing";
-    let row = await tableConfigurator.getPreInsertRow(this, { dbx: this.getFinalDbo(localParams), preValidate, validate, localParams, row: _row })
-    if (preValidate) {
-      row = await preValidate({ row, dbx: this.tx?.dbTX || this.dboBuilder.dbo, localParams });
-    }
-
-    const extraKeys = getExtraKeys(row);
-    const colInserts = getReferenceColumnInserts.bind(this)(row);
+  const _data = await Promise.all((isMultiInsert ? data : [data]).map(async row => {
+    // const { preValidate, validate } = tableRules?.insert ?? {};
+    // const { tableConfigurator } = this.dboBuilder.prostgles;
+    // if(!tableConfigurator) throw "tableConfigurator missing";
+    // let row = await tableConfigurator.getPreInsertRow(this, { dbx: this.getFinalDbo(localParams), validate, localParams, row: _row })
+    // if (preValidate) {
+    //   row = await preValidate({ row, dbx: this.tx?.dbTX || this.dboBuilder.dbo, localParams });
+    // }
 
     /* Potentially a nested join */
     if (hasNestedInserts) {
-
+      const extraKeys = getExtraKeys(row);
+      const colInserts = getReferenceColumnInserts(this, row);
+      
       /* Ensure we're using the same transaction */
       const _this = this.tx ? this : dbTX![this.name] as TableHandler;
 
       const omitedKeys = extraKeys.concat(colInserts.map(c => c.col));
 
-      // let rootData = isMultiInsert? data.map(d => omitKeys(d, omitedKeys)) : omitKeys(data, omitedKeys);
       const rootData: AnyObject = omitKeys(row, omitedKeys);
 
       let insertedChildren: AnyObject[];
@@ -204,7 +210,6 @@ export async function insertDataParse(
             colsRefT1.map(col => {
               tbl2Row[col.name] = fullRootResult[col.references![0]!.fcols[0]!];
             })
-            // console.log({ rootResult, tbl2Row, t3Child, colsRefT3, colsRefT1, t: this.t?.ctx?.start });
 
             await childInsert(tbl2Row, tbl2!);//.then(() => {});
           }));
@@ -286,17 +291,21 @@ const referencedInsert = async (tableHandler: TableHandler, dbTX: TableHandlers
       .map(m => (dbTX![targetTable] as TableHandler)
         .insert(m, { returning: "*" }, undefined, childRules, localParams)
         .catch(e => {
-          console.trace({ childInsertErr: e })
-          return Promise.reject(e);
-          // return Promise.reject({ childInsertErr: e });
+          return Promise.reject(e); 
         })
       )
   );
 
 }
 
+type ReferenceColumnInsert<ExpectSingleInsert> = {
+  tableName: string;
+  col: string;
+  fcol: string;
+  singleInsert: boolean;
+  data: ExpectSingleInsert extends true? AnyObject : (AnyObject | AnyObject[]);
+}
 
-// const ALLOWED_COL_TYPES: PG_COLUMN_UDT_DATA_TYPE[] = ["int2", "int4", "int8", "numeric", "uuid", "text", "varchar", "char"];
 /**
  * Insert through the reference column. e.g.:
  *  {
@@ -304,16 +313,15 @@ const referencedInsert = async (tableHandler: TableHandler, dbTX: TableHandlers
  *    fkey_column: { ...referenced_table_data }
  *  }
  */
-export const getReferenceColumnInserts = function(this: TableHandler, parentRow: AnyObject, expectSingleInsert = false){ 
-  return getKeys(parentRow)
-    .map(k => {
-      if(parentRow[k] && isObject(parentRow[k])){
-        const insertedRefCol = this.columns.find(c => c.name === k);
-        if(insertedRefCol?.references?.length){
-          return {
-            insertedRefCol,
-            insertedRefColRef: insertedRefCol.references!
-          }
+export const getReferenceColumnInserts = <ExpectSingleInsert extends boolean>(tableHandler: TableHandler, parentRow: AnyObject, expectSingleInsert?: ExpectSingleInsert): ReferenceColumnInsert<ExpectSingleInsert>[] => { 
+  return Object.entries(parentRow)
+    .map(([insertedFieldName, insertedFieldValue]) => {
+      if(insertedFieldValue && isObject(insertedFieldValue)){
+        const insertedRefCol = tableHandler.columns.find(c => c.name === insertedFieldName && c.references?.length);
+        if(!insertedRefCol) return undefined;
+        return {
+          insertedRefCol,
+          insertedRefColRef: insertedRefCol.references!
         }
       }
     
@@ -335,12 +343,13 @@ export const getReferenceColumnInserts = function(this: TableHandler, parentRow:
       if(expectSingleInsert && !singleInsert){
         throw "Expected singleInsert";
       }
-      return {
+      const res = {
         tableName: insertedRefCol.references![0]!.ftable!,
         col: insertedRefCol.name,
         fcol: insertedRefCol.references![0]!.fcols[0]!,
         singleInsert,
         data: parentRow[insertedRefCol.name],
       }
+      return res;
     }).filter(isDefined);
 }