prostgles-server 1.5.122 → 2.0.3

package/lib/DboBuilder.ts

@@ -10,20 +10,63 @@ declare global { export interface Promise<T> extends Bluebird<T> {} }
 import * as pgPromise from 'pg-promise';
 import pg = require('pg-promise/typescript/pg-subset');
 import { 
-    ColumnInfo, ValidatedColumnInfo, FieldFilter, SelectParams, SubscribeParams, OrderBy, InsertParams, UpdateParams, DeleteParams,   
+    ColumnInfo, ValidatedColumnInfo, FieldFilter, SelectParams, SubscribeParams, OrderBy, InsertParams, UpdateParams, DeleteParams, SQLOptions,
     DbJoinMaker, 
     unpatchText,
     isEmpty,
     asName,
     PG_COLUMN_UDT_DATA_TYPE,
     TS_PG_Types,
+    TableInfo as TInfo,
+    SQLHandler,
+    AnyObject,
 } from "prostgles-types";
 
+export type Media = {
+    "id"?: string;
+    "title"?: string;
+    "extension"?: string;
+    "content_type"?: string;
+    "local_url"?: string;
+    "url"?: string;
+    "signed_url"?: string;
+    "signed_url_expires"?: number;
+    "name"?: string;
+    "original_name"?: string;
+    "etag"?: string;
+}
+
+export interface TxHandler {
+    [key: string]: TableHandler | ViewHandler;
+}
+export type TxCB = {
+    (t: TxHandler, _t: pgPromise.ITask<{}>): (any | void);
+}
+export type TX = {
+    (t: TxCB): Promise<(any | void)>;
+}
 export type DbHandler = {
     [key: string]: Partial<TableHandler>;
-  } & DbJoinMaker & {
-    sql?: (query: string, params?: any, options?: any) => Promise<any>;
-  };
+  } & 
+  DbJoinMaker & 
+  {
+    sql?: SQLHandler
+  } & {
+    tx?: TX
+  }
+  
+// <TXKey extends string = "tx"> 
+//   & {
+//     [K in TXKey]: TX
+//   };
+
+// const d: DbHandler = { } as any;
+// d.
+//   export type DbHandlerTX = { [key: string]: TX } | DbHandler;
+//   export type DbHandlerTX = DbHandler
+//    & Partial<{
+//     [key: string]: TX
+//   }>
 
 import { get } from "./utils";
 import { getNewQuery, makeQuery, COMPUTED_FIELDS, SelectItem, FieldSpec, asNameAlias, SelectItemBuilder, FUNCTIONS } from "./QueryBuilder";
@@ -40,7 +83,7 @@ export const pgp: PGP = pgPromise({
     // ,query: function (e) { console.log({psql: e.query, params: e.params}); }
 });
 
-export type TableInfo = {
+export type TableInfo = TInfo & {
     schema: string;
     name: string;
     oid: number;
@@ -57,14 +100,24 @@ export type TableOrViewInfo = TableInfo & ViewInfo & {
 }
 
 export type LocalParams = {
+    httpReq?: any;
     socket?: any;
     func?: () => any;
     has_rules?: boolean;
     testRule?: boolean;
     tableAlias?: string;
     // subOne?: boolean;
-    dbTX?: any;
+    dbTX?: TxHandler;
+
+    // localTX?: pgPromise.ITask<{}>;
+    localDBTX?: DbHandler;
+
     returnQuery?: boolean;
+
+    nestedJoin?: {
+        depth: number;
+        data: AnyObject;
+    }
 }
 function replaceNonAlphaNumeric(string: string): string {
     return string.replace(/[\W_]+/g,"_");
@@ -129,13 +182,14 @@ type Query = {
 }
 
 export type JoinInfo = { 
-    table: string, 
-    on: [[string, string]], 
     expectOne: boolean, 
-    source: string, 
-    target: string 
-}[]
-
+    paths: { 
+        table: string, 
+        on: [[string, string]],
+        source: string, 
+        target: string 
+    }[]
+}
 type JoinPaths = {
     t1: string;
     t2: string;
@@ -303,11 +357,14 @@ export class ViewHandler {
     joinGraph: Graph;
     joinPaths: JoinPaths;
     dboBuilder: DboBuilder;
+
     t: pgPromise.ITask<{}>;
+
     is_view: boolean = true;
     filterDef: string = "";
 
     pubSubManager: PubSubManager;
+    is_media: boolean = false;
     constructor(db: DB, tableOrViewInfo: TableOrViewInfo, pubSubManager: PubSubManager, dboBuilder: DboBuilder, t?: pgPromise.ITask<{}>, joinPaths?: JoinPaths){
         if(!db || !tableOrViewInfo) throw "";
 
@@ -334,7 +391,7 @@ export class ViewHandler {
         // if(this.tsDataName === "T") this.tsDataName = this.tsDataName + "_";
         // this.tsDataDef = `export type ${this.tsDataName} = {\n`;
         this.columns.map(({ name, udt_name }) => {
-            this.tsColumnDefs.push(`${escapeTSNames(name, false)}?: ${postgresToTsType(udt_name)};`);
+            this.tsColumnDefs.push(`${escapeTSNames(name, false)}?: ${postgresToTsType(udt_name) as string};`);
         });
         // this.tsDataDef += "};";
         // this.tsDataDef += "\n";
@@ -376,12 +433,15 @@ export class ViewHandler {
         return []
     }
 
-    async validateViewRules(fields: FieldFilter, filterFields: FieldFilter, returningFields: FieldFilter, forcedFilter: object, rule: string){
+    async validateViewRules(fields: FieldFilter, filterFields: FieldFilter, returningFields: FieldFilter, forcedFilter: object, rule: "update" | "select" | "insert" | "delete"){
 
         /* Safely test publish rules */
         if(fields) {
             try {
-                this.parseFieldFilter(fields);
+                const _fields = this.parseFieldFilter(fields);
+                if(this.is_media && rule === "insert" && !_fields.includes("id")){
+                    throw "Must allow id insert for media table"
+                }
             } catch(e){
                 throw ` issue with publish.${this.name}.${rule}.fields: \nVALUE: ` + JSON.stringify(fields, null, 2) + "\nERROR: " + JSON.stringify(e, null, 2);
             }
@@ -430,7 +490,7 @@ export class ViewHandler {
     }
 
     getJoins(source: string, target: string, path?: string[]): JoinInfo {
-        let result = [];
+        let paths: JoinInfo["paths"] = [];
 
         if(!this.joinPaths) throw "Joins dissallowed";
 
@@ -450,7 +510,7 @@ export class ViewHandler {
         if(!jp || !this.joinPaths.find(j => path? j.path.join() === path.join() : j.t1 === source && j.t2 === target))  throw `Joining ${source} <-...-> ${target} dissallowed or missing`;
 
         /* Make the join chain info excluding root table */
-        result = (path || jp.path).slice(1).map((t2, i, arr) => {
+        paths = (path || jp.path).slice(1).map((t2, i, arr) => {
             const t1 = i === 0? source : arr[i-1];
             
             if(!this.joins) this.joins = JSON.parse(JSON.stringify(this.dboBuilder.joins));
@@ -482,7 +542,22 @@ export class ViewHandler {
                 on
             };
         });
-        return result;
+        let expectOne = false; 
+        paths.map(({ source, target, on }, i) => {
+            // if(expectOne && on.length === 1){
+            //     const sourceCol = on[0][1];
+            //     const targetCol = on[0][0];
+                
+            //     const sCol = this.dboBuilder.dbo[source].columns.find(c => c.name === sourceCol)
+            //     const tCol = this.dboBuilder.dbo[target].columns.find(c => c.name === targetCol)
+            //     console.log({ sourceCol, targetCol, sCol, source, tCol, target, on})
+            //     expectOne = sCol.is_pkey && tCol.is_pkey
+            // }
+        })
+        return {
+            paths,
+            expectOne
+        };
     }
 
 
@@ -490,13 +565,38 @@ export class ViewHandler {
         if(filter === null || filter && !isPojoObject(filter)) throw `invalid filter -> ${JSON.stringify(filter)} \nExpecting:    undefined | {} | { field_name: "value" } | { field: { $gt: 22 } } ... `;
     }
 
-    async getInfo(param1, param2, param3, tableRules?: TableRule, localParams?: LocalParams): Promise<{ oid: number; comment: string; }>{
+    async getInfo(param1?, param2?, param3?, tableRules?: TableRule, localParams?: LocalParams): Promise<{ oid: number; comment: string; is_media: boolean; has_media?: "one" | "many" }>{
         const p = this.getValidatedRules(tableRules, localParams);
         if(!p.select.getInfo) throw "Not allowed";
 
+        let has_media = undefined;
+
+        const mediaTable = this.dboBuilder.prostgles?.opts?.fileTable?.tableName;
+        if(!this.is_media && mediaTable){
+            if(this.dboBuilder.prostgles?.opts?.fileTable?.referencedTables?.[this.name]){
+                has_media = this.dboBuilder.prostgles?.opts?.fileTable?.referencedTables?.[this.name];
+            } else {
+                const jp = this.dboBuilder.joinPaths.find(jp => jp.t1 === this.name && jp.t2 === mediaTable);
+                if(jp && jp.path.length < 4){
+                    await Promise.all(jp.path.map(async tableName => {
+                        const cols = (await this.dboBuilder.dbo[tableName].getColumns()).filter(c => jp.path.includes(c.references?.ftable));
+                        if(cols.length && has_media !== "many"){
+                            if(cols.find(c => !c.is_pkey)){
+                                has_media = "many"
+                            } else {
+                                has_media = "one"
+                            }
+                        }
+                    }));
+                }
+            }
+        }
+        
         return {
             oid: this.tableOrViewInfo.oid,
             comment: this.tableOrViewInfo.comment,
+            is_media: this.is_media,      // this.name === this.dboBuilder.prostgles?.opts?.fileTable?.tableName
+            has_media,
         }
     }
 
@@ -513,8 +613,8 @@ export class ViewHandler {
             let _lang = lang;
             return this.columns.map(c => {
                 let label = c.comment || c.name;
-                const iConf = this.dboBuilder.prostgles?.i18n?.column_labels?.[this.name]?.[c.name];
-                const fallbackLang = this.dboBuilder.prostgles?.i18n?.fallbackLang;
+                const iConf = this.dboBuilder.prostgles?.opts?.i18n?.column_labels?.[this.name]?.[c.name];
+                const fallbackLang = this.dboBuilder.prostgles?.opts?.i18n?.fallbackLang;
                 _lang = _lang || fallbackLang as string;
                 
                 if((lang || fallbackLang) && iConf){
@@ -680,9 +780,10 @@ export class ViewHandler {
     async find(filter?: Filter, selectParams?: SelectParams , param3_unused = null, tableRules?: TableRule, localParams?: LocalParams): Promise<any[]>{
         try {
             filter = filter || {};
+            const allowedReturnTypes: Array<SelectParams["returnType"]> = ["row", "value", "values"]
             const { returnType } = selectParams || {};
-            if(returnType && !["row", "value", "values"].includes(returnType)){
-                throw `returnType (${returnType}) can only be ${["row", "values"].join(" OR ")}`
+            if(returnType && !allowedReturnTypes.includes(returnType)){
+                throw `returnType (${returnType}) can only be ${allowedReturnTypes.join(" OR ")}`
             }
             
             const { testRule = false, returnQuery = false } = localParams || {};
@@ -715,7 +816,6 @@ export class ViewHandler {
 
             let q = await getNewQuery(this as unknown as TableHandler, filter, selectParams, param3_unused, tableRules, localParams),
                 _query = makeQuery(this as unknown as TableHandler, q, undefined, undefined, selectParams);
-            // console.log(_query)
             
             if(testRule){
                 try {
@@ -727,8 +827,6 @@ export class ViewHandler {
                 }
             }
 
-            // console.log(_query);
-
             if(returnQuery) return (_query as unknown as any[]);
             if(["row", "value"].includes(returnType)) {
                 return (this.t || this.db).oneOrNone(_query).then(data => {
@@ -926,21 +1024,23 @@ export class ViewHandler {
 
         const makeTableChain = (finalFilter: string) => {
 
-            let joinPaths: JoinInfo = [];
+            let joinPaths: JoinInfo["paths"] = [];
+            let expectOne = true;
             tables.map((t2, depth) => {
                 let t1 = depth? tables[depth - 1] : thisTable;
                 let exactPaths = [t1, t2];
 
                 if(!depth && eConfig.shortestJoin) exactPaths = undefined;
-                
-                joinPaths = joinPaths.concat(this.getJoins(t1, t2, exactPaths));
+                const jinf= this.getJoins(t1, t2, exactPaths);
+                expectOne = expectOne && jinf.expectOne
+                joinPaths = joinPaths.concat(jinf.paths);
             });
 
-            let r = makeJoin(joinPaths, 0);
-            // console.log(r);
+            let r = makeJoin({ paths: joinPaths, expectOne }, 0);
             return r;
             
-            function makeJoin(paths: JoinInfo, ji: number) {
+            function makeJoin(joinInfo: JoinInfo, ji: number) {
+                const { paths } = joinInfo;
                 const jp = paths[ji];
 
                 let prevTable = ji? paths[ji - 1].table : jp.source;
@@ -951,7 +1051,6 @@ export class ViewHandler {
                 let cond = `${jp.on.map(([c1, c2]) => 
                     `${prevTableAlias}.${asName(c1)} = ${tableAlias}.${asName(c2)}`).join("\n AND ")
                 }`;
-                // console.log(join, cond);
     
                 let j = `SELECT 1 \n` +
                         `FROM ${asName(table)} ${tableAlias} \n` +
@@ -966,7 +1065,7 @@ export class ViewHandler {
                 const indent = (a, b) => a;    
     
                 if(ji < paths.length - 1){
-                    j += `AND ${makeJoin(paths, ji + 1)} \n`
+                    j += `AND ${makeJoin(joinInfo, ji + 1)} \n`
                 }
 
                 j = indent(j, ji + 1);
@@ -985,9 +1084,9 @@ export class ViewHandler {
             tableAlias;
 
         /* Check if allowed to view data */
-        if(localParams && localParams.socket && this.dboBuilder.publishParser){
+        if(localParams && (localParams.socket || localParams.httpReq) && this.dboBuilder.publishParser){
             /* Need to think about joining through dissallowed tables */
-            t2Rules = await this.dboBuilder.publishParser.getValidatedRequestRuleWusr({ tableName: t2, command: "find", socket: localParams.socket });
+            t2Rules = await this.dboBuilder.publishParser.getValidatedRequestRuleWusr({ tableName: t2, command: "find", localParams });
             if(!t2Rules || !t2Rules.select) throw "Dissallowed";
             ({ forcedFilter, filterFields } = t2Rules.select);
         }
@@ -1001,13 +1100,13 @@ export class ViewHandler {
                 addKeywords: false, 
                 tableAlias, 
                 localParams, 
-                tableRule: tableRules
+                tableRule: t2Rules //tableRules
             }))
         } catch(err) {
             // console.trace(err)
             throw "Issue with preparing $exists query for table " + t2 + "\n->" + JSON.stringify(err);
         }
-        // console.log(f2, finalWhere);
+        
         if(!isJoined){
             res = ` EXISTS (SELECT 1 \nFROM ${asName(t2)} \n${finalWhere? `WHERE ${finalWhere}` : ""}) `
         } else {
@@ -1117,7 +1216,6 @@ export class ViewHandler {
                 return false;
             })
         }
-        // console.log(allowedSelect);
         
         /* Add remaining allowed fields */
         allowedSelect = allowedSelect.concat(
@@ -1169,7 +1267,7 @@ export class ViewHandler {
             pgp,
             select: allowedSelect
         });
-        // console.log({ f, q })
+        
         let templates: string[] = [q].filter(q=>q);
 
         if(existsCond) templates.push(existsCond);
@@ -1178,7 +1276,6 @@ export class ViewHandler {
         return templates.sort() /*  sorted to ensure duplicate subscription channels are not created due to different condition order */
             .join(" AND \n");
 
-        // console.log(templates)
         // return templates; //pgp.as.format(template, data);
                             
         /* 
@@ -1268,7 +1365,7 @@ export class ViewHandler {
         if(!_ob || !_ob.length) return "";
 
         const validatedAggAliases = select.filter(s => s.type !== "joinedColumn").map(s => s.alias)
-        // console.log({ validatedAggAliases })
+        
         let bad_param = _ob.find(({ key }) => 
             !(validatedAggAliases || []).includes(key) &&
             (
@@ -1529,6 +1626,7 @@ export class TableHandler extends ViewHandler {
             batching: null
         };
         this.is_view = false;
+        this.is_media = dboBuilder.prostgles.isMedia(this.name)
     }
 
     /* TO DO: Maybe finished query batching */
@@ -1641,7 +1739,7 @@ export class TableHandler extends ViewHandler {
                     )
                 )
             );
-            // console.log(queries)
+            
             return this.db.tx(t => {
                 const _queries = queries.map(q => t.none(q as unknown as string))
                 return t.batch(_queries)
@@ -1726,13 +1824,12 @@ export class TableHandler extends ViewHandler {
                     if(unrecProps.length) throw "Unrecognised params in textPatch field: " + unrecProps.join(", ");
                     patchedTextData.push({ ...d, fieldName: c.name });
                 }
-                // console.log("update2", patchedTextData);
             });
 
             if(patchedTextData && patchedTextData.length){
                 if(tableRules && !tableRules.select) throw "Select needs to be permitted to patch data";
                 const rows = await this.find(filter, { select: patchedTextData.reduce((a, v) => ({ ...a, [v.fieldName]: 1 }), {}) }, null, tableRules);
-                // console.log(rows)
+                
                 if(rows.length !== 1) {
                     throw "Cannot patch data within a filter that affects more/less than 1 row";
                 }
@@ -1761,10 +1858,9 @@ export class TableHandler extends ViewHandler {
             let qType = "none";
             if(returning){
                 qType = multi? "any" : "one";
-                query += await this.prepareReturning(returning, this.parseFieldFilter(returningFields));
+                query += this.makeReturnQuery(await this.prepareReturning(returning, this.parseFieldFilter(returningFields)));
             }
 
-            // console.log(query)
             if(returnQuery) return query as unknown as void;
             if(this.t){
                 return this.t[qType](query).catch(err => makeErr(err, localParams));
@@ -1795,7 +1891,248 @@ export class TableHandler extends ViewHandler {
         return { data, allowedCols: this.columns.filter(c => dataKeys.includes(c.name)).map(c => c.name) }
     }
     
-    async insert(data: (object | object[]), param2?: InsertParams, param3_unused?, tableRules?: TableRule, localParams: LocalParams = null): Promise<any | any[] | boolean>{
+
+    private async insertDataParse(data: (AnyObject | AnyObject[]), param2?: InsertParams, param3_unused?, tableRules?: TableRule, _localParams: LocalParams = null): Promise<{
+        data?: AnyObject | AnyObject[];
+        insertResult?: AnyObject | AnyObject[];
+    }>{
+        const localParams = _localParams || {};
+        const {dbTX} = localParams;
+
+        const isMultiInsert = Array.isArray(data);
+        const getExtraKeys = d => Object.keys(d).filter(k => !this.columns.find(c => c.name === k));
+
+        /* Nested insert is not allowed for the file table */
+        const isNestedInsert = this.is_media? false : (Array.isArray(data)? data : [data]).some(d => getExtraKeys(d).length);
+
+        /**
+         * Make sure nested insert uses a transaction
+         */
+        if(isNestedInsert && (!this.t || !dbTX)){
+            return {
+                insertResult: await this.dboBuilder.getTX((dbTX) => 
+                    (dbTX[this.name] as TableHandler).insert(
+                        data, 
+                        param2, 
+                        param3_unused, 
+                        tableRules, 
+                        {dbTX, ...localParams}
+                    )
+                )
+            }
+        }
+        
+        const preValidate = tableRules?.insert?.preValidate,
+            validate = tableRules?.insert?.validate;
+
+        let _data = await Promise.all((Array.isArray(data)? data : [data]).map(async row => {
+            if(preValidate){
+                row = await preValidate(row);
+            }
+
+            const dataKeys = Object.keys(row);
+            const extraKeys = getExtraKeys(row);
+
+            /* Upload file then continue insert */
+            if(this.is_media){
+                if(!this.dboBuilder.prostgles?.fileManager) throw "fileManager not set up";
+                const { data, name } = row;
+                
+                if(dataKeys.length !== 2) throw "Expecting only two properties: { name: string; data: File }";
+
+                // if(!Buffer.isBuffer(data)) throw "data is not of type Buffer"
+                if(!data) throw "data not provided"
+                if(typeof name !== "string"){
+                    throw "name is not of type string"
+                }
+
+                const media_id = (await this.db.oneOrNone("SELECT gen_random_uuid() as name")).name;
+                const type = await this.dboBuilder.prostgles.fileManager.getMIME(data, name)
+                const media_name = `${media_id}.${type.ext}`;
+                let media: Media = {
+                    id: media_id,
+                    name: media_name,
+                    original_name: name,
+                    extension: type.ext,
+                    content_type: type.mime
+                }
+
+                if(validate){
+                    media = await validate(media);
+                }
+
+                const _media: Media = await this.dboBuilder.prostgles.fileManager.uploadAsMedia({
+                    item: {
+                        data,
+                        name: media.name,
+                        content_type: media.content_type as any
+                    },
+                    // imageCompression: {
+                    //     inside: {
+                    //         width: 1100,
+                    //         height: 630
+                    //     }
+                    // }
+                });
+
+                return {
+                    ...media,
+                    ..._media,
+                };
+
+            /* Potentially a nested join */ 
+            } else if(extraKeys.length){
+
+                /* Ensure we're using the same transaction */
+                const _this = this.t? this : dbTX[this.name] as TableHandler;
+
+                let rootData = filterObj(data, null, extraKeys);
+
+                let insertedChildren: AnyObject[];
+                let targetTableRules: TableRule;
+
+                if(validate){
+                    rootData = await validate(rootData);
+                }
+
+                const fullRootResult = await _this.insert(rootData, { returning: "*" }, null, tableRules, localParams);
+                let returnData: AnyObject;
+                const returning = param2?.returning;
+                if(returning){
+                    returnData = {}
+                    const returningItems = await this.prepareReturning(returning, this.parseFieldFilter(tableRules?.insert?.returningFields));
+                    returningItems.filter(s => s.selected).map(rs => {
+                        returnData[rs.alias] = fullRootResult[rs.alias];
+                    })
+                }
+                
+                await Promise.all(extraKeys.map(async targetTable => {
+                    const childDataItems = Array.isArray(row[targetTable])? row[targetTable] : [row[targetTable]];
+                    // console.log({childDataItems})
+
+                    /* Must be allowed to insert into media table */
+                    const canInsert = async (tbl: string) => {
+                        const childRules = await this.dboBuilder.publishParser.getValidatedRequestRuleWusr({ tableName: tbl, command: "insert", localParams });
+                        if(!childRules || !childRules.insert) throw "Dissallowed nested insert into table " + childRules;
+                        return childRules;
+                    }
+
+                    // console.log(JSON.stringify(this.dboBuilder.joinPaths, null, 2))
+                    const jp = this.dboBuilder.joinPaths.find(jp => jp.t1 === this.name && jp.t2 === targetTable);
+                    if(!jp) throw `Could not find a valid table for the nested data { ${targetTable} } `;
+
+                    const childInsert = async (cdata, tableName) => {
+                        // console.log("childInsert", {data, tableName})
+                        if(!cdata || !dbTX[tableName] || !("insert" in dbTX[tableName])) throw "childInsertErr: Child table handler missing for: " + tableName;
+
+                        const tableRules = await canInsert(tableName);
+                        return Promise.all(
+                            (Array.isArray(cdata)? cdata : [cdata])
+                                .map(m => (dbTX[tableName] as TableHandler)
+                                .insert(m, { returning: "*" }, null, tableRules, localParams)
+                                .catch(e => {
+                                    console.trace({ childInsertErr: e })
+                                    return Promise.reject({ childInsertErr: e });
+                                })
+                            )
+                        );
+                    }
+                    
+                    const { path } = jp;
+                    const [tbl1, tbl2, tbl3] = path; 
+                    targetTableRules = await canInsert(targetTable);   //  tbl3
+
+                    const cols2 = this.dboBuilder.dbo[tbl2].columns || [];
+                    if(!this.dboBuilder.dbo[tbl2]) throw "Invalid/disallowed table: " + tbl2;
+                    const colsRefT1 = cols2?.filter(c => c.references?.cols.length === 1 && c.references?.ftable === tbl1);
+
+
+                    if(!path.length) {
+                        throw "Nested inserts join path not found for " + [this.name, targetTable];
+                    } else if(path.length === 2){
+                        if(targetTable !== tbl2) throw "Did not expect this";
+
+                        if(!colsRefT1.length) throw `Target table ${tbl2} does not reference any columns from the root table ${this.name}. Cannot do nested insert`;
+
+                        // console.log(JSON.stringify(colsRefT1, null, 2))
+                        insertedChildren = await childInsert(
+                            childDataItems.map(d => {
+                                let result = {...d};
+                                colsRefT1.map(col => {
+                                    result[col.references.cols[0]] = fullRootResult[col.references.fcols[0]]
+                                })
+                                return result;
+                            }), 
+                            targetTable
+                        );
+
+                    } else if(path.length === 3){
+                        if(targetTable !== tbl3) throw "Did not expect this";
+                        const colsRefT3 = cols2?.filter(c => c.references?.cols.length === 1 && c.references?.ftable === tbl3);
+                        if(!colsRefT1.length || !colsRefT3.length) throw "Incorrectly referenced or missing columns for nested insert";
+
+                        if(targetTable !== this.dboBuilder.prostgles.fileManager.tableName){
+                            throw "Only media allowed to have nested inserts more than 2 tables apart"
+                        }
+
+                        /* We expect tbl2 to have only 2 columns (media_id and foreign_id) */
+                        if(!cols2 || cols2.find(c => !["media_id", "foreign_id"].includes(c.name))){
+                            throw "Second joining table not of expected format";
+                        }
+
+                        insertedChildren = await childInsert(childDataItems, targetTable);
+                        
+                        /* Insert in key_lookup table */
+                        await Promise.all(insertedChildren.map(async t3Child => {
+                            let tbl2Row = {};
+
+                            colsRefT3.map(col => {
+                                tbl2Row[col.name] = t3Child[col.references.fcols[0]];
+                            })
+                            colsRefT1.map(col => {
+                                tbl2Row[col.name] = fullRootResult[col.references.fcols[0]];
+                            })
+                            // console.log({ rootResult, tbl2Row, t3Child, colsRefT3, colsRefT1, t: this.t?.ctx?.start });
+                            
+                            await childInsert(tbl2Row, tbl2);//.then(() => {});
+                        }));
+
+                    } else throw "Unexpected path for Nested inserts";
+
+                    /* Return also the nested inserted data */
+                    if(targetTableRules && insertedChildren?.length && returning){
+                        const targetTableHandler = dbTX[targetTable] as TableHandler;
+                        const targetReturning = await targetTableHandler.prepareReturning("*", targetTableHandler.parseFieldFilter(targetTableRules?.insert?.returningFields));
+                        let clientTargetInserts = insertedChildren.map(d => {
+                            let _d = { ...d };
+                            let res = {};
+                            targetReturning.map(r => {
+                                res[r.alias] = _d[r.alias]
+                            });
+                            return res;
+                        });
+    
+                        returnData[targetTable] = clientTargetInserts.length === 1? clientTargetInserts[0] : clientTargetInserts;
+                    }
+                }));
+
+                return returnData
+            }
+
+            return row;
+        }));
+
+        const result = isMultiInsert? _data : _data[0];
+        let res = isNestedInsert? 
+            { insertResult: result } : 
+            { data: result };
+            
+        return res;
+    }
+
+    async insert(rowOrRows: (AnyObject | AnyObject[]), param2?: InsertParams, param3_unused?, tableRules?: TableRule, _localParams: LocalParams = null): Promise<any | any[] | boolean>{
+        const localParams = _localParams || {};
+        const {dbTX} = localParams
         try {
 
             const { returning, onConflictDoNothing, fixIssues = false } = param2 || {};
@@ -1803,8 +2140,8 @@ export class TableHandler extends ViewHandler {
 
             let returningFields: FieldFilter,
                 forcedData: object,
-                validate: any,
-                preValidate: any,
+                // validate: TableRule["insert"]["validate"],
+                // preValidate: any,
                 fields: FieldFilter;
     
             if(tableRules){
@@ -1812,17 +2149,17 @@ export class TableHandler extends ViewHandler {
                 returningFields = tableRules.insert.returningFields;
                 forcedData = tableRules.insert.forcedData;
                 fields = tableRules.insert.fields;
-                validate = tableRules.insert.validate;
-                preValidate = tableRules.insert.preValidate;
+                // validate = tableRules.insert.validate;
+                // preValidate = tableRules.insert.preValidate;
     
                 /* If no returning fields specified then take select fields as returning */
-                if(!returningFields) returningFields = get(tableRules, "select.fields");
-                if(!returningFields) returningFields = get(tableRules, "insert.fields");
+                if(!returningFields) returningFields = get(tableRules, "select.fields") || get(tableRules, "insert.fields");
 
                 if(!fields) throw ` invalid insert rule for ${this.name} -> fields missing `;
 
                 /* Safely test publish rules */
                 if(testRule){
+                    // if(this.is_media && tableRules.insert.preValidate) throw "Media table cannot have a preValidate. It already is used internally by prostgles for file upload";
                     await this.validateViewRules(fields, null, returningFields, null, "insert");
                     if(forcedData) {
                         const keys = Object.keys(forcedData);
@@ -1836,6 +2173,7 @@ export class TableHandler extends ViewHandler {
                             }
                         }
                     }
+
                     return true;
                 }
             }
@@ -1844,43 +2182,50 @@ export class TableHandler extends ViewHandler {
             if(typeof onConflictDoNothing === "boolean" && onConflictDoNothing){
                 conflict_query = " ON CONFLICT DO NOTHING ";
             }
+
+            if(param2){
+                const good_params = ["returning", "multi", "onConflictDoNothing", "fixIssues"];
+                const bad_params = Object.keys(param2).filter(k => !good_params.includes(k));
+                if(bad_params && bad_params.length) throw "Invalid params: " + bad_params.join(", ") + " \n Expecting: " + good_params.join(", ");
+            }
             
-            if(!data) data = {}; //throw "Provide data in param1";
-            let returningSelect = await this.prepareReturning(returning, this.parseFieldFilter(returningFields));
+            if(!rowOrRows) rowOrRows = {}; //throw "Provide data in param1";
+            let returningSelect = this.makeReturnQuery(await this.prepareReturning(returning, this.parseFieldFilter(returningFields)));
             const makeQuery = async (_row, isOne = false) => {
                 let row = { ..._row };
-                if(preValidate){
-                    row = await preValidate(row);
+                
+                if(!isPojoObject(row)) {
+                    console.trace(row)
+                    throw "\ninvalid insert data provided -> " + JSON.stringify(row);
                 }
-                if(!isPojoObject(row)) throw "\ninvalid insert data provided -> " + JSON.stringify(row);
 
                 const { data, allowedCols } = this.validateNewData({ row, forcedData, allowedFields: fields, tableRules, fixIssues });
                 let _data = { ...data };
-                if(validate){
-                    _data = await validate(_data);
-                }
+
                 let insertQ = "";
                 if(!Object.keys(_data).length) insertQ = `INSERT INTO ${asName(this.name)} DEFAULT VALUES `;
                 else insertQ = this.colSet.getInsertQuery(_data, allowedCols) // pgp.helpers.insert(_data, columnSet); 
                 return insertQ + conflict_query + returningSelect;
             };
-
-
-            if(param2){
-                const good_params = ["returning", "multi", "onConflictDoNothing", "fixIssues"];
-                const bad_params = Object.keys(param2).filter(k => !good_params.includes(k));
-                if(bad_params && bad_params.length) throw "Invalid params: " + bad_params.join(", ") + " \n Expecting: " + good_params.join(", ");
-            }
     
             let query = "";
             let queryType = "none";
+            
+            /**
+             * If media it will: upload file and continue insert
+             * If nested insert it will: make separate inserts and not continue main insert
+             */
+            const { data, insertResult } = await this.insertDataParse(rowOrRows, param2, param3_unused, tableRules, localParams);
+            
+            if(insertResult) return insertResult;
+            
             if(Array.isArray(data)){
                 // if(returning) throw "Sorry but [returning] is dissalowed for multi insert";
                 let queries = await Promise.all(data.map(async p => {
                     const q = await makeQuery(p);
                     return q;
                 }));
-                // console.log(queries)
+                
                 query = pgp.helpers.concat(queries);
                 if(returning) queryType = "many";
             } else {
@@ -1888,18 +2233,27 @@ export class TableHandler extends ViewHandler {
                 if(returning) queryType = "one";
             }
             
-            // console.log(query);
             if(returnQuery)  return query;
-            if(this.t) return this.t[queryType](query).catch(err => makeErr(err, localParams));
-            return this.db.tx(t => t[queryType](query)).catch(err => makeErr(err, localParams));
+            let result;
+
+            // console.log(this.t?.ctx?.start, "insert in " + this.name, data);
+
+            const tx = dbTX?.[this.name]?.t || this.t;
+            if(tx) {
+                result = tx[queryType](query).catch(err => makeErr(err, localParams));
+            } else {
+                result = this.db.tx(t => t[queryType](query)).catch(err => makeErr(err, localParams));
+            }
+            
+            return result;
         } catch(e){
             if(localParams && localParams.testRule) throw e;
             throw { err: parseError(e), msg: `Issue with dbo.${this.name}.insert()` };
-        }
+        } 
     };
 
-    prepareReturning = async (returning: FieldFilter, allowedFields: string[], tableAlias?: string): Promise<string> => {
-        let result = "";
+    prepareReturning = async (returning: FieldFilter, allowedFields: string[]): Promise<SelectItem[]> => {
+        let result: SelectItem[] = [];
         if(returning){
             let sBuilder = new SelectItemBuilder({
                 allFields: this.column_names.slice(0),
@@ -1910,13 +2264,16 @@ export class TableHandler extends ViewHandler {
             });
             await sBuilder.parseUserSelect(returning);
 
-            if(sBuilder.select.length) result = "RETURNING ";
-
-            result += sBuilder.select.map(s => s.getQuery() + " AS " + asNameAlias(s.alias, tableAlias )).join(", ")
+            return sBuilder.select;
         } 
         
         return result;
     }
+
+    makeReturnQuery(items?: SelectItem[]){
+        if(items?.length) return " RETURNING " + items.map(s => s.getQuery() + " AS " + asName(s.alias)).join(", ");
+        return "";
+    }
     
     async delete(filter?: Filter, params?: DeleteParams, param3_unused?, table_rules?: TableRule, localParams: LocalParams = null): Promise<any> {    //{ socket, func, has_rules = false, socketDb } = {}
         try {
@@ -1972,7 +2329,7 @@ export class TableHandler extends ViewHandler {
                 if(!returningFields) {
                     throw "Returning dissallowed";
                 }
-                _query += await this.prepareReturning(returning, this.parseFieldFilter(returningFields));
+                _query += this.makeReturnQuery(await this.prepareReturning(returning, this.parseFieldFilter(returningFields)));
             }
             
             if(returnQuery) return _query;
@@ -2001,18 +2358,12 @@ export class TableHandler extends ViewHandler {
                 return tblH.find(filter, { select: "", limit: 1 }, {}, table_rules, localParams)
                     .then(exists => {
                         if(exists && exists.length){
-                            // console.log(filter, "exists");
                             return tblH.update(filter, newData, params, table_rules, localParams);
                         } else {
-                            // console.log(filter, "existnts")
                             return tblH.insert({ ...newData, ...filter }, params, null, table_rules, localParams);
                         }
                     });
             }
-                // .catch(existnts => {
-                //     console.log(filter, "existnts")
-                //     return this.insert({ ...filter, ...newData}, params);
-                // });
         } catch(e){
             if(localParams && localParams.testRule) throw e;
             throw { err: parseError(e), msg: `Issue with dbo.${this.name}.upsert()` };
@@ -2116,40 +2467,9 @@ export class TableHandler extends ViewHandler {
     }
 
 }
-type TransactionHandler = {
-    (): Promise<TxHandler>
-}
-export interface TxHandler {
-    [key: string]: TableHandler | ViewHandler;
-}
-export type TxCB = {
-    (t: TxHandler): (any | void);
-}
-export type TX = {
-    (t: TxCB): Promise<(any | void)>;
-}
-// export type JoinMaker = (filter?: object, select?: FieldFilter, options?: SelectParams) => any;
-
-// export type TableJoin = {
-//     [key: string]: JoinMaker;
-// }
-// export type DbJoinMaker = {
-//     innerJoin: TableJoin;
-//     leftJoin: TableJoin;
-//     innerJoinOne: TableJoin;
-//     leftJoinOne: TableJoin;
-// }
-
-// export type DbHandler = {
-//     [key: string]: TableHandler | ViewHandler;
-// } & DbJoinMaker;
-
-export type DbHandlerTX = { [key: string]: TX } | DbHandler;
 
 
 import { JOIN_TYPES } from "./Prostgles";
-import { type } from "os";
-import { join } from "path";
 
 export class DboBuilder {
     tablesOrViews: TableOrViewInfo[];
@@ -2157,7 +2477,8 @@ export class DboBuilder {
     db: DB;
     schema: string = "public";
 
-    dbo: DbHandler | DbHandlerTX;
+    // dbo: DbHandler | DbHandlerTX;
+    dbo: DbHandler;
     pubSubManager: PubSubManager;
 
     pojoDefinitions: string[];
@@ -2177,8 +2498,8 @@ export class DboBuilder {
     private constructor(prostgles: Prostgles){
         this.prostgles = prostgles;
         this.db = this.prostgles.db;
-        this.schema = this.prostgles.schema || "public";
-        this.dbo = { };
+        this.schema = this.prostgles.opts.schema || "public";
+        this.dbo = { } as unknown as DbHandler;
         // this.joins = this.prostgles.joins;
         
     }
@@ -2186,7 +2507,7 @@ export class DboBuilder {
     private init = async () => {
         let onSchemaChange;
         
-        if(this.prostgles.watchSchema){
+        if(this.prostgles.opts.watchSchema){
             onSchemaChange = (event: { command: string; query: string }) => { 
                 this.prostgles.onSchemaChange(event)
             }
@@ -2216,14 +2537,19 @@ export class DboBuilder {
     }
 
     async parseJoins(): Promise<JoinPaths> {
-        if(this.prostgles.joins){
-            let _joins = await this.prostgles.joins;
+        if(this.prostgles.opts.joins){
+            let _joins = await this.prostgles.opts.joins;
+            let inferredJoins = await getInferredJoins(this.db, this.prostgles.opts.schema);
             if(typeof _joins === "string" && _joins === "inferred"){
-                _joins = await getInferredJoins(this.db, this.prostgles.schema);
+                _joins = inferredJoins
+            /* If joins are specified then include inferred joins except the explicit tables */
+            } else if(Array.isArray(_joins)){
+                const joinTables = _joins.map(j => j.tables).flat();
+                _joins = _joins.concat(inferredJoins.filter(j => !j.tables.find(t => joinTables.includes(t))))
             }
             let joins = JSON.parse(JSON.stringify(_joins)) as Join[];
             this.joins = joins;
-            // console.log(joins);
+            
             // Validate joins
             try {
                 // 1 find duplicates
@@ -2302,9 +2628,6 @@ export class DboBuilder {
                     }
                 });
             });
-            // console.log(this.joinPaths)
-            // console.log(888, this.prostgles.joins);
-            // console.log(this.joinGraph, findShortestPath(this.joinGraph, "colors", "drawings"));
         }
 
         return this.joinPaths;
@@ -2314,7 +2637,7 @@ export class DboBuilder {
         
     }
 
-    async build(): Promise<DbHandler | DbHandlerTX>{
+    async build(): Promise<DbHandler>{
 
         // await this.pubSubManager.init()
 
@@ -2350,7 +2673,6 @@ export type TxCB = {
             i18nDef += `      [key in ${tov.columns.map(c => JSON.stringify(c.name)).join(" | ")}]: { [lang_id in keyof LANG_IDS]: string }; \n`;
             i18nDef += `    }; \n`;
 
-            // console.log("dboInit", tov.name, tov.columns.map(c => c.name))
             const filterKeywords = Object.values(this.prostgles.keywords);
             const $filterCol = tov.columns.find(c => filterKeywords.includes(c.name));
             if($filterCol){
@@ -2421,13 +2743,88 @@ export type TxCB = {
         }
 
 
-        if(this.prostgles.transactions){
+        if(this.prostgles.opts.transactions){
             let txKey = "tx";
-            if(typeof this.prostgles.transactions === "string") txKey = this.prostgles.transactions;
+            if(typeof this.prostgles.opts.transactions === "string") txKey = this.prostgles.opts.transactions;
             this.dboDefinition += ` ${txKey}: (t: TxCB) => Promise<any | void> ;\n`;
 
-            this.dbo[txKey] = (cb: TxCB) => this.getTX(cb);
+            (this.dbo[txKey] as TX) = (cb: TxCB) => this.getTX(cb);
+        }
+
+        if(!this.dbo.sql){
+
+            let needType = true;// this.publishRawSQL && typeof this.publishRawSQL === "function";
+            let DATA_TYPES = !needType? [] : await this.db.any("SELECT oid, typname FROM pg_type");
+            let USER_TABLES = !needType? [] :  await this.db.any("SELECT relid, relname FROM pg_catalog.pg_statio_user_tables");
+
+            this.dbo.sql = async (query: string, params: any, options: SQLOptions, localParams?: LocalParams) => {
+                const canRunSQL = async (localParams: LocalParams) => {
+                    if(!localParams) return true;
+
+                    const { socket } = localParams;
+                    const publishParams = await this.prostgles.publishParser.getPublishParams({ socket });
+                    let res = await this.prostgles.opts.publishRawSQL(publishParams);
+                    return Boolean(res && typeof res === "boolean" || res === "*");
+                }
+
+                if(!(await canRunSQL(localParams))) throw "Not allowed to run SQL";
+
+                const { returnType }: SQLOptions = options || ({} as any);
+                const { socket } = localParams || {};
+
+                if(returnType === "noticeSubscription"){
+                    if(!socket) throw "Only allowed with client socket"
+                    return await this.prostgles.dbEventsManager.addNotice(socket);
+                } else if(returnType === "statement"){
+                    try {
+                        return pgp.as.format(query, params);
+                    } catch (err){
+                        throw err.toString();
+                    }
+                } else if(this.db) {
+
+                    let qres = await this.db.result(query, params)
+                    const { duration, fields, rows, command } = qres;
+
+                    if(command === "LISTEN"){
+                        if(!socket) throw "Only allowed with client socket"
+                        return await this.prostgles.dbEventsManager.addNotify(query, socket);
+
+                    } else if(returnType === "rows") {
+                        return rows;
+                        
+                    } else if(returnType === "row") {
+                        return rows[0];
+                        
+                    } else if(returnType === "value") {
+                        return Object.values(rows?.[0] || {})?.[0];
+                        
+                    } else if(returnType === "values") {
+                        return rows.map(r => Object.values(r[0]));
+                        
+                    } else {
+                        if(fields && DATA_TYPES.length){
+                            qres.fields = fields.map(f => {
+                                const dataType = DATA_TYPES.find(dt => +dt.oid === +f.dataTypeID),
+                                    tableName = USER_TABLES.find(t => +t.relid === +f.tableID),
+                                    { name } = f;
+
+                                return {
+                                    ...f,
+                                    ...(dataType? { dataType: dataType.typname } : {}),
+                                    ...(tableName? { tableName: tableName.relname } : {}),
+                                }
+                            });
+                        }
+                        return qres;
+                    }
+
+                } else console.error("db missing");
+            }
+        } else {
+            console.warn(`Could not create dbo.sql handler because there is already a table named "sql"`)
         }
+
         this.dboDefinition += "};\n";
         
         this.tsTypesDefinition = [
@@ -2447,15 +2844,23 @@ export type TxCB = {
 
     getTX = (dbTX: TxCB) => {
         return this.db.tx((t) => {
-            let txDB = {};
+            let txDB: TxHandler = {};
             this.tablesOrViews.map(tov => {
                 if(tov.is_view){
+
                     txDB[tov.name] = new ViewHandler(this.db, tov, this.pubSubManager, this, t, this.joinPaths);
                 } else {
                     txDB[tov.name] = new TableHandler(this.db, tov, this.pubSubManager, this, t, this.joinPaths);
+                
+                    /**
+                     * Pass only the transaction object to ensure consistency
+                     */
+                    //     txDB[tov.name] = new ViewHandler(t, tov, this.pubSubManager, this, t, this.joinPaths);
+                    // } else {
+                    //     txDB[tov.name] = new TableHandler(t as any, tov, this.pubSubManager, this, t, this.joinPaths);
                 }
             });
-            return dbTX(txDB);
+            return dbTX(txDB, t);
         });
     }
 }
@@ -2888,6 +3293,6 @@ async function getInferredJoins(db: DB, schema: string = "public"): Promise<Join
             })
         }
     });
-    // console.log(joins);
+    
     return joins;
 }