prostgles-server 1.5.122 → 2.0.3

package/dist/DboBuilder.js

@@ -17,6 +17,17 @@ exports.isPlainObject = exports.DboBuilder = exports.TableHandler = exports.View
 const Bluebird = require("bluebird");
 const pgPromise = require("pg-promise");
 const prostgles_types_1 = require("prostgles-types");
+// <TXKey extends string = "tx"> 
+//   & {
+//     [K in TXKey]: TX
+//   };
+// const d: DbHandler = { } as any;
+// d.
+//   export type DbHandlerTX = { [key: string]: TX } | DbHandler;
+//   export type DbHandlerTX = DbHandler
+//    & Partial<{
+//     [key: string]: TX
+//   }>
 const utils_1 = require("./utils");
 const QueryBuilder_1 = require("./QueryBuilder");
 const Prostgles_1 = require("./Prostgles");
@@ -112,6 +123,7 @@ class ViewHandler {
         this.tsColumnDefs = [];
         this.is_view = true;
         this.filterDef = "";
+        this.is_media = false;
         if (!db || !tableOrViewInfo)
             throw "";
         this.db = db;
@@ -176,7 +188,10 @@ class ViewHandler {
             /* Safely test publish rules */
             if (fields) {
                 try {
-                    this.parseFieldFilter(fields);
+                    const _fields = this.parseFieldFilter(fields);
+                    if (this.is_media && rule === "insert" && !_fields.includes("id")) {
+                        throw "Must allow id insert for media table";
+                    }
                 }
                 catch (e) {
                     throw ` issue with publish.${this.name}.${rule}.fields: \nVALUE: ` + JSON.stringify(fields, null, 2) + "\nERROR: " + JSON.stringify(e, null, 2);
@@ -223,7 +238,7 @@ class ViewHandler {
         return { query, toOne: false };
     }
     getJoins(source, target, path) {
-        let result = [];
+        let paths = [];
         if (!this.joinPaths)
             throw "Joins dissallowed";
         if (path && !path.length)
@@ -243,7 +258,7 @@ class ViewHandler {
         if (!jp || !this.joinPaths.find(j => path ? j.path.join() === path.join() : j.t1 === source && j.t2 === target))
             throw `Joining ${source} <-...-> ${target} dissallowed or missing`;
         /* Make the join chain info excluding root table */
-        result = (path || jp.path).slice(1).map((t2, i, arr) => {
+        paths = (path || jp.path).slice(1).map((t2, i, arr) => {
             const t1 = i === 0 ? source : arr[i - 1];
             if (!this.joins)
                 this.joins = JSON.parse(JSON.stringify(this.dboBuilder.joins));
@@ -271,20 +286,60 @@ class ViewHandler {
                 on
             };
         });
-        return result;
+        let expectOne = false;
+        paths.map(({ source, target, on }, i) => {
+            // if(expectOne && on.length === 1){
+            //     const sourceCol = on[0][1];
+            //     const targetCol = on[0][0];
+            //     const sCol = this.dboBuilder.dbo[source].columns.find(c => c.name === sourceCol)
+            //     const tCol = this.dboBuilder.dbo[target].columns.find(c => c.name === targetCol)
+            //     console.log({ sourceCol, targetCol, sCol, source, tCol, target, on})
+            //     expectOne = sCol.is_pkey && tCol.is_pkey
+            // }
+        });
+        return {
+            paths,
+            expectOne
+        };
     }
     checkFilter(filter) {
         if (filter === null || filter && !isPojoObject(filter))
             throw `invalid filter -> ${JSON.stringify(filter)} \nExpecting:    undefined | {} | { field_name: "value" } | { field: { $gt: 22 } } ... `;
     }
     getInfo(param1, param2, param3, tableRules, localParams) {
+        var _a, _b, _c, _e, _f, _g, _h, _j, _k, _l, _m;
         return __awaiter(this, void 0, void 0, function* () {
             const p = this.getValidatedRules(tableRules, localParams);
             if (!p.select.getInfo)
                 throw "Not allowed";
+            let has_media = undefined;
+            const mediaTable = (_c = (_b = (_a = this.dboBuilder.prostgles) === null || _a === void 0 ? void 0 : _a.opts) === null || _b === void 0 ? void 0 : _b.fileTable) === null || _c === void 0 ? void 0 : _c.tableName;
+            if (!this.is_media && mediaTable) {
+                if ((_h = (_g = (_f = (_e = this.dboBuilder.prostgles) === null || _e === void 0 ? void 0 : _e.opts) === null || _f === void 0 ? void 0 : _f.fileTable) === null || _g === void 0 ? void 0 : _g.referencedTables) === null || _h === void 0 ? void 0 : _h[this.name]) {
+                    has_media = (_m = (_l = (_k = (_j = this.dboBuilder.prostgles) === null || _j === void 0 ? void 0 : _j.opts) === null || _k === void 0 ? void 0 : _k.fileTable) === null || _l === void 0 ? void 0 : _l.referencedTables) === null || _m === void 0 ? void 0 : _m[this.name];
+                }
+                else {
+                    const jp = this.dboBuilder.joinPaths.find(jp => jp.t1 === this.name && jp.t2 === mediaTable);
+                    if (jp && jp.path.length < 4) {
+                        yield Promise.all(jp.path.map((tableName) => __awaiter(this, void 0, void 0, function* () {
+                            const cols = (yield this.dboBuilder.dbo[tableName].getColumns()).filter(c => { var _a; return jp.path.includes((_a = c.references) === null || _a === void 0 ? void 0 : _a.ftable); });
+                            if (cols.length && has_media !== "many") {
+                                if (cols.find(c => !c.is_pkey)) {
+                                    has_media = "many";
+                                }
+                                else {
+                                    has_media = "one";
+                                }
+                            }
+                        })));
+                    }
+                }
+            }
             return {
                 oid: this.tableOrViewInfo.oid,
                 comment: this.tableOrViewInfo.comment,
+                is_media: this.is_media,
+                has_media,
             };
         });
     }
@@ -298,10 +353,10 @@ class ViewHandler {
                 // console.log("getColumns", this.name, this.columns.map(c => c.name))
                 let _lang = lang;
                 return this.columns.map(c => {
-                    var _a, _b, _c, _d, _e, _f;
+                    var _a, _b, _c, _e, _f, _g, _h, _j;
                     let label = c.comment || c.name;
-                    const iConf = (_d = (_c = (_b = (_a = this.dboBuilder.prostgles) === null || _a === void 0 ? void 0 : _a.i18n) === null || _b === void 0 ? void 0 : _b.column_labels) === null || _c === void 0 ? void 0 : _c[this.name]) === null || _d === void 0 ? void 0 : _d[c.name];
-                    const fallbackLang = (_f = (_e = this.dboBuilder.prostgles) === null || _e === void 0 ? void 0 : _e.i18n) === null || _f === void 0 ? void 0 : _f.fallbackLang;
+                    const iConf = (_f = (_e = (_c = (_b = (_a = this.dboBuilder.prostgles) === null || _a === void 0 ? void 0 : _a.opts) === null || _b === void 0 ? void 0 : _b.i18n) === null || _c === void 0 ? void 0 : _c.column_labels) === null || _e === void 0 ? void 0 : _e[this.name]) === null || _f === void 0 ? void 0 : _f[c.name];
+                    const fallbackLang = (_j = (_h = (_g = this.dboBuilder.prostgles) === null || _g === void 0 ? void 0 : _g.opts) === null || _h === void 0 ? void 0 : _h.i18n) === null || _j === void 0 ? void 0 : _j.fallbackLang;
                     _lang = _lang || fallbackLang;
                     if ((lang || fallbackLang) && iConf) {
                         let langLabel;
@@ -447,9 +502,10 @@ class ViewHandler {
         return __awaiter(this, void 0, void 0, function* () {
             try {
                 filter = filter || {};
+                const allowedReturnTypes = ["row", "value", "values"];
                 const { returnType } = selectParams || {};
-                if (returnType && !["row", "value", "values"].includes(returnType)) {
-                    throw `returnType (${returnType}) can only be ${["row", "values"].join(" OR ")}`;
+                if (returnType && !allowedReturnTypes.includes(returnType)) {
+                    throw `returnType (${returnType}) can only be ${allowedReturnTypes.join(" OR ")}`;
                 }
                 const { testRule = false, returnQuery = false } = localParams || {};
                 if (testRule)
@@ -477,7 +533,6 @@ class ViewHandler {
                         throw ` invalid publish.${this.name}.select.maxLimit -> expecting integer but got ` + maxLimit;
                 }
                 let q = yield QueryBuilder_1.getNewQuery(this, filter, selectParams, param3_unused, tableRules, localParams), _query = QueryBuilder_1.makeQuery(this, q, undefined, undefined, selectParams);
-                // console.log(_query)
                 if (testRule) {
                     try {
                         yield this.db.any("EXPLAIN " + _query);
@@ -488,7 +543,6 @@ class ViewHandler {
                         throw `INTERNAL ERROR: Publish config is not valid for publish.${this.name}.select `;
                     }
                 }
-                // console.log(_query);
                 if (returnQuery)
                     return _query;
                 if (["row", "value"].includes(returnType)) {
@@ -667,24 +721,26 @@ class ViewHandler {
             }
             const makeTableChain = (finalFilter) => {
                 let joinPaths = [];
+                let expectOne = true;
                 tables.map((t2, depth) => {
                     let t1 = depth ? tables[depth - 1] : thisTable;
                     let exactPaths = [t1, t2];
                     if (!depth && eConfig.shortestJoin)
                         exactPaths = undefined;
-                    joinPaths = joinPaths.concat(this.getJoins(t1, t2, exactPaths));
+                    const jinf = this.getJoins(t1, t2, exactPaths);
+                    expectOne = expectOne && jinf.expectOne;
+                    joinPaths = joinPaths.concat(jinf.paths);
                 });
-                let r = makeJoin(joinPaths, 0);
-                // console.log(r);
+                let r = makeJoin({ paths: joinPaths, expectOne }, 0);
                 return r;
-                function makeJoin(paths, ji) {
+                function makeJoin(joinInfo, ji) {
+                    const { paths } = joinInfo;
                     const jp = paths[ji];
                     let prevTable = ji ? paths[ji - 1].table : jp.source;
                     let table = paths[ji].table;
                     let tableAlias = prostgles_types_1.asName(ji < paths.length - 1 ? `jd${ji}` : table);
                     let prevTableAlias = prostgles_types_1.asName(ji ? `jd${ji - 1}` : thisTable);
                     let cond = `${jp.on.map(([c1, c2]) => `${prevTableAlias}.${prostgles_types_1.asName(c1)} = ${tableAlias}.${prostgles_types_1.asName(c2)}`).join("\n AND ")}`;
-                    // console.log(join, cond);
                     let j = `SELECT 1 \n` +
                         `FROM ${prostgles_types_1.asName(table)} ${tableAlias} \n` +
                         `WHERE ${cond} \n`; //
@@ -694,7 +750,7 @@ class ViewHandler {
                     }
                     const indent = (a, b) => a;
                     if (ji < paths.length - 1) {
-                        j += `AND ${makeJoin(paths, ji + 1)} \n`;
+                        j += `AND ${makeJoin(joinInfo, ji + 1)} \n`;
                     }
                     j = indent(j, ji + 1);
                     let res = `EXISTS ( \n` +
@@ -705,9 +761,9 @@ class ViewHandler {
             };
             let t2Rules = undefined, forcedFilter, filterFields, tableAlias;
             /* Check if allowed to view data */
-            if (localParams && localParams.socket && this.dboBuilder.publishParser) {
+            if (localParams && (localParams.socket || localParams.httpReq) && this.dboBuilder.publishParser) {
                 /* Need to think about joining through dissallowed tables */
-                t2Rules = yield this.dboBuilder.publishParser.getValidatedRequestRuleWusr({ tableName: t2, command: "find", socket: localParams.socket });
+                t2Rules = yield this.dboBuilder.publishParser.getValidatedRequestRuleWusr({ tableName: t2, command: "find", localParams });
                 if (!t2Rules || !t2Rules.select)
                     throw "Dissallowed";
                 ({ forcedFilter, filterFields } = t2Rules.select);
@@ -721,14 +777,13 @@ class ViewHandler {
                     addKeywords: false,
                     tableAlias,
                     localParams,
-                    tableRule: tableRules
+                    tableRule: t2Rules //tableRules
                 }));
             }
             catch (err) {
                 // console.trace(err)
                 throw "Issue with preparing $exists query for table " + t2 + "\n->" + JSON.stringify(err);
             }
-            // console.log(f2, finalWhere);
             if (!isJoined) {
                 res = ` EXISTS (SELECT 1 \nFROM ${prostgles_types_1.asName(t2)} \n${finalWhere ? `WHERE ${finalWhere}` : ""}) `;
             }
@@ -823,7 +878,6 @@ class ViewHandler {
                     return false;
                 });
             }
-            // console.log(allowedSelect);
             /* Add remaining allowed fields */
             allowedSelect = allowedSelect.concat(p.allColumns.filter(c => allowed_colnames.includes(c.name) &&
                 !allowedSelect.find(s => s.alias === c.name)).map(f => ({
@@ -858,14 +912,12 @@ class ViewHandler {
                 pgp: exports.pgp,
                 select: allowedSelect
             });
-            // console.log({ f, q })
             let templates = [q].filter(q => q);
             if (existsCond)
                 templates.push(existsCond);
             templates = templates.concat(computedColConditions);
             return templates.sort() /*  sorted to ensure duplicate subscription channels are not created due to different condition order */
                 .join(" AND \n");
-            // console.log(templates)
             // return templates; //pgp.as.format(template, data);
             /*
                 SHOULD CHECK DATA TYPES TO AVOID "No operator matches the given data type" error
@@ -954,7 +1006,6 @@ class ViewHandler {
         if (!_ob || !_ob.length)
             return "";
         const validatedAggAliases = select.filter(s => s.type !== "joinedColumn").map(s => s.alias);
-        // console.log({ validatedAggAliases })
         let bad_param = _ob.find(({ key }) => !(validatedAggAliases || []).includes(key) &&
             (!column_names.includes(key) ||
                 (allowedFields.length && !allowedFields.includes(key))));
@@ -1155,8 +1206,8 @@ function isPojoObject(obj) {
 class TableHandler extends ViewHandler {
     constructor(db, tableOrViewInfo, pubSubManager, dboBuilder, t, joinPaths) {
         super(db, tableOrViewInfo, pubSubManager, dboBuilder, t, joinPaths);
-        this.prepareReturning = (returning, allowedFields, tableAlias) => __awaiter(this, void 0, void 0, function* () {
-            let result = "";
+        this.prepareReturning = (returning, allowedFields) => __awaiter(this, void 0, void 0, function* () {
+            let result = [];
             if (returning) {
                 let sBuilder = new QueryBuilder_1.SelectItemBuilder({
                     allFields: this.column_names.slice(0),
@@ -1166,9 +1217,7 @@ class TableHandler extends ViewHandler {
                     isView: this.is_view
                 });
                 yield sBuilder.parseUserSelect(returning);
-                if (sBuilder.select.length)
-                    result = "RETURNING ";
-                result += sBuilder.select.map(s => s.getQuery() + " AS " + QueryBuilder_1.asNameAlias(s.alias, tableAlias)).join(", ");
+                return sBuilder.select;
             }
             return result;
         });
@@ -1180,6 +1229,7 @@ class TableHandler extends ViewHandler {
             batching: null
         };
         this.is_view = false;
+        this.is_media = dboBuilder.prostgles.isMedia(this.name);
     }
     /* TO DO: Maybe finished query batching */
     willBatch(query) {
@@ -1269,7 +1319,6 @@ class TableHandler extends ViewHandler {
                 const queries = yield Promise.all(data.map(([filter, data]) => __awaiter(this, void 0, void 0, function* () {
                     return yield this.update(filter, data, Object.assign(Object.assign({}, (params || {})), { returning: undefined }), tableRules, Object.assign(Object.assign({}, (localParams || {})), { returnQuery: true }));
                 })));
-                // console.log(queries)
                 return this.db.tx(t => {
                     const _queries = queries.map(q => t.none(q));
                     return t.batch(_queries);
@@ -1341,13 +1390,11 @@ class TableHandler extends ViewHandler {
                             throw "Unrecognised params in textPatch field: " + unrecProps.join(", ");
                         patchedTextData.push(Object.assign(Object.assign({}, d), { fieldName: c.name }));
                     }
-                    // console.log("update2", patchedTextData);
                 });
                 if (patchedTextData && patchedTextData.length) {
                     if (tableRules && !tableRules.select)
                         throw "Select needs to be permitted to patch data";
                     const rows = yield this.find(filter, { select: patchedTextData.reduce((a, v) => (Object.assign(Object.assign({}, a), { [v.fieldName]: 1 })), {}) }, null, tableRules);
-                    // console.log(rows)
                     if (rows.length !== 1) {
                         throw "Cannot patch data within a filter that affects more/less than 1 row";
                     }
@@ -1374,9 +1421,8 @@ class TableHandler extends ViewHandler {
                 let qType = "none";
                 if (returning) {
                     qType = multi ? "any" : "one";
-                    query += yield this.prepareReturning(returning, this.parseFieldFilter(returningFields));
+                    query += this.makeReturnQuery(yield this.prepareReturning(returning, this.parseFieldFilter(returningFields)));
                 }
-                // console.log(query)
                 if (returnQuery)
                     return query;
                 if (this.t) {
@@ -1406,29 +1452,221 @@ class TableHandler extends ViewHandler {
         // let cs = new pgp.helpers.ColumnSet(this.columnSet.columns.filter(c => dataKeys.includes(c.name)), { table: this.name });
         return { data, allowedCols: this.columns.filter(c => dataKeys.includes(c.name)).map(c => c.name) };
     }
-    insert(data, param2, param3_unused, tableRules, localParams = null) {
+    insertDataParse(data, param2, param3_unused, tableRules, _localParams = null) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            const localParams = _localParams || {};
+            const { dbTX } = localParams;
+            const isMultiInsert = Array.isArray(data);
+            const getExtraKeys = d => Object.keys(d).filter(k => !this.columns.find(c => c.name === k));
+            /* Nested insert is not allowed for the file table */
+            const isNestedInsert = this.is_media ? false : (Array.isArray(data) ? data : [data]).some(d => getExtraKeys(d).length);
+            /**
+             * Make sure nested insert uses a transaction
+             */
+            if (isNestedInsert && (!this.t || !dbTX)) {
+                return {
+                    insertResult: yield this.dboBuilder.getTX((dbTX) => dbTX[this.name].insert(data, param2, param3_unused, tableRules, Object.assign({ dbTX }, localParams)))
+                };
+            }
+            const preValidate = (_a = tableRules === null || tableRules === void 0 ? void 0 : tableRules.insert) === null || _a === void 0 ? void 0 : _a.preValidate, validate = (_b = tableRules === null || tableRules === void 0 ? void 0 : tableRules.insert) === null || _b === void 0 ? void 0 : _b.validate;
+            let _data = yield Promise.all((Array.isArray(data) ? data : [data]).map((row) => __awaiter(this, void 0, void 0, function* () {
+                var _c, _e;
+                if (preValidate) {
+                    row = yield preValidate(row);
+                }
+                const dataKeys = Object.keys(row);
+                const extraKeys = getExtraKeys(row);
+                /* Upload file then continue insert */
+                if (this.is_media) {
+                    if (!((_c = this.dboBuilder.prostgles) === null || _c === void 0 ? void 0 : _c.fileManager))
+                        throw "fileManager not set up";
+                    const { data, name } = row;
+                    if (dataKeys.length !== 2)
+                        throw "Expecting only two properties: { name: string; data: File }";
+                    // if(!Buffer.isBuffer(data)) throw "data is not of type Buffer"
+                    if (!data)
+                        throw "data not provided";
+                    if (typeof name !== "string") {
+                        throw "name is not of type string";
+                    }
+                    const media_id = (yield this.db.oneOrNone("SELECT gen_random_uuid() as name")).name;
+                    const type = yield this.dboBuilder.prostgles.fileManager.getMIME(data, name);
+                    const media_name = `${media_id}.${type.ext}`;
+                    let media = {
+                        id: media_id,
+                        name: media_name,
+                        original_name: name,
+                        extension: type.ext,
+                        content_type: type.mime
+                    };
+                    if (validate) {
+                        media = yield validate(media);
+                    }
+                    const _media = yield this.dboBuilder.prostgles.fileManager.uploadAsMedia({
+                        item: {
+                            data,
+                            name: media.name,
+                            content_type: media.content_type
+                        },
+                    });
+                    return Object.assign(Object.assign({}, media), _media);
+                    /* Potentially a nested join */
+                }
+                else if (extraKeys.length) {
+                    /* Ensure we're using the same transaction */
+                    const _this = this.t ? this : dbTX[this.name];
+                    let rootData = PubSubManager_1.filterObj(data, null, extraKeys);
+                    let insertedChildren;
+                    let targetTableRules;
+                    if (validate) {
+                        rootData = yield validate(rootData);
+                    }
+                    const fullRootResult = yield _this.insert(rootData, { returning: "*" }, null, tableRules, localParams);
+                    let returnData;
+                    const returning = param2 === null || param2 === void 0 ? void 0 : param2.returning;
+                    if (returning) {
+                        returnData = {};
+                        const returningItems = yield this.prepareReturning(returning, this.parseFieldFilter((_e = tableRules === null || tableRules === void 0 ? void 0 : tableRules.insert) === null || _e === void 0 ? void 0 : _e.returningFields));
+                        returningItems.filter(s => s.selected).map(rs => {
+                            returnData[rs.alias] = fullRootResult[rs.alias];
+                        });
+                    }
+                    yield Promise.all(extraKeys.map((targetTable) => __awaiter(this, void 0, void 0, function* () {
+                        var _f;
+                        const childDataItems = Array.isArray(row[targetTable]) ? row[targetTable] : [row[targetTable]];
+                        // console.log({childDataItems})
+                        /* Must be allowed to insert into media table */
+                        const canInsert = (tbl) => __awaiter(this, void 0, void 0, function* () {
+                            const childRules = yield this.dboBuilder.publishParser.getValidatedRequestRuleWusr({ tableName: tbl, command: "insert", localParams });
+                            if (!childRules || !childRules.insert)
+                                throw "Dissallowed nested insert into table " + childRules;
+                            return childRules;
+                        });
+                        // console.log(JSON.stringify(this.dboBuilder.joinPaths, null, 2))
+                        const jp = this.dboBuilder.joinPaths.find(jp => jp.t1 === this.name && jp.t2 === targetTable);
+                        if (!jp)
+                            throw `Could not find a valid table for the nested data { ${targetTable} } `;
+                        const childInsert = (cdata, tableName) => __awaiter(this, void 0, void 0, function* () {
+                            // console.log("childInsert", {data, tableName})
+                            if (!cdata || !dbTX[tableName] || !("insert" in dbTX[tableName]))
+                                throw "childInsertErr: Child table handler missing for: " + tableName;
+                            const tableRules = yield canInsert(tableName);
+                            return Promise.all((Array.isArray(cdata) ? cdata : [cdata])
+                                .map(m => dbTX[tableName]
+                                .insert(m, { returning: "*" }, null, tableRules, localParams)
+                                .catch(e => {
+                                console.trace({ childInsertErr: e });
+                                return Promise.reject({ childInsertErr: e });
+                            })));
+                        });
+                        const { path } = jp;
+                        const [tbl1, tbl2, tbl3] = path;
+                        targetTableRules = yield canInsert(targetTable); //  tbl3
+                        const cols2 = this.dboBuilder.dbo[tbl2].columns || [];
+                        if (!this.dboBuilder.dbo[tbl2])
+                            throw "Invalid/disallowed table: " + tbl2;
+                        const colsRefT1 = cols2 === null || cols2 === void 0 ? void 0 : cols2.filter(c => { var _a, _b; return ((_a = c.references) === null || _a === void 0 ? void 0 : _a.cols.length) === 1 && ((_b = c.references) === null || _b === void 0 ? void 0 : _b.ftable) === tbl1; });
+                        if (!path.length) {
+                            throw "Nested inserts join path not found for " + [this.name, targetTable];
+                        }
+                        else if (path.length === 2) {
+                            if (targetTable !== tbl2)
+                                throw "Did not expect this";
+                            if (!colsRefT1.length)
+                                throw `Target table ${tbl2} does not reference any columns from the root table ${this.name}. Cannot do nested insert`;
+                            // console.log(JSON.stringify(colsRefT1, null, 2))
+                            insertedChildren = yield childInsert(childDataItems.map(d => {
+                                let result = Object.assign({}, d);
+                                colsRefT1.map(col => {
+                                    result[col.references.cols[0]] = fullRootResult[col.references.fcols[0]];
+                                });
+                                return result;
+                            }), targetTable);
+                        }
+                        else if (path.length === 3) {
+                            if (targetTable !== tbl3)
+                                throw "Did not expect this";
+                            const colsRefT3 = cols2 === null || cols2 === void 0 ? void 0 : cols2.filter(c => { var _a, _b; return ((_a = c.references) === null || _a === void 0 ? void 0 : _a.cols.length) === 1 && ((_b = c.references) === null || _b === void 0 ? void 0 : _b.ftable) === tbl3; });
+                            if (!colsRefT1.length || !colsRefT3.length)
+                                throw "Incorrectly referenced or missing columns for nested insert";
+                            if (targetTable !== this.dboBuilder.prostgles.fileManager.tableName) {
+                                throw "Only media allowed to have nested inserts more than 2 tables apart";
+                            }
+                            /* We expect tbl2 to have only 2 columns (media_id and foreign_id) */
+                            if (!cols2 || cols2.find(c => !["media_id", "foreign_id"].includes(c.name))) {
+                                throw "Second joining table not of expected format";
+                            }
+                            insertedChildren = yield childInsert(childDataItems, targetTable);
+                            /* Insert in key_lookup table */
+                            yield Promise.all(insertedChildren.map((t3Child) => __awaiter(this, void 0, void 0, function* () {
+                                let tbl2Row = {};
+                                colsRefT3.map(col => {
+                                    tbl2Row[col.name] = t3Child[col.references.fcols[0]];
+                                });
+                                colsRefT1.map(col => {
+                                    tbl2Row[col.name] = fullRootResult[col.references.fcols[0]];
+                                });
+                                // console.log({ rootResult, tbl2Row, t3Child, colsRefT3, colsRefT1, t: this.t?.ctx?.start });
+                                yield childInsert(tbl2Row, tbl2); //.then(() => {});
+                            })));
+                        }
+                        else
+                            throw "Unexpected path for Nested inserts";
+                        /* Return also the nested inserted data */
+                        if (targetTableRules && (insertedChildren === null || insertedChildren === void 0 ? void 0 : insertedChildren.length) && returning) {
+                            const targetTableHandler = dbTX[targetTable];
+                            const targetReturning = yield targetTableHandler.prepareReturning("*", targetTableHandler.parseFieldFilter((_f = targetTableRules === null || targetTableRules === void 0 ? void 0 : targetTableRules.insert) === null || _f === void 0 ? void 0 : _f.returningFields));
+                            let clientTargetInserts = insertedChildren.map(d => {
+                                let _d = Object.assign({}, d);
+                                let res = {};
+                                targetReturning.map(r => {
+                                    res[r.alias] = _d[r.alias];
+                                });
+                                return res;
+                            });
+                            returnData[targetTable] = clientTargetInserts.length === 1 ? clientTargetInserts[0] : clientTargetInserts;
+                        }
+                    })));
+                    return returnData;
+                }
+                return row;
+            })));
+            const result = isMultiInsert ? _data : _data[0];
+            let res = isNestedInsert ?
+                { insertResult: result } :
+                { data: result };
+            return res;
+        });
+    }
+    insert(rowOrRows, param2, param3_unused, tableRules, _localParams = null) {
+        var _a;
         return __awaiter(this, void 0, void 0, function* () {
+            const localParams = _localParams || {};
+            const { dbTX } = localParams;
             try {
                 const { returning, onConflictDoNothing, fixIssues = false } = param2 || {};
                 const { testRule = false, returnQuery = false } = localParams || {};
-                let returningFields, forcedData, validate, preValidate, fields;
+                let returningFields, forcedData, 
+                // validate: TableRule["insert"]["validate"],
+                // preValidate: any,
+                fields;
                 if (tableRules) {
                     if (!tableRules.insert)
                         throw "insert rules missing for " + this.name;
                     returningFields = tableRules.insert.returningFields;
                     forcedData = tableRules.insert.forcedData;
                     fields = tableRules.insert.fields;
-                    validate = tableRules.insert.validate;
-                    preValidate = tableRules.insert.preValidate;
+                    // validate = tableRules.insert.validate;
+                    // preValidate = tableRules.insert.preValidate;
                     /* If no returning fields specified then take select fields as returning */
                     if (!returningFields)
-                        returningFields = utils_1.get(tableRules, "select.fields");
-                    if (!returningFields)
-                        returningFields = utils_1.get(tableRules, "insert.fields");
+                        returningFields = utils_1.get(tableRules, "select.fields") || utils_1.get(tableRules, "insert.fields");
                     if (!fields)
                         throw ` invalid insert rule for ${this.name} -> fields missing `;
                     /* Safely test publish rules */
                     if (testRule) {
+                        // if(this.is_media && tableRules.insert.preValidate) throw "Media table cannot have a preValidate. It already is used internally by prostgles for file upload";
                         yield this.validateViewRules(fields, null, returningFields, null, "insert");
                         if (forcedData) {
                             const keys = Object.keys(forcedData);
@@ -1449,21 +1687,23 @@ class TableHandler extends ViewHandler {
                 if (typeof onConflictDoNothing === "boolean" && onConflictDoNothing) {
                     conflict_query = " ON CONFLICT DO NOTHING ";
                 }
-                if (!data)
-                    data = {}; //throw "Provide data in param1";
-                let returningSelect = yield this.prepareReturning(returning, this.parseFieldFilter(returningFields));
+                if (param2) {
+                    const good_params = ["returning", "multi", "onConflictDoNothing", "fixIssues"];
+                    const bad_params = Object.keys(param2).filter(k => !good_params.includes(k));
+                    if (bad_params && bad_params.length)
+                        throw "Invalid params: " + bad_params.join(", ") + " \n Expecting: " + good_params.join(", ");
+                }
+                if (!rowOrRows)
+                    rowOrRows = {}; //throw "Provide data in param1";
+                let returningSelect = this.makeReturnQuery(yield this.prepareReturning(returning, this.parseFieldFilter(returningFields)));
                 const makeQuery = (_row, isOne = false) => __awaiter(this, void 0, void 0, function* () {
                     let row = Object.assign({}, _row);
-                    if (preValidate) {
-                        row = yield preValidate(row);
-                    }
-                    if (!isPojoObject(row))
+                    if (!isPojoObject(row)) {
+                        console.trace(row);
                         throw "\ninvalid insert data provided -> " + JSON.stringify(row);
+                    }
                     const { data, allowedCols } = this.validateNewData({ row, forcedData, allowedFields: fields, tableRules, fixIssues });
                     let _data = Object.assign({}, data);
-                    if (validate) {
-                        _data = yield validate(_data);
-                    }
                     let insertQ = "";
                     if (!Object.keys(_data).length)
                         insertQ = `INSERT INTO ${prostgles_types_1.asName(this.name)} DEFAULT VALUES `;
@@ -1471,21 +1711,21 @@ class TableHandler extends ViewHandler {
                         insertQ = this.colSet.getInsertQuery(_data, allowedCols); // pgp.helpers.insert(_data, columnSet); 
                     return insertQ + conflict_query + returningSelect;
                 });
-                if (param2) {
-                    const good_params = ["returning", "multi", "onConflictDoNothing", "fixIssues"];
-                    const bad_params = Object.keys(param2).filter(k => !good_params.includes(k));
-                    if (bad_params && bad_params.length)
-                        throw "Invalid params: " + bad_params.join(", ") + " \n Expecting: " + good_params.join(", ");
-                }
                 let query = "";
                 let queryType = "none";
+                /**
+                 * If media it will: upload file and continue insert
+                 * If nested insert it will: make separate inserts and not continue main insert
+                 */
+                const { data, insertResult } = yield this.insertDataParse(rowOrRows, param2, param3_unused, tableRules, localParams);
+                if (insertResult)
+                    return insertResult;
                 if (Array.isArray(data)) {
                     // if(returning) throw "Sorry but [returning] is dissalowed for multi insert";
                     let queries = yield Promise.all(data.map((p) => __awaiter(this, void 0, void 0, function* () {
                         const q = yield makeQuery(p);
                         return q;
                     })));
-                    // console.log(queries)
                     query = exports.pgp.helpers.concat(queries);
                     if (returning)
                         queryType = "many";
@@ -1495,12 +1735,18 @@ class TableHandler extends ViewHandler {
                     if (returning)
                         queryType = "one";
                 }
-                // console.log(query);
                 if (returnQuery)
                     return query;
-                if (this.t)
-                    return this.t[queryType](query).catch(err => makeErr(err, localParams));
-                return this.db.tx(t => t[queryType](query)).catch(err => makeErr(err, localParams));
+                let result;
+                // console.log(this.t?.ctx?.start, "insert in " + this.name, data);
+                const tx = ((_a = dbTX === null || dbTX === void 0 ? void 0 : dbTX[this.name]) === null || _a === void 0 ? void 0 : _a.t) || this.t;
+                if (tx) {
+                    result = tx[queryType](query).catch(err => makeErr(err, localParams));
+                }
+                else {
+                    result = this.db.tx(t => t[queryType](query)).catch(err => makeErr(err, localParams));
+                }
+                return result;
             }
             catch (e) {
                 if (localParams && localParams.testRule)
@@ -1510,6 +1756,11 @@ class TableHandler extends ViewHandler {
         });
     }
     ;
+    makeReturnQuery(items) {
+        if (items === null || items === void 0 ? void 0 : items.length)
+            return " RETURNING " + items.map(s => s.getQuery() + " AS " + prostgles_types_1.asName(s.alias)).join(", ");
+        return "";
+    }
     delete(filter, params, param3_unused, table_rules, localParams = null) {
         return __awaiter(this, void 0, void 0, function* () {
             try {
@@ -1557,7 +1808,7 @@ class TableHandler extends ViewHandler {
                     if (!returningFields) {
                         throw "Returning dissallowed";
                     }
-                    _query += yield this.prepareReturning(returning, this.parseFieldFilter(returningFields));
+                    _query += this.makeReturnQuery(yield this.prepareReturning(returning, this.parseFieldFilter(returningFields)));
                 }
                 if (returnQuery)
                     return _query;
@@ -1590,20 +1841,14 @@ class TableHandler extends ViewHandler {
                         return tblH.find(filter, { select: "", limit: 1 }, {}, table_rules, localParams)
                             .then(exists => {
                             if (exists && exists.length) {
-                                // console.log(filter, "exists");
                                 return tblH.update(filter, newData, params, table_rules, localParams);
                             }
                             else {
-                                // console.log(filter, "existnts")
                                 return tblH.insert(Object.assign(Object.assign({}, newData), filter), params, null, table_rules, localParams);
                             }
                         });
                     });
                 }
-                // .catch(existnts => {
-                //     console.log(filter, "existnts")
-                //     return this.insert({ ...filter, ...newData}, params);
-                // });
             }
             catch (e) {
                 if (localParams && localParams.testRule)
@@ -1707,7 +1952,7 @@ class DboBuilder {
         this.schema = "public";
         this.init = () => __awaiter(this, void 0, void 0, function* () {
             let onSchemaChange;
-            if (this.prostgles.watchSchema) {
+            if (this.prostgles.opts.watchSchema) {
                 onSchemaChange = (event) => {
                     this.prostgles.onSchemaChange(event);
                 };
@@ -1730,14 +1975,20 @@ class DboBuilder {
                     }
                     else {
                         txDB[tov.name] = new TableHandler(this.db, tov, this.pubSubManager, this, t, this.joinPaths);
+                        /**
+                         * Pass only the transaction object to ensure consistency
+                         */
+                        //     txDB[tov.name] = new ViewHandler(t, tov, this.pubSubManager, this, t, this.joinPaths);
+                        // } else {
+                        //     txDB[tov.name] = new TableHandler(t as any, tov, this.pubSubManager, this, t, this.joinPaths);
                     }
                 });
-                return dbTX(txDB);
+                return dbTX(txDB, t);
             });
         };
         this.prostgles = prostgles;
         this.db = this.prostgles.db;
-        this.schema = this.prostgles.schema || "public";
+        this.schema = this.prostgles.opts.schema || "public";
         this.dbo = {};
         // this.joins = this.prostgles.joins;
     }
@@ -1749,14 +2000,19 @@ class DboBuilder {
     }
     parseJoins() {
         return __awaiter(this, void 0, void 0, function* () {
-            if (this.prostgles.joins) {
-                let _joins = yield this.prostgles.joins;
+            if (this.prostgles.opts.joins) {
+                let _joins = yield this.prostgles.opts.joins;
+                let inferredJoins = yield getInferredJoins(this.db, this.prostgles.opts.schema);
                 if (typeof _joins === "string" && _joins === "inferred") {
-                    _joins = yield getInferredJoins(this.db, this.prostgles.schema);
+                    _joins = inferredJoins;
+                    /* If joins are specified then include inferred joins except the explicit tables */
+                }
+                else if (Array.isArray(_joins)) {
+                    const joinTables = _joins.map(j => j.tables).flat();
+                    _joins = _joins.concat(inferredJoins.filter(j => !j.tables.find(t => joinTables.includes(t))));
                 }
                 let joins = JSON.parse(JSON.stringify(_joins));
                 this.joins = joins;
-                // console.log(joins);
                 // Validate joins
                 try {
                     // 1 find duplicates
@@ -1822,9 +2078,6 @@ class DboBuilder {
                         }
                     });
                 });
-                // console.log(this.joinPaths)
-                // console.log(888, this.prostgles.joins);
-                // console.log(this.joinGraph, findShortestPath(this.joinGraph, "colors", "drawings"));
             }
             return this.joinPaths;
         });
@@ -1857,7 +2110,6 @@ export type TxCB = {
                 i18nDef += `    ${JSON.stringify(tov.name)}: { \n`;
                 i18nDef += `      [key in ${tov.columns.map(c => JSON.stringify(c.name)).join(" | ")}]: { [lang_id in keyof LANG_IDS]: string }; \n`;
                 i18nDef += `    }; \n`;
-                // console.log("dboInit", tov.name, tov.columns.map(c => c.name))
                 const filterKeywords = Object.values(this.prostgles.keywords);
                 const $filterCol = tov.columns.find(c => filterKeywords.includes(c.name));
                 if ($filterCol) {
@@ -1917,13 +2169,81 @@ export type TxCB = {
                     this.dboDefinition += `  ${joinType}: JoinMakerTables;\n`;
                 });
             }
-            if (this.prostgles.transactions) {
+            if (this.prostgles.opts.transactions) {
                 let txKey = "tx";
-                if (typeof this.prostgles.transactions === "string")
-                    txKey = this.prostgles.transactions;
+                if (typeof this.prostgles.opts.transactions === "string")
+                    txKey = this.prostgles.opts.transactions;
                 this.dboDefinition += ` ${txKey}: (t: TxCB) => Promise<any | void> ;\n`;
                 this.dbo[txKey] = (cb) => this.getTX(cb);
             }
+            if (!this.dbo.sql) {
+                let needType = true; // this.publishRawSQL && typeof this.publishRawSQL === "function";
+                let DATA_TYPES = !needType ? [] : yield this.db.any("SELECT oid, typname FROM pg_type");
+                let USER_TABLES = !needType ? [] : yield this.db.any("SELECT relid, relname FROM pg_catalog.pg_statio_user_tables");
+                this.dbo.sql = (query, params, options, localParams) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    const canRunSQL = (localParams) => __awaiter(this, void 0, void 0, function* () {
+                        if (!localParams)
+                            return true;
+                        const { socket } = localParams;
+                        const publishParams = yield this.prostgles.publishParser.getPublishParams({ socket });
+                        let res = yield this.prostgles.opts.publishRawSQL(publishParams);
+                        return Boolean(res && typeof res === "boolean" || res === "*");
+                    });
+                    if (!(yield canRunSQL(localParams)))
+                        throw "Not allowed to run SQL";
+                    const { returnType } = options || {};
+                    const { socket } = localParams || {};
+                    if (returnType === "noticeSubscription") {
+                        if (!socket)
+                            throw "Only allowed with client socket";
+                        return yield this.prostgles.dbEventsManager.addNotice(socket);
+                    }
+                    else if (returnType === "statement") {
+                        try {
+                            return exports.pgp.as.format(query, params);
+                        }
+                        catch (err) {
+                            throw err.toString();
+                        }
+                    }
+                    else if (this.db) {
+                        let qres = yield this.db.result(query, params);
+                        const { duration, fields, rows, command } = qres;
+                        if (command === "LISTEN") {
+                            if (!socket)
+                                throw "Only allowed with client socket";
+                            return yield this.prostgles.dbEventsManager.addNotify(query, socket);
+                        }
+                        else if (returnType === "rows") {
+                            return rows;
+                        }
+                        else if (returnType === "row") {
+                            return rows[0];
+                        }
+                        else if (returnType === "value") {
+                            return (_a = Object.values((rows === null || rows === void 0 ? void 0 : rows[0]) || {})) === null || _a === void 0 ? void 0 : _a[0];
+                        }
+                        else if (returnType === "values") {
+                            return rows.map(r => Object.values(r[0]));
+                        }
+                        else {
+                            if (fields && DATA_TYPES.length) {
+                                qres.fields = fields.map(f => {
+                                    const dataType = DATA_TYPES.find(dt => +dt.oid === +f.dataTypeID), tableName = USER_TABLES.find(t => +t.relid === +f.tableID), { name } = f;
+                                    return Object.assign(Object.assign(Object.assign({}, f), (dataType ? { dataType: dataType.typname } : {})), (tableName ? { tableName: tableName.relname } : {}));
+                                });
+                            }
+                            return qres;
+                        }
+                    }
+                    else
+                        console.error("db missing");
+                });
+            }
+            else {
+                console.warn(`Could not create dbo.sql handler because there is already a table named "sql"`);
+            }
             this.dboDefinition += "};\n";
             this.tsTypesDefinition = [
                 common_types,
@@ -2349,7 +2669,6 @@ function getInferredJoins(db, schema = "public") {
                 });
             }
         });
-        // console.log(joins);
         return joins;
     });
 }