pogi 2.11.0

package/src/pgUtils.ts

@@ -0,0 +1,176 @@
+import {QueryOptions, ResultFieldType, QueryAble} from "./queryAble";
+import {FieldType} from "./pgDb";
+import {PgDbLogger} from "./pgDbLogger";
+import * as _ from 'lodash';
+
+const util = require('util');
+const NAMED_PARAMS_REGEXP = /(?:^|[^:]):(!?[a-zA-Z0-9_]+)/g;    // do not convert "::type cast"
+const ASC_DESC_REGEXP = /^([^" (]+)( asc| desc)?$/;
+
+export let pgUtils = {
+
+    logError(logger: PgDbLogger, options: { error?: string|Error, sql: string, params: any, connection }) {
+        let { error, sql, params, connection } = options;
+        logger.error(error, sql, util.inspect(logger.paramSanitizer ? logger.paramSanitizer(params) : params, false, null), connection ? connection.processID : null);
+    },
+
+    quoteField(f) {
+        return f.indexOf('"') == -1 && f.indexOf('(') == -1 ? '"' + f + '"' : f;
+    },
+
+    processQueryFields(options: QueryOptions): string {
+        let s = options && options.distinct ? ' DISTINCT ' : ' ';
+        if (options && options.fields) {
+            if (Array.isArray(options.fields)) {
+                return s + options.fields.map(pgUtils.quoteField).join(', ');
+            } else {
+                return s + options.fields;
+            }
+        } else {
+            return s + ' *';
+        }
+    },
+
+    /**
+     * :named -> $1 (not works with DDL (schema, table, column))
+     * :!named -> "value" (for DDL (schema, table, column))
+     * do not touch ::type cast
+     */
+    processNamedParams(sql: string, params: Object) {
+        let sql2 = [];
+        let params2 = [];
+
+        let p = NAMED_PARAMS_REGEXP.exec(sql);
+        let lastIndex = 0;
+        while (p) {
+            let ddl = false;
+            let name = p[1];
+            if (name[0] == '!') {
+                name = name.slice(1);
+                ddl = true;
+            }
+
+            if (!(name in params)) {
+                throw new Error(`No ${p[1]} in params (keys: ${Object.keys(params)})`);
+            }
+            sql2.push(sql.slice(lastIndex, NAMED_PARAMS_REGEXP.lastIndex - p[1].length - 1));
+
+            if (ddl) {
+                sql2.push('"' + ('' + params[name]).replace(/"/g, '""') + '"');
+            } else {
+                params2.push(params[name]);
+                sql2.push('$' + params2.length);
+            }
+            lastIndex = NAMED_PARAMS_REGEXP.lastIndex;
+            p = NAMED_PARAMS_REGEXP.exec(sql);
+        }
+        sql2.push(sql.substr(lastIndex));
+
+        return {
+            sql: sql2.join(''),
+            params: params2
+        }
+    },
+
+    processQueryOptions(options: QueryOptions): string {
+        options = options || {};
+        let sql = '';
+
+        if (options.groupBy) {
+            if (Array.isArray(options.groupBy)) {
+                sql += ' GROUP BY ' + options.groupBy.map(pgUtils.quoteField).join(',');
+            } else {
+                sql += ' GROUP BY ' + pgUtils.quoteField(options.groupBy);
+            }
+        }
+        if (options.orderBy) {
+            if (typeof options.orderBy == 'string') {
+                sql += ' ORDER BY ' + pgUtils.quoteField(options.orderBy);
+            }
+            else if (Array.isArray(options.orderBy)) {
+                let orderBy = options.orderBy.map(v =>
+                    v[0] == '+' ? pgUtils.quoteField(v.substr(1, v.length - 1)) + ' asc' :
+                        v[0] == '-' ? pgUtils.quoteField(v.substr(1, v.length - 1)) + ' desc' :
+                            v.replace(ASC_DESC_REGEXP, '"$1"$2'));
+                sql += ' ORDER BY ' + orderBy.join(',');
+            } else {
+                let orderBy = [];
+                _.forEach(options.orderBy, (v, k) => orderBy.push(pgUtils.quoteField(k) + ' ' + v));
+                sql += ' ORDER BY ' + orderBy.join(',');
+            }
+        }
+        if (options.limit) {
+            sql += util.format(' LIMIT %d', options.limit);
+        }
+        if (options.offset) {
+            sql += util.format(' OFFSET %d', options.offset);
+        }
+        if (options.forUpdate){
+            sql += ' FOR UPDATE';
+        }
+        return sql;
+    },
+
+    /**
+     * NOTE-DATE: there are 2 approaches to keep tz (the time correctly):
+     *    1) use Date.toISOString() function, but then the $x placeholder should be TIMESTAMP WITH TIME ZONE $x
+     *    2) use Date, and then no need to change the placeholder $x
+     *    lets use 2)
+     */
+    transformInsertUpdateParams(param: any, fieldType: FieldType) {
+        return (param != null && fieldType == FieldType.JSON) ? JSON.stringify(param) :
+            (param != null && fieldType == FieldType.TIME && !(param instanceof Date)) ? new Date(param) : param;
+    },
+
+    postProcessResult(res: any[], fields: ResultFieldType[], pgdbTypeParsers: { [oid: number]: (s:string) => any }) {
+        if (res) {
+            if (res[0] && !Array.isArray(res[0])) {
+                if (Object.keys(res[0]).length != fields.length) {
+                    throw Error("Name collision for the query, two or more fields have the same name.");
+                }
+            }
+            pgUtils.convertTypes(res, fields, pgdbTypeParsers);
+        }
+    },
+
+    convertTypes(res: any[], fields: ResultFieldType[], pgdbTypeParsers: { [oid: number]: (s:string) => any }) {
+        let isArrayMode = Array.isArray(res[0]);
+        fields.forEach((field, i) => {
+            if (pgdbTypeParsers[field.dataTypeID]) {
+                if (isArrayMode) {
+                    res.forEach(e => e[i] = e[i] == null ? null : pgdbTypeParsers[field.dataTypeID](e[i]));
+                } else {
+                    res.forEach(e => e[field.name] = e[field.name] == null ? null : pgdbTypeParsers[field.dataTypeID](e[field.name]));
+                }
+            }
+        });
+    },
+
+    createFunctionCaller(q: QueryAble, fn: { schema: string, name: string, return_single_row: boolean, return_single_value: boolean }) {
+        return async (...args) => {
+            let placeHolders = [];
+            let params = [];
+            args.forEach((arg) => {
+                placeHolders.push('$' + (placeHolders.length + 1));
+                params.push(arg);
+            });
+            let res = await q.query(`SELECT "${fn.schema}"."${fn.name}"(${placeHolders.join(',')})`, params);
+
+            if (fn.return_single_value) {
+                let keys = res[0] ? Object.keys(res[0]) : [];
+                if (keys.length != 1) {
+                    throw Error(`Return type error. schema: ${fn.schema} fn: ${fn.name} expected return type: single value, current value:` + JSON.stringify(res))
+                }
+                res = res.map((r) => r[keys[0]]);
+            }
+            if (fn.return_single_row) {
+                if (res.length != 1) {
+                    throw Error(`Return type error. schema: ${fn.schema} fn: ${fn.name} expected return type: single value, current value:` + JSON.stringify(res))
+                }
+                return res[0];
+            } else {
+                return res;
+            }
+        }
+    }
+};

package/src/queryAble.ts

@@ -0,0 +1,393 @@
+import { PgDbLogger } from "./pgDbLogger";
+import { pgUtils } from "./pgUtils";
+import * as stream from "stream";
+
+const util = require('util');
+const QueryStream = require('pg-query-stream');
+const through = require('through');
+
+export interface QueryOptions {
+    limit?: number;
+    offset?: number;
+    orderBy?: string | string[] | { [fieldName: string]: 'asc' | 'desc' };//free text or column list
+    groupBy?: string | string[];//free text or column list
+    fields?: string | string[];//free text or column list
+    logger?: PgDbLogger;
+    forUpdate?: boolean;
+    distinct?: boolean;
+    skipUndefined?: boolean;
+}
+
+export interface SqlQueryOptions {
+    logger?: PgDbLogger;
+}
+
+export interface ResultFieldType {
+    name: string,
+    tableID: number,
+    columnID: number,
+    dataTypeID: number,
+    dataTypeSize: number,
+    dataTypeModifier: number,
+    format: string
+}
+
+export interface ResultType {
+    command: 'SELECT' | 'UPDATE' | 'DELETE',
+    rowCount: number,
+    oid: number,
+    rows: any[],
+    fields: ResultFieldType[],
+    _parsers: Function[][],
+    RowCtor: Function[],
+    rowsAsArray: boolean,
+    _getTypeParser: Function[]
+}
+
+export interface PgRowResult {
+    columns: string[],
+    rows: any[]
+}
+
+let defaultLogger = {
+    log: () => { },
+    error: () => { }
+};
+
+export class QueryAble {
+    db;
+    schema;
+    protected logger: PgDbLogger;
+
+    constructor() {
+    }
+
+    setLogger(logger: PgDbLogger) {
+        this.logger = logger;
+    }
+
+    getLogger(useConsoleAsDefault = false) {
+        return this.logger || this.schema && this.schema.logger || this.db.logger || (useConsoleAsDefault ? console : defaultLogger);
+    }
+
+    /** alias to {@link query} */
+    async run(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any[]> {
+        return this.query(sql, params, options);
+    }
+
+    /**
+     * Params can be
+     * 1) array, then sql should have $1 $2 for placeholders
+     * 2) object, then sql should have:
+     *    :example -> for params in statements (set/where), will be transformed to $1 $2 ...
+     *    :!example -> for DDL names (schema, table, column), will be replaced in the query
+     * e.g. query('select * from a.b where id=$1;',['the_stage_is_set']);
+     * e.g. query('select * from :!schema.:!table where id=:id;',{schema:'a',table:'b', id:'the_stage_is_set'});
+     */
+    async query(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any[]> {
+        let connection = this.db.connection;
+        let logger = (options && options.logger || this.getLogger(false));
+        return this.internalQuery({ connection, sql, params, logger });
+    }
+
+    protected async internalQuery(options: { connection, sql: string, params?: any, logger?}): Promise<any[]>;
+    protected async internalQuery(options: { connection, sql: string, params?: any, logger?, rowMode: true }): Promise<PgRowResult>;
+    protected async internalQuery(options: { connection, sql: string, params?: any, logger?, rowMode?: boolean }): Promise<any[] | PgRowResult> {
+        if (this.db.needToFixConnectionForListen()) {
+            await this.db.runRestartConnectionForListen();
+        }
+        let { connection, sql, params, logger } = options;
+        logger = logger || this.getLogger(false);
+
+        try {
+            if (params && !Array.isArray(params)) {
+                let p = pgUtils.processNamedParams(sql, params);
+                sql = p.sql;
+                params = p.params;
+            }
+
+            if (connection) {
+                logger.log('reused connection', sql, util.inspect(params, false, null), connection.processID);
+                let res = await connection.query({ text: sql, values: params, rowMode: options?.rowMode ? 'array' : undefined });
+                await this.checkAndFixOids(connection, res.fields);
+                this.postProcessFields(res.rows, res.fields, logger);
+
+                return options?.rowMode ? { columns: (res.fields || []).map(f => f.name), rows: res.rows || [] } : res.rows;
+            } else {
+                connection = await this.db.pool.connect();
+                logger.log('new connection', sql, util.inspect(params, false, null), connection.processID);
+                connection.on('error', (err: Error) => { });
+                let res = await connection.query({ text: sql, values: params, rowMode: options?.rowMode ? 'array' : undefined });
+                await this.checkAndFixOids(connection, res.fields);
+                connection.release();
+                connection = null;
+                this.postProcessFields(res.rows, res.fields, logger);
+
+                return options?.rowMode ? { columns: (res.fields || []).map(f => f.name), rows: res.rows || [] } : res.rows;
+            }
+        } catch (e) {
+            pgUtils.logError(logger, { error: e, sql, params, connection });
+            if (connection) {
+                try {
+                    //If any problem has happened in a dedicated connection, (wrong sql format or non-accessible postgres server)
+                    //close the connection to be a free connection in the pool,
+                    //but keep the db.connection member non - null to crash in all of the following commands
+                    connection.release();
+                } catch (e) {
+                    logger.error('connection error', e.message);
+                }
+            }
+            throw e;
+        }
+    }
+
+    /**
+     * Same as query but response is two array: columns and rows and rows are arrays also not objects
+     * This is useful for queries which have colliding column names
+     */
+    async queryAsRows(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<PgRowResult> {
+        let connection = this.db.connection;
+        let logger = (options && options.logger || this.getLogger(false));
+        return this.internalQuery({ connection, sql, params, logger, rowMode: true });
+    }
+
+    /**
+     * If the callback function return true, the connection will be closed.
+     */
+    async queryWithOnCursorCallback(sql: string, params: any[] | {}, options: SqlQueryOptions, callback: (any) => any): Promise<void> {
+        if (this.db.needToFixConnectionForListen()) {
+            await this.db.runRestartConnectionForListen();
+        }
+        let connection = this.db.connection;
+        let logger = this.getLogger(true);
+
+        try {
+            if (params && !Array.isArray(params)) {
+                let p = pgUtils.processNamedParams(sql, params);
+                sql = p.sql;
+                params = p.params;
+            }
+
+            let queryInternal = async () => {
+                this.getLogger(false).log(sql, util.inspect(params, false, null), connection.processID);
+                let fieldsToFix: ResultFieldType[];
+                let isFirst = true;
+
+                let query = new QueryStream(sql, params);
+                let stream = connection.query(query);
+                await new Promise((resolve, reject) => {
+                    query.handleError = (err: Error, connection) => {
+                        reject(err);
+                    };
+                    stream.on('data', (res) => {
+                        try {
+                            let fields = stream._result && stream._result.fields || stream.cursor._result && stream.cursor._result.fields;
+                            if (isFirst) {
+                                if (this.hasUnknownOids(fields)) {
+                                    fieldsToFix = fields;
+                                    stream.destroy();
+                                    return;
+                                }
+                                isFirst = false;
+                            }
+                            this.postProcessFields([res], fields, this.getLogger(false));
+
+                            if (callback(res)) {
+                                stream.destroy();
+                            }
+                        } catch (e) {
+                            reject(e);
+                        }
+                    });
+
+                    stream.on('close', resolve);
+                    stream.on('error', reject);
+                });
+                if (fieldsToFix) {
+                    await this.checkAndFixOids(connection, fieldsToFix);
+                    query = new QueryStream(sql, params);
+                    stream = connection.query(query);
+                    await new Promise((resolve, reject) => {
+                        query.handleError = (err: Error, connection) => {
+                            reject(err);
+                        };
+                        stream.on('data', (res) => {
+                            try {
+                                let fields = stream._result && stream._result.fields || stream.cursor._result && stream.cursor._result.fields;
+                                this.postProcessFields([res], fields, this.getLogger(false));
+
+                                if (callback(res)) {
+                                    stream.destroy();
+                                }
+                            } catch (e) {
+                                reject(e);
+                            }
+                        });
+
+                        stream.on('close', resolve);
+                        stream.on('error', reject);
+                    });
+                }
+            }
+
+            if (connection) {
+                await queryInternal();
+            } else {
+                connection = await this.db.pool.connect();
+                logger.log('new connection', sql, util.inspect(params, false, null), connection.processID);
+                connection.on('error', (err: Error) => { });
+                await queryInternal();
+                connection.release();
+                connection = null;
+            }
+        } catch (e) {
+            pgUtils.logError(logger, { error: e, sql, params, connection });
+            if (connection) {
+                try {
+                    connection.release();
+                } catch (e) {
+                    logger.error('connection error', e.message);
+                }
+            }
+            throw e;
+        }
+    }
+
+    async queryAsStream(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<stream.Readable> {
+        if (this.db.needToFixConnectionForListen()) {
+            await this.db.runRestartConnectionForListen();
+        }
+        let connection = this.db.connection;
+        let logger = (options && options.logger || this.getLogger(false));
+        let pgStream;
+        let queriable = this;
+        let isFirst = true;
+        let convertTypeFilter = through(function (data) {
+            try {
+                let fields = pgStream._result && pgStream._result.fields || pgStream.cursor._result && pgStream.cursor._result.fields;
+                if (isFirst) {
+                    if (queriable.hasUnknownOids(fields)) {
+                        throw new Error('[337] Query returns fields with unknown oid.');
+                    }
+                    isFirst = false;
+                }
+                queriable.postProcessFields([data], fields, queriable.db.pgdbTypeParsers);
+
+                this.emit('data', data);
+            } catch (err) {
+                this.emit('error', err);
+            }
+        });
+        convertTypeFilter.on('error', (e) => {
+            if (connection) {
+                try {
+                    connection.release();
+                } catch (e) {
+                    logger.error('connection error', e.message);
+                }
+            }
+            connection = null;
+            pgUtils.logError(logger, { error: e, sql, params, connection });
+        });
+
+        try {
+            if (params && !Array.isArray(params)) {
+                let p = pgUtils.processNamedParams(sql, params);
+                sql = p.sql;
+                params = p.params;
+            }
+
+            if (connection) {
+                logger.log(sql, util.inspect(params, false, null), connection.processID);
+                let query = new QueryStream(sql, params);
+                query.handleError = (err: Error, connection) => {
+                    convertTypeFilter.emit('error', err);
+                };
+                pgStream = connection.query(query);
+                return pgStream.pipe(convertTypeFilter);
+            } else {
+                connection = await this.db.pool.connect();
+                logger.log('new connection', sql, util.inspect(params, false, null), connection.processID);
+                connection.on('error', (err: Error) => { });
+                let query = new QueryStream(sql, params);
+                query.handleError = (err: Error, connection) => {
+                    convertTypeFilter.emit('error', err);
+                };
+                pgStream = connection.query(query);
+                pgStream.on('close', () => {
+                    if (connection) connection.release();
+                    connection = null;
+                });
+                pgStream.on('error', (e) => {
+                    pgUtils.logError(logger, { error: e, sql, params, connection });
+                    if (connection) connection.release();
+                    connection = null;
+                });
+                return pgStream.pipe(convertTypeFilter);
+            }
+        } catch (e) {
+            pgUtils.logError(logger, { error: e, sql, params, connection });
+            throw e;
+        }
+    }
+
+    async queryOne(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any> {
+        let res = await this.query(sql, params, options);
+        if (res.length > 1) {
+            let logger = (options && options.logger || this.getLogger(false));
+            let error = Error('More then one rows exists');
+            pgUtils.logError(logger, { error, sql, params, connection: this.db.connection });
+            throw error;
+        }
+        return res[0];
+    }
+
+    async queryFirst(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any> {
+        let res = await this.query(sql, params, options);
+        return res[0];
+    }
+
+    /** @return one record's one field */
+    async queryOneField(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any> {
+        let res = await this.query(sql, params, options);
+        if (!res.length) {
+            return null;
+        }
+        let fieldName = Object.keys(res[0])[0];
+        if (res.length > 1) {
+            let logger = (options && options.logger || this.getLogger(false));
+            let error = Error('More then one field exists!');
+            pgUtils.logError(logger, { error, sql, params, connection: this.db.connection });
+            throw error;
+        }
+        return res.length == 1 ? res[0][fieldName] : null;
+    }
+
+    /** @return one column for the matching records */
+    async queryOneColumn(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any[]> {
+        let res = await this.query(sql, params, options);
+        if (!res.length) {
+            return [];
+        }
+        let fieldName = Object.keys(res[0])[0];
+        return res.map(r => r[fieldName]);
+    }
+
+    private postProcessFields(rows: any[], fields: ResultFieldType[], logger) {
+        pgUtils.postProcessResult(rows, fields, this.db.pgdbTypeParsers);
+        if (this.db.postProcessResult) this.db.postProcessResult(rows, fields, logger);
+    }
+
+    private async checkAndFixOids(connection, fields: ResultFieldType[]) {
+        if (fields) {
+            let oidList = fields.map(field => field.dataTypeID);
+            return this.db.resetMissingParsers(connection, oidList);
+        }
+    }
+
+    private hasUnknownOids(fields: ResultFieldType[]): boolean {
+        let oidList = fields.map(field => field.dataTypeID);
+        let unknownOids = oidList.filter(oid => !this.db.knownOids[oid]);
+        return !!unknownOids.length;
+    }
+}