pogi 2.10.2 → 3.0.0-beta2

package/src/queryAble.ts

@@ -1,66 +1,26 @@
 import { PgDbLogger } from "./pgDbLogger";
 import { pgUtils } from "./pgUtils";
 import * as stream from "stream";
+import { IPgSchema } from "./pgSchemaInterface";
+import * as pg from 'pg';
+import util = require('util');
+import QueryStream = require('pg-query-stream');
+import through = require('through');
+import {ResultFieldType, IPgDb} from "./pgDbInterface";
+import {SqlQueryOptions, IQueryAble, PgRowResult } from "./queryAbleInterface"
 
-const util = require('util');
-const QueryStream = require('pg-query-stream');
-const through = require('through');
-
-export interface QueryOptions {
-    limit?: number;
-    offset?: number;
-    orderBy?: string | string[] | { [fieldName: string]: 'asc' | 'desc' };//free text or column list
-    groupBy?: string | string[];//free text or column list
-    fields?: string | string[];//free text or column list
-    logger?: PgDbLogger;
-    forUpdate?: boolean;
-    distinct?: boolean;
-    skipUndefined?: boolean;
-}
-
-export interface SqlQueryOptions {
-    logger?: PgDbLogger;
-}
-
-export interface ResultFieldType {
-    name: string,
-    tableID: number,
-    columnID: number,
-    dataTypeID: number,
-    dataTypeSize: number,
-    dataTypeModifier: number,
-    format: string
-}
-
-export interface ResultType {
-    command: 'SELECT' | 'UPDATE' | 'DELETE',
-    rowCount: number,
-    oid: number,
-    rows: any[],
-    fields: ResultFieldType[],
-    _parsers: Function[][],
-    RowCtor: Function[],
-    rowsAsArray: boolean,
-    _getTypeParser: Function[]
-}
-
-export interface PgRowResult {
-    columns: string[],
-    rows: any[]
-}
 
 let defaultLogger = {
     log: () => { },
     error: () => { }
 };
 
-export class QueryAble {
-    db;
-    schema;
-    protected logger: PgDbLogger;
+export abstract class QueryAble implements IQueryAble {
+    db!: IPgDb & IQueryAble;  // assigned in async init
+    schema!: IPgSchema;
+    /*protected*/ logger!: PgDbLogger;
 
-    constructor() {
-    }
+    public static connectionErrorListener = () => { }
 
     setLogger(logger: PgDbLogger) {
         this.logger = logger;
@@ -84,15 +44,18 @@ export class QueryAble {
      * e.g. query('select * from a.b where id=$1;',['the_stage_is_set']);
      * e.g. query('select * from :!schema.:!table where id=:id;',{schema:'a',table:'b', id:'the_stage_is_set'});
      */
-    async query(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any[]> {
+    async query(sql: string, params?: any[] | {} | null, options?: SqlQueryOptions): Promise<any[]> {
         let connection = this.db.connection;
         let logger = (options && options.logger || this.getLogger(false));
         return this.internalQuery({ connection, sql, params, logger });
     }
 
-    protected async internalQuery(options: { connection, sql: string, params?: any, logger?}): Promise<any[]>;
-    protected async internalQuery(options: { connection, sql: string, params?: any, logger?, rowMode: true }): Promise<PgRowResult>;
-    protected async internalQuery(options: { connection, sql: string, params?: any, logger?, rowMode?: boolean }): Promise<any[] | PgRowResult> {
+    protected async internalQuery(options: { connection: pg.PoolClient | null, sql: string, params?: any, logger?: PgDbLogger }): Promise<any[]>;
+    protected async internalQuery(options: { connection: pg.PoolClient | null, sql: string, params?: any, logger?: PgDbLogger, rowMode: true }): Promise<PgRowResult>;
+    protected async internalQuery(options: { connection: pg.PoolClient | null, sql: string, params?: any, logger?: PgDbLogger, rowMode?: boolean }): Promise<any[] | PgRowResult> {
+        if (this.db.needToFixConnectionForListen()) {
+            await this.db.runRestartConnectionForListen();
+        }
         let { connection, sql, params, logger } = options;
         logger = logger || this.getLogger(false);
 
@@ -102,40 +65,38 @@ export class QueryAble {
                 sql = p.sql;
                 params = p.params;
             }
-
+            let query = options?.rowMode ? { text: sql, values: params, rowMode: 'array' } : { text: sql, values: params };
+            let res;
             if (connection) {
                 logger.log('reused connection', sql, util.inspect(params, false, null), connection.processID);
-                let res = await connection.query({ text: sql, values: params, rowMode: options?.rowMode ? 'array' : undefined });
-                pgUtils.postProcessResult(res.rows, res.fields, this.db.pgdbTypeParsers);
-                if (this.db.postProcessResult) this.db.postProcessResult(res.rows, res.fields, logger);
-
-                return options?.rowMode ? { columns: (res.fields || []).map(f => f.name), rows: res.rows || [] } : res.rows;
+                res = await connection.query(query);
+                await this.checkAndFixOids(connection, res.fields);
             } else {
                 connection = await this.db.pool.connect();
                 logger.log('new connection', sql, util.inspect(params, false, null), connection.processID);
 
+                connection.on('error', QueryAble.connectionErrorListener);
+                res = await connection.query(query);
+                await this.checkAndFixOids(connection, res.fields);
+
+                connection.off('error', QueryAble.connectionErrorListener);
+                connection.release();
+                connection = null;
+            }
+            this.postProcessFields(res.rows, res.fields, logger);
+            return options?.rowMode ? { columns: (res.fields || []).map(f => f.name), rows: res.rows || [] } : res.rows;
+        } catch (e) {
+            pgUtils.logError(logger, { error: <Error>e, sql, params, connection });
+            if (connection) {
                 try {
-                    let res = await connection.query({ text: sql, values: params, rowMode: options?.rowMode ? 'array' : undefined });
+                    //If any problem has happened in a dedicated connection, (wrong sql format or non-accessible postgres server)
+                    //close the connection to be a free connection in the pool,
+                    //but keep the db.connection member non - null to crash in all of the following commands
                     connection.release();
-                    connection = null;
-                    pgUtils.postProcessResult(res.rows, res.fields, this.db.pgdbTypeParsers);
-                    if (this.db.postProcessResult) this.db.postProcessResult(res.rows, res.fields, logger);
-
-                    return options?.rowMode ? { columns: (res.fields || []).map(f => f.name), rows: res.rows || [] } : res.rows;
                 } catch (e) {
-                    pgUtils.logError(logger, { error: e, sql, params, connection });
-                    try {
-                        if (connection)
-                            connection.release();
-                    } catch (e) {
-                        logger.error('connection error2', e.message);
-                    }
-                    connection = null;
-                    throw e;
+                    logger.error('connection error', (<Error>e).message);
                 }
             }
-        } catch (e) {
-            pgUtils.logError(logger, { error: e, sql, params, connection });
             throw e;
         }
     }
@@ -153,128 +114,191 @@ export class QueryAble {
     /**
      * If the callback function return true, the connection will be closed.
      */
-    async queryWithOnCursorCallback(sql: string, params: any[] | {}, options: SqlQueryOptions, callback: (any) => any): Promise<void> {
-        let connection = this.db.connection;
+    async queryWithOnCursorCallback(sql: string, params: any[] | Record<string, any> | null, options: SqlQueryOptions | null, callback: (res: any) => any): Promise<void> {
+        if (this.db.needToFixConnectionForListen()) {
+            await this.db.runRestartConnectionForListen();
+        }
+        let connection = this.db.connection!;
+        let logger = this.getLogger(true);
+        let positionedParams: any[] | null | undefined;
 
         try {
             if (params && !Array.isArray(params)) {
                 let p = pgUtils.processNamedParams(sql, params);
                 sql = p.sql;
                 params = p.params;
+            } else {
+                positionedParams = params;
             }
 
-            if (connection) {
-                this.getLogger(false).log(sql, util.inspect(params, false, null), connection.processID);
-                let query = new QueryStream(sql, params);
+            let queryInternal = async () => {
+                this.getLogger(false).log(sql, util.inspect(positionedParams, false, null), connection.processID);
+                let fieldsToFix: ResultFieldType[] | undefined;
+                let isFirst = true;
+
+                let query = new QueryStream(sql, positionedParams);
                 let stream = connection.query(query);
                 await new Promise((resolve, reject) => {
-                    stream.on('data', (res) => {
+                    query.handleError = reject;
+                    stream.on('data', (res: any) => {
                         try {
                             let fields = stream._result && stream._result.fields || stream.cursor._result && stream.cursor._result.fields;
-                            pgUtils.postProcessResult([res], fields, this.db.pgdbTypeParsers);
-                            if (this.db.postProcessResult) this.db.postProcessResult([res], fields, this.getLogger(false));
+                            if (isFirst) {
+                                if (this.hasUnknownOids(fields)) {
+                                    fieldsToFix = fields;
+                                    stream.destroy();
+                                    return;
+                                }
+                                isFirst = false;
+                            }
+                            this.postProcessFields([res], fields, this.getLogger(false));
 
                             if (callback(res)) {
-                                stream.emit('close');
+                                stream.destroy();
                             }
                         } catch (e) {
                             reject(e);
                         }
                     });
 
-                    stream.on('end', resolve);
+                    stream.on('close', resolve);
                     stream.on('error', reject);
                 });
-
-            } else {
-                try {
-                    connection = await this.db.pool.connect();
-                    this.getLogger(false).log(sql, util.inspect(params, false, null), connection.processID);
-                    let query = new QueryStream(sql, params);
-                    let stream = connection.query(query);
+                if (fieldsToFix) {
+                    await this.checkAndFixOids(connection, fieldsToFix);
+                    query = new QueryStream(sql, positionedParams);
+                    stream = connection.query(query);
                     await new Promise((resolve, reject) => {
-                        stream.on('data', (res) => {
+                        query.handleError = reject;
+                        stream.on('data', (res: any) => {
                             try {
                                 let fields = stream._result && stream._result.fields || stream.cursor._result && stream.cursor._result.fields;
-                                pgUtils.postProcessResult([res], fields, this.db.pgdbTypeParsers);
-                                if (this.db.postProcessResult) this.db.postProcessResult([res], fields, this.getLogger(false));
+                                this.postProcessFields([res], fields, this.getLogger(false));
 
                                 if (callback(res)) {
-                                    stream.emit('close');
+                                    stream.destroy();
                                 }
                             } catch (e) {
                                 reject(e);
                             }
                         });
 
-                        stream.on('end', resolve);
+                        stream.on('close', resolve);
                         stream.on('error', reject);
                     });
-                } finally {
-                    try {
-                        connection.release();
-                    } catch (e) {
-                        this.getLogger(true).error('connection error', e.message);
-                    }
                 }
             }
+
+            if (connection) {
+                await queryInternal();
+            } else {
+                connection = await this.db.pool.connect();
+                logger.log('new connection', sql, util.inspect(positionedParams, false, null), connection.processID);
+                connection.on('error', QueryAble.connectionErrorListener);
+                await queryInternal();
+
+                connection.off('error', QueryAble.connectionErrorListener);
+                connection.release();
+            }
         } catch (e) {
-            let logger = this.getLogger(true);
-            pgUtils.logError(logger, { error: e, sql, params, connection });
+            pgUtils.logError(logger, { error: <Error>e, sql, params, connection });
+            if (connection) {
+                try {
+                    connection.release();
+                } catch (e) {
+                    logger.error('connection error', (<Error>e).message);
+                }
+            }
             throw e;
         }
     }
 
-    async queryAsStream(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<stream.Readable> {
+    async queryAsStream(sql: string, params?: any[] | Record<string, any> | null, options?: SqlQueryOptions | null): Promise<stream.Readable> {
+        if (this.db.needToFixConnectionForListen()) {
+            await this.db.runRestartConnectionForListen();
+        }
         let connection = this.db.connection;
         let logger = (options && options.logger || this.getLogger(false));
-        let pgStream;
-        let pgdb = this.db;
-        let convertTypeFilter = through(function (data) {
+        let pgStream: any;
+        let queriable = this;
+        let isFirst = true;
+        let convertTypeFilter = through(function (this: stream, data) {
             try {
                 let fields = pgStream._result && pgStream._result.fields || pgStream.cursor._result && pgStream.cursor._result.fields;
-                pgUtils.postProcessResult([data], fields, pgdb.pgdbTypeParsers);
-                if (pgdb.postProcessResult) pgdb.postProcessResult([data], fields, logger);
+                if (isFirst) {
+                    if (queriable.hasUnknownOids(fields)) {
+                        throw new Error('[337] Query returns fields with unknown oid.');
+                    }
+                    isFirst = false;
+                }
+                queriable.postProcessFields([data], fields, queriable.db.pgdbTypeParsers);
 
                 this.emit('data', data);
             } catch (err) {
                 this.emit('error', err);
             }
         });
-        convertTypeFilter.on('error', (e) => {
+        convertTypeFilter.on('error', (e: Error) => {
+            if (connection) {
+                try {
+                    connection.release();
+                } catch (e) {
+                    logger.error('connection error', (<Error>e).message);
+                }
+            }
+            connection = null;
             pgUtils.logError(logger, { error: e, sql, params, connection });
         });
 
+        let positionedParams: any[] | undefined | null;
+
         try {
             if (params && !Array.isArray(params)) {
                 let p = pgUtils.processNamedParams(sql, params);
                 sql = p.sql;
                 params = p.params;
+            } else {
+                positionedParams = params;
             }
 
             if (connection) {
-                logger.log(sql, util.inspect(params, false, null), connection.processID);
-                let query = new QueryStream(sql, params);
+                logger.log(sql, util.inspect(positionedParams, false, null), connection.processID);
+                let query = new QueryStream(sql, positionedParams);
+                query.handleError = (err: Error) => {
+                    convertTypeFilter.emit('error', err);
+                };
                 pgStream = connection.query(query);
                 return pgStream.pipe(convertTypeFilter);
             } else {
                 connection = await this.db.pool.connect();
-                logger.log(sql, util.inspect(params, false, null), connection.processID);
-                let query = new QueryStream(sql, params);
+                logger.log('new connection', sql, util.inspect(positionedParams, false, null), connection.processID);
+                connection.on('error', QueryAble.connectionErrorListener);
+
+                let query = new QueryStream(sql, positionedParams);
+                query.handleError = (err: Error, _connection: pg.PoolClient) => {
+                    convertTypeFilter.emit('error', err);
+                };
                 pgStream = connection.query(query);
                 pgStream.on('close', () => {
-                    if (connection) connection.release();
+                    if (connection) {
+                        connection.off('error', QueryAble.connectionErrorListener);
+                        connection.release();
+                    }
                     connection = null;
                 });
-                pgStream.on('error', (e) => {
-                    pgUtils.logError(logger, { error: e, sql, params, connection });
-                    if (connection) connection.release();
+                pgStream.on('error', (e: Error) => {
+                    pgUtils.logError(logger, { error: e, sql, params: positionedParams, connection });
+
+                    if (connection) {
+                        connection.off('error', QueryAble.connectionErrorListener);
+                        connection.release();
+                    }
                     connection = null;
                 });
                 return pgStream.pipe(convertTypeFilter);
             }
         } catch (e) {
-            pgUtils.logError(logger, { error: e, sql, params, connection });
+            pgUtils.logError(logger, { error: <Error>e, sql, params: positionedParams, connection });
             throw e;
         }
     }
@@ -320,4 +344,22 @@ export class QueryAble {
         let fieldName = Object.keys(res[0])[0];
         return res.map(r => r[fieldName]);
     }
+
+    private postProcessFields(rows: any[], fields: ResultFieldType[], logger: PgDbLogger) {
+        pgUtils.postProcessResult(rows, fields, this.db.pgdbTypeParsers);
+        if (this.db.postProcessResult) this.db.postProcessResult(rows, fields, logger);
+    }
+
+    private async checkAndFixOids(connection: pg.PoolClient, fields: ResultFieldType[]) {
+        if (fields) {
+            let oidList = fields.map(field => field.dataTypeID);
+            return this.db.resetMissingParsers(connection, oidList);
+        }
+    }
+
+    private hasUnknownOids(fields: ResultFieldType[]): boolean {
+        let oidList = fields.map(field => field.dataTypeID);
+        let unknownOids = oidList.filter(oid => !this.db.knownOids[oid]);
+        return !!unknownOids.length;
+    }
 }

package/src/queryAbleInterface.ts

@@ -0,0 +1,104 @@
+import { PgDbLogger } from "./pgDbLogger";
+import { IPgSchema } from "./pgSchemaInterface";
+import * as stream from "stream";
+import { ResultFieldType, IPgDb } from "./pgDbInterface";
+import { ForceEscapeColumnsOptions } from "./connectionOptions";
+
+export interface QueryOptions {
+    limit?: number;
+    offset?: number;
+    orderBy?: string | string[] | { [fieldName: string]: 'asc' | 'desc' };//free text or column list
+    /** 
+     * only used with orderBy
+     * true -> nulls first, 
+     * false -> nulls last 
+     */
+    orderByNullsFirst?: boolean;
+    groupBy?: string | string[];//free text or column list
+    fields?: string | string[];//free text or column list
+    logger?: PgDbLogger;
+    forUpdate?: boolean;
+    distinct?: boolean;
+    skipUndefined?: boolean;
+    forceEscapeColumns?: boolean | ForceEscapeColumnsOptions
+}
+
+export interface SqlQueryOptions {
+    logger?: PgDbLogger;
+}
+
+
+
+export interface ResultType {
+    command: 'SELECT' | 'UPDATE' | 'DELETE',
+    rowCount: number,
+    oid: number,
+    rows: any[],
+    fields: ResultFieldType[],
+    _parsers: Function[][],
+    RowCtor: Function[],
+    rowsAsArray: boolean,
+    _getTypeParser: Function[]
+}
+
+export interface PgRowResult {
+    columns: string[],
+    rows: any[]
+}
+
+let defaultLogger = {
+    log: () => { },
+    error: () => { }
+};
+
+
+
+export interface IQueryAble {
+    db: IPgDb & IQueryAble;  // assigned in async init
+    schema: IPgSchema;
+    logger: PgDbLogger;
+
+    /*connectionErrorListener : () => { }*/
+
+    setLogger(logger: PgDbLogger): void
+
+    getLogger(useConsoleAsDefault: boolean): PgDbLogger
+
+    /** alias to {@link query} */
+    run(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any[]>
+
+    /**
+     * Params can be
+     * 1) array, then sql should have $1 $2 for placeholders
+     * 2) object, then sql should have:
+     *    :example -> for params in statements (set/where), will be transformed to $1 $2 ...
+     *    :!example -> for DDL names (schema, table, column), will be replaced in the query
+     * e.g. query('select * from a.b where id=$1;',['the_stage_is_set']);
+     * e.g. query('select * from :!schema.:!table where id=:id;',{schema:'a',table:'b', id:'the_stage_is_set'});
+     */
+    query(sql: string, params?: any[] | {} | null, options?: SqlQueryOptions): Promise<any[]>
+
+    /**
+     * Same as query but response is two array: columns and rows and rows are arrays also not objects
+     * This is useful for queries which have colliding column names
+     */
+    queryAsRows(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<PgRowResult>
+
+    /**
+     * If the callback function return true, the connection will be closed.
+     */
+    queryWithOnCursorCallback(sql: string, params: any[] | Record<string, any> | null, options: SqlQueryOptions | null, callback: (res: any) => any): Promise<void>
+
+    queryAsStream(sql: string, params?: any[] | Record<string, any> | null, options?: SqlQueryOptions | null): Promise<stream.Readable>
+
+    queryOne(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any>
+
+    queryFirst(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any>
+
+    /** @return one record's one field */
+    queryOneField(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any>
+
+    /** @return one column for the matching records */
+    queryOneColumn(sql: string, params?: any[] | {}, options?: SqlQueryOptions): Promise<any[]>
+
+}