postgrejs 2.23.0 → 3.0.1

package/connection/pool.js

@@ -3,12 +3,17 @@ import { coerceToBoolean, coerceToInt } from 'putil-varhelpers';
 import { ConnectionState } from '../constants.js';
 import { SafeEventEmitter } from '../safe-event-emitter.js';
 import { getConnectionConfig } from '../util/connection-config.js';
+import { QueryRequest } from '../util/sql-tag.js';
+import { startsCopy, startsTransaction } from '../util/starts-transaction.js';
 import { Connection } from './connection.js';
 import { getIntlConnection, IntlConnection } from './intl-connection.js';
 export class Pool extends SafeEventEmitter {
     _pool;
     _notificationListeners = new SafeEventEmitter();
     _notificationConnection;
+    _pipelineSlots = [];
+    _pipelineMaxQueries;
+    _pipelineMaxConnections;
     config;
     constructor(config) {
         super();
@@ -25,9 +30,11 @@ export class Pool extends SafeEventEmitter {
         poolOptions.min = coerceToInt(cfg.min, 0);
         poolOptions.minIdle = coerceToInt(cfg.minIdle, 0);
         poolOptions.validation = coerceToBoolean(cfg.validation, false);
+        this._pipelineMaxQueries = Math.max(coerceToInt(cfg.pipelineMaxQueries, 100), 1);
+        this._pipelineMaxConnections = Math.max(coerceToInt(cfg.pipelineMaxConnections, poolOptions.max), 0);
         const poolFactory = {
             create: async () => {
-                /* istanbul ignore next */
+                /* c8 ignore start */
                 if (this.listenerCount('debug')) {
                     this.emit('debug', {
                         location: 'Pool.factory.create',
@@ -35,10 +42,11 @@ export class Pool extends SafeEventEmitter {
                         message: `new connection creating`,
                     });
                 }
+                /* c8 ignore stop */
                 const intlCon = new IntlConnection(cfg);
                 await intlCon.connect();
                 intlCon.on('close', () => this._pool.destroy(intlCon));
-                /* istanbul ignore next */
+                /* c8 ignore start */
                 if (this.listenerCount('debug')) {
                     this.emit('debug', {
                         location: 'Pool.factory.create',
@@ -46,10 +54,11 @@ export class Pool extends SafeEventEmitter {
                         message: `[${intlCon.processID}] connection created`,
                     });
                 }
+                /* c8 ignore stop */
                 return intlCon;
             },
             destroy: intlCon => {
-                /* istanbul ignore next */
+                /* c8 ignore start */
                 if (this.listenerCount('debug')) {
                     this.emit('debug', {
                         location: 'Pool.factory.destroy',
@@ -57,10 +66,11 @@ export class Pool extends SafeEventEmitter {
                         message: `[${intlCon.processID}] connection destroy`,
                     });
                 }
+                /* c8 ignore stop */
                 return intlCon.close();
             },
-            reset: async (intlCon) => {
-                /* istanbul ignore next */
+            reset: (intlCon) => {
+                /* c8 ignore start */
                 if (this.listenerCount('debug')) {
                     this.emit('debug', {
                         location: 'Pool.factory.reset',
@@ -68,19 +78,14 @@ export class Pool extends SafeEventEmitter {
                         message: `[${intlCon.processID}] connection reset`,
                     });
                 }
-                try {
-                    if (intlCon.state === ConnectionState.READY) {
-                        await intlCon.execute('ROLLBACK;UNLISTEN *');
-                    }
-                }
-                finally {
-                    intlCon.removeAllListeners();
-                    intlCon.once('close', () => this._pool.destroy(intlCon));
-                    intlCon._refCount = 0;
-                }
+                /* c8 ignore stop */
+                intlCon.owner = undefined;
+                intlCon.removeAllListeners();
+                intlCon.once('close', () => this._pool.destroy(intlCon));
+                intlCon._refCount = 0;
             },
             validate: async (intlCon) => {
-                /* istanbul ignore next */
+                /* c8 ignore start */
                 if (this.listenerCount('debug')) {
                     this.emit('debug', {
                         location: 'Pool.factory.validate',
@@ -88,6 +93,7 @@ export class Pool extends SafeEventEmitter {
                         message: `[${intlCon.processID}] connection validate`,
                     });
                 }
+                /* c8 ignore stop */
                 if (intlCon.state !== ConnectionState.READY)
                     throw new Error('Connection is not active');
                 await intlCon.execute('select 1;');
@@ -98,16 +104,15 @@ export class Pool extends SafeEventEmitter {
         this._pool.on('error', (...args) => this.emit('error', ...args));
         this._pool.on('acquire', (...args) => this.emit('acquire', ...args));
         this._pool.on('destroy', (...args) => this.emit('destroy', ...args));
-        this._pool.start();
     }
     /**
-     * Returns number of connections that are currently acquired
+     * Returns the number of connections that are currently acquired
      */
     get acquiredConnections() {
         return this._pool.acquired;
     }
     /**
-     * Returns number of unused connections in the pool
+     * Returns the number of unused connections in the pool
      */
     get idleConnections() {
         return this._pool.available;
@@ -118,12 +123,15 @@ export class Pool extends SafeEventEmitter {
     get totalConnections() {
         return this._pool.size;
     }
+    start() {
+        return this._pool.start();
+    }
     /**
      * Obtains a connection from the connection pool
      */
     async acquire() {
         const intlCon = await this._pool.acquire();
-        /* istanbul ignore next */
+        /* c8 ignore start */
         if (this.listenerCount('debug')) {
             this.emit('debug', {
                 location: 'Pool.acquire',
@@ -131,8 +139,9 @@ export class Pool extends SafeEventEmitter {
                 message: `[${intlCon.processID}] acquired`,
             });
         }
+        /* c8 ignore stop */
         const connection = new Connection(this, intlCon);
-        /* istanbul ignore next */
+        /* c8 ignore next */
         if (this.listenerCount('debug'))
             connection.on('debug', (...args) => this.emit('debug', ...args));
         if (this.listenerCount('execute'))
@@ -154,24 +163,63 @@ export class Pool extends SafeEventEmitter {
      * Executes a script
      */
     async execute(sql, options) {
-        const connection = await this.acquire();
+        if (sql instanceof QueryRequest)
+            sql = sql.stringify({ ...options, typeMap: options?.typeMap });
+        const slot = this._canPipeline(options?.pipeline, sql, options?.autoCommit, options?.signal)
+            ? this._acquireShared()
+            : undefined;
+        if (!slot) {
+            const connection = await this.acquire();
+            try {
+                return await connection.execute(sql, options);
+            }
+            finally {
+                await this.release(connection);
+            }
+        }
         try {
-            return await connection.execute(sql, options);
+            const shared = slot.connection || (await slot.promise);
+            const result = await shared.execute(sql, options);
+            this._checkSharedTransaction(slot, shared);
+            return result;
         }
         finally {
-            await this.release(connection);
+            slot.load--;
         }
     }
     /**
      * Executes a query
      */
     async query(sql, options) {
-        const connection = await this.acquire();
+        if (sql instanceof QueryRequest) {
+            if (options?.params)
+                throw new TypeError('A statement built with sql`` carries its own parameters; passing `params` as well is ambiguous');
+            options = { ...options, params: sql.params };
+            sql = sql.sql;
+        }
+        // A cursor hands the caller something that outlives this call and must
+        // keep its own portal on its own connection, so it can never share.
+        const slot = !options?.cursor &&
+            this._canPipeline(options?.pipeline, sql, options?.autoCommit, options?.signal)
+            ? this._acquireShared()
+            : undefined;
+        if (!slot) {
+            const connection = await this.acquire();
+            try {
+                return await connection.query(sql, options);
+            }
+            finally {
+                await this.release(connection);
+            }
+        }
         try {
-            return await connection.query(sql, options);
+            const shared = slot.connection || (await slot.promise);
+            const result = await shared.query(sql, options);
+            this._checkSharedTransaction(slot, shared);
+            return result;
         }
         finally {
-            await this.release(connection);
+            slot.load--;
         }
     }
     async prepare(sql, options) {
@@ -186,8 +234,20 @@ export class Pool extends SafeEventEmitter {
     async listen(channel, callback) {
         if (!/^[A-Z]\w+$/i.test(channel))
             throw new TypeError(`Invalid channel name`);
+        // Bug: _initNotificationConnection() only ever bootstraps the shared
+        // connection and registers every channel known at that moment - it
+        // returns immediately once that connection already exists, so a second,
+        // different channel added afterwards was recorded here but never
+        // reached the server at all (same root cause fixed in
+        // Connection.listen(), one layer up).
+        const alreadyListening = !!this._notificationListeners.listenerCount(channel);
         this._notificationListeners.on(channel, callback);
-        await this._initNotificationConnection();
+        if (!this._notificationConnection) {
+            await this._initNotificationConnection();
+        }
+        else if (!alreadyListening) {
+            await this._notificationConnection.listen(channel, callback);
+        }
     }
     async unListen(channel) {
         if (!/^[A-Z]\w+$/i.test(channel))
@@ -207,6 +267,86 @@ export class Pool extends SafeEventEmitter {
             await conn.close();
         }
     }
+    _canPipeline(pipeline, sql, autoCommit, signal) {
+        return (pipeline === true &&
+            !signal &&
+            this._pipelineMaxQueries > 1 &&
+            this._pipelineMaxConnections > 0 &&
+            autoCommit !== false &&
+            this.config.autoCommit !== false &&
+            !startsTransaction(sql) &&
+            !startsCopy(sql));
+    }
+    /**
+     * Picks the pooled connection a pipelined query should ride on, or
+     * undefined when the caller should take a connection of its own.
+     *
+     * Deliberately synchronous: it must never await, because everything in a
+     * Promise.all() burst has to pick its connection and reach execute()
+     * within a single event loop tick for PgSocket to coalesce the whole
+     * burst into one write per connection. Opening another connection is
+     * started here but never awaited by the caller that triggered it.
+     */
+    _acquireShared() {
+        const slots = this._pipelineSlots;
+        const l = slots.length;
+        let best;
+        let slot;
+        let i;
+        for (i = 0; i < l; i++) {
+            slot = slots[i];
+            /* c8 ignore next */
+            if (slot.connection?.inTransaction)
+                continue;
+            if (!best || slot.load < best.load)
+                best = slot;
+        }
+        if ((!best || best.load > 0) && l < this._pipelineMaxConnections)
+            return this._growPipeline();
+        if (!best || best.load >= this._pipelineMaxQueries)
+            return undefined;
+        best.load++;
+        return best;
+    }
+    /**
+     * Starts borrowing one more connection and returns its slot right away,
+     * already carrying the caller that triggered the growth.
+     */
+    _growPipeline() {
+        const slot = { load: 1 };
+        slot.promise = this.acquire().then(connection => {
+            slot.connection = connection;
+            connection.once('idle', () => {
+                this._dropPipelineSlot(slot);
+                this.release(connection).catch(e => this.emit('error', e));
+            });
+            return connection;
+        }, e => {
+            this._dropPipelineSlot(slot);
+            throw e;
+        });
+        this._pipelineSlots.push(slot);
+        return slot;
+    }
+    /**
+     * Stops new queries joining a slot. Queries already on it are unaffected.
+     */
+    _dropPipelineSlot(slot) {
+        const i = this._pipelineSlots.indexOf(slot);
+        if (i >= 0)
+            this._pipelineSlots.splice(i, 1);
+    }
+    /**
+     * Backstop for a transaction startsTransaction() could not see coming -
+     * one opened inside a stored procedure, for instance. The connection is
+     * still sound, but nothing else may join it while it is in a
+     * transaction, so the slot goes away and the connection returns to the
+     * pool once its own queries finish.
+     */
+    _checkSharedTransaction(slot, connection) {
+        if (connection.inTransaction)
+            this._dropPipelineSlot(slot);
+    }
     async _initNotificationConnection() {
         if (this._notificationConnection)
             return;
@@ -226,7 +366,15 @@ export class Pool extends SafeEventEmitter {
             setTimeout(() => {
                 if (!this._notificationListeners.eventNames().length)
                     return;
-                conn.connect().catch(() => reConnect());
+                // Bug: a successful reconnect never re-issued LISTEN for anything -
+                // Connection.close() clears `conn`'s own _notificationListeners,
+                // and registerEvents() (which reads the Pool's, a separate
+                // instance) was only ever called once, right after the first
+                // connect(). A connection drop silently ended every subscription.
+                conn
+                    .connect()
+                    .then(registerEvents)
+                    .catch(() => reConnect());
             }, 500).unref();
         };
         await conn.connect();

package/connection/portal.d.ts

@@ -5,19 +5,33 @@ import type { Connection } from './connection.js';
 import type { PreparedStatement } from './prepared-statement.js';
 export interface PortalExecuteResult {
     code: Protocol.BackendMessageCode;
-    rows?: any[];
+    rows?: Protocol.DataRowMessage[];
     command?: string;
     rowCount?: number;
 }
 export declare class Portal {
     private readonly _statement;
     private readonly _name?;
-    private _columnFormat;
     constructor(statement: PreparedStatement, name: string);
     get connection(): Connection;
     get name(): Maybe<string>;
-    bind(params: Maybe<any[]>, queryOptions: QueryOptions): Promise<void>;
-    retrieveFields(): Promise<Protocol.RowDescription[]>;
+    /**
+     * Bind + Describe(portal) collapsed into one round trip instead of two
+     * separately-awaited calls. Ends in Flush, not Sync - a Sync here would
+     * commit an implicit (autocommit) transaction and destroy this portal
+     * before any fetch() could use it (proved live: Bind+Describe+Sync with
+     * no explicit BEGIN left transactionStatus 'I' and a later Execute on the
+     * same named portal failed with "portal ... does not exist").
+     */
+    bindAndRetrieveFields(params: Maybe<any[]>, queryOptions: QueryOptions): Promise<Protocol.RowDescription[]>;
     execute(fetchCount?: number): Promise<PortalExecuteResult>;
+    /**
+     * Close + Sync as one request (see PgSocket.sendCloseAndSyncMessages()
+     * for why they must share a single capture): ReadyForQuery is the end
+     * marker, CloseComplete is informational on the way there. That also
+     * makes this safe to call as cleanup after a failed Bind - the server is
+     * in error state then, skips the Close entirely, and answers only the
+     * Sync.
+     */
     close(): Promise<void>;
 }

package/connection/portal.js

@@ -1,11 +1,9 @@
-import { DEFAULT_COLUMN_FORMAT } from '../constants.js';
 import { GlobalTypeMap } from '../data-type-map.js';
 import { Protocol } from '../protocol/protocol.js';
 import { getIntlConnection } from './intl-connection.js';
 export class Portal {
     _statement;
     _name;
-    _columnFormat = DEFAULT_COLUMN_FORMAT;
     constructor(statement, name) {
         this._statement = statement;
         this._name = name;
@@ -16,49 +14,32 @@ export class Portal {
     get name() {
         return this._name;
     }
-    async bind(params, queryOptions) {
+    /**
+     * Bind + Describe(portal) collapsed into one round trip instead of two
+     * separately-awaited calls. Ends in Flush, not Sync - a Sync here would
+     * commit an implicit (autocommit) transaction and destroy this portal
+     * before any fetch() could use it (proved live: Bind+Describe+Sync with
+     * no explicit BEGIN left transactionStatus 'I' and a later Execute on the
+     * same named portal failed with "portal ... does not exist").
+     */
+    async bindAndRetrieveFields(params, queryOptions) {
         const intoCon = getIntlConnection(this.connection);
         intoCon.ref();
         try {
             const socket = intoCon.socket;
-            this._columnFormat =
-                queryOptions.columnFormat != null
-                    ? queryOptions.columnFormat
-                    : Protocol.DataFormat.binary;
-            socket.sendBindMessage({
-                typeMap: queryOptions.typeMap || GlobalTypeMap,
-                statement: this._statement.name,
-                portal: this.name,
-                paramTypes: this._statement.paramTypes,
-                params,
-                queryOptions,
-            });
-            socket.sendFlushMessage();
-            return await socket.capture(async (code, msg, done) => {
+            return await socket.sendBindDescribeMessages({
+                bind: {
+                    typeMap: queryOptions.typeMap || GlobalTypeMap,
+                    statement: this._statement.name,
+                    portal: this.name,
+                    paramTypes: this._statement.paramTypes,
+                    params,
+                    queryOptions,
+                },
+                describe: { type: 'P', name: this.name },
+            }, (code, msg, done) => {
                 switch (code) {
                     case Protocol.BackendMessageCode.BindComplete:
-                        done();
-                        break;
-                    case Protocol.BackendMessageCode.NoticeResponse:
-                        break;
-                    default:
-                        done(new Error(`Server returned unexpected response message (${String.fromCharCode(code)})`));
-                }
-            });
-        }
-        finally {
-            intoCon.unref();
-        }
-    }
-    async retrieveFields() {
-        const intoCon = getIntlConnection(this.connection);
-        intoCon.ref();
-        try {
-            const socket = intoCon.socket;
-            socket.sendDescribeMessage({ type: 'P', name: this.name });
-            socket.sendFlushMessage();
-            return await socket.capture(async (code, msg, done) => {
-                switch (code) {
                     case Protocol.BackendMessageCode.NoticeResponse:
                         break;
                     case Protocol.BackendMessageCode.NoData:
@@ -67,6 +48,9 @@ export class Portal {
                     case Protocol.BackendMessageCode.RowDescription:
                         done(undefined, msg.fields);
                         break;
+                    case Protocol.BackendMessageCode.ErrorResponse:
+                        done(msg);
+                        break;
                     default:
                         done(new Error(`Server returned unexpected response message (${String.fromCharCode(code)})`));
                 }
@@ -81,13 +65,11 @@ export class Portal {
         intoCon.ref();
         try {
             const socket = intoCon.socket;
-            socket.sendExecuteMessage({
+            const rows = [];
+            const executePromise = socket.sendExecuteMessage({
                 portal: this.name,
                 fetchCount: fetchCount || 100,
-            });
-            socket.sendFlushMessage();
-            const rows = [];
-            return await socket.capture(async (code, msg, done) => {
+            }, (code, msg, done) => {
                 switch (code) {
                     case Protocol.BackendMessageCode.NoticeResponse:
                         break;
@@ -95,16 +77,12 @@ export class Portal {
                         done(undefined, { code });
                         break;
                     case Protocol.BackendMessageCode.DataRow:
-                        if (Array.isArray(this._columnFormat)) {
-                            rows.push(msg.columns.map((buf, i) => this._columnFormat[i] ===
-                                Protocol.DataFormat.text
-                                ? buf.toString('utf8')
-                                : buf));
-                        }
-                        else if (this._columnFormat === Protocol.DataFormat.binary)
-                            rows.push(msg.columns);
-                        else
-                            rows.push(msg.columns.map((buf) => buf.toString('utf8')));
+                        // msg is the whole DataRowMessage (columnCount + one raw
+                        // buffer for the whole row) - per-column boundary-finding
+                        // and decode now happens lazily in parseRow/get-parsers.ts
+                        // instead of eagerly here, so this can just pass it straight
+                        // through regardless of this._columnFormat's shape.
+                        rows.push(msg);
                         break;
                     case Protocol.BackendMessageCode.PortalSuspended:
                         done(undefined, { code, rows });
@@ -117,31 +95,47 @@ export class Portal {
                             rowCount: msg.rowCount,
                         });
                         break;
+                    case Protocol.BackendMessageCode.ErrorResponse:
+                        done(msg);
+                        break;
                     default:
                         done(new Error(`Server returned unexpected response message (${String.fromCharCode(code)})`));
                 }
             });
+            socket.sendFlushMessage();
+            return await executePromise;
         }
         finally {
             intoCon.unref();
         }
     }
+    /**
+     * Close + Sync as one request (see PgSocket.sendCloseAndSyncMessages()
+     * for why they must share a single capture): ReadyForQuery is the end
+     * marker, CloseComplete is informational on the way there. That also
+     * makes this safe to call as cleanup after a failed Bind - the server is
+     * in error state then, skips the Close entirely, and answers only the
+     * Sync.
+     */
     async close() {
         const intoCon = getIntlConnection(this.connection);
         intoCon.ref();
         try {
             const socket = intoCon.socket;
-            socket.sendCloseMessage({ type: 'P', name: this.name });
-            socket.sendSyncMessage();
-            return await socket.capture(async (code, msg, done) => {
+            let error;
+            await socket.sendCloseAndSyncMessages({ type: 'P', name: this.name }, (code, msg, done) => {
                 switch (code) {
+                    case Protocol.BackendMessageCode.CloseComplete:
                     case Protocol.BackendMessageCode.NoticeResponse:
                         break;
-                    case Protocol.BackendMessageCode.CloseComplete:
+                    case Protocol.BackendMessageCode.ErrorResponse:
+                        // Deferred: only ReadyForQuery is guaranteed to arrive
+                        // exactly once, so done() waits for it.
+                        error = msg;
                         break;
                     case Protocol.BackendMessageCode.ReadyForQuery:
                         intoCon.transactionStatus = msg.status;
-                        done();
+                        done(error);
                         break;
                     default:
                         done(new Error(`Server returned unexpected response message (${String.fromCharCode(code)})`));

package/connection/prepared-statement.d.ts

@@ -8,9 +8,11 @@ export declare class PreparedStatement extends SafeEventEmitter implements Async
     private readonly _connection;
     private readonly _sql;
     private readonly _name;
-    private readonly _paramTypes;
+    private readonly _paramTypes?;
+    private _fields?;
     protected _onErrorSavePoint: string;
     private _refCount;
+    private _closed;
     constructor(connection: Connection, sql: string, paramTypes?: OID[]);
     static prepare(connection: Connection, sql: string, options?: StatementPrepareOptions): Promise<PreparedStatement>;
     get connection(): Connection;
@@ -18,9 +20,28 @@ export declare class PreparedStatement extends SafeEventEmitter implements Async
     get sql(): string;
     get paramTypes(): Maybe<Maybe<OID>[]>;
     execute(options?: QueryOptions): Promise<QueryResult>;
+    protected _executeWithTransaction(options?: QueryOptions): Promise<QueryResult>;
     close(): Promise<void>;
+    /**
+     * Asks the server to cancel whatever this statement's connection is
+     * currently running. See Connection.cancel(); prefer the per-call `signal`
+     * option, which reports the abort to the caller that asked for it.
+     */
     cancel(): Promise<void>;
     protected _execute(options?: QueryOptions): Promise<QueryResult>;
+    /**
+     * Called by Cursor.close() instead of separately awaiting
+     * Portal.close() then this.close(). Returns false (no wire action taken,
+     * caller must still close just the portal itself) when either this
+     * statement was already fully closed by someone else (mirrors close()'s
+     * own `if (this._closed) return;` idempotency guard - re-sending
+     * Close(statement) for an already-closed name would error) or the
+     * decremented refcount is still > 0 (statement still referenced
+     * elsewhere - e.g. another cursor from the same PreparedStatement, or
+     * the statement's own +1 from prepare()). Returns true only when it
+     * actually performed the combined Close(portal)+Close(statement)+Sync.
+     */
+    _maybeCloseWithPortal(portalName: string): Promise<boolean>;
     protected _close(): Promise<void>;
     [Symbol.asyncDispose](): Promise<void>;
 }