turbine-orm 0.7.1 → 0.8.0

package/dist/pipeline-submittable.js

@@ -0,0 +1,397 @@
+/**
+ * turbine-orm — Real Postgres pipeline protocol implementation
+ *
+ * Uses the pg extended-query protocol wire methods (parse/bind/describe/execute/sync)
+ * exposed on pg.Client's Connection object to send multiple queries in a single
+ * TCP flush. This achieves true 1-RTT pipeline execution instead of the sequential
+ * await-per-query approach.
+ *
+ * The approach (listener-swap):
+ *   1. Detach the pg.Client's event listeners from the Connection
+ *   2. Attach our own state-machine listeners
+ *   3. Cork the TCP stream, push all protocol messages, uncork (one TCP write)
+ *   4. Drive a state machine over backend response events
+ *   5. Restore original listeners and release the client
+ *
+ * This is the same pattern used by pg-cursor and pg-query-stream, but extended
+ * to handle N queries in a single pipeline.
+ */
+import Result from 'pg/lib/result';
+import { prepareValue } from 'pg/lib/utils';
+import { wrapPgError } from './errors.js';
+// ---------------------------------------------------------------------------
+// Event names we intercept
+// ---------------------------------------------------------------------------
+/**
+ * All backend message event names that the pg Client listens for.
+ * We snapshot, detach, and restore listeners for these events.
+ */
+const INTERCEPTED_EVENTS = [
+    'readyForQuery',
+    'rowDescription',
+    'dataRow',
+    'commandComplete',
+    'parseComplete',
+    'bindComplete',
+    'errorMessage',
+    'emptyQuery',
+    'portalSuspended',
+    'noData',
+    'notice',
+    'copyInResponse',
+    'copyData',
+];
+function snapshotListeners(emitter) {
+    const map = new Map();
+    for (const event of INTERCEPTED_EVENTS) {
+        const listeners = emitter.rawListeners(event);
+        if (listeners.length > 0) {
+            map.set(event, [...listeners]);
+        }
+    }
+    return map;
+}
+function detachListeners(emitter) {
+    for (const event of INTERCEPTED_EVENTS) {
+        emitter.removeAllListeners(event);
+    }
+}
+function restoreListeners(emitter, snapshot) {
+    for (const [event, listeners] of snapshot) {
+        for (const fn of listeners) {
+            emitter.on(event, fn);
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Core pipeline execution
+// ---------------------------------------------------------------------------
+/**
+ * Execute multiple queries using the Postgres extended-query pipeline protocol.
+ *
+ * All protocol messages are buffered into a single TCP write via cork/uncork.
+ * The backend processes them in order and sends back results which our state
+ * machine collects.
+ *
+ * @param client - A pg PoolClient with an accessible Connection
+ * @param queries - Array of DeferredQuery descriptors
+ * @param options - Pipeline options (transactional, timeout)
+ * @returns Array of transformed results in the same order as queries
+ */
+export async function runPipelined(client, queries, options = {}) {
+    const { transactional = true, timeout } = options;
+    const connection = client.connection;
+    // Snapshot and detach the Client's listeners
+    const savedListeners = snapshotListeners(connection);
+    detachListeners(connection);
+    // Block the Client from processing any queued queries while we own the connection
+    const savedReadyForQuery = client.readyForQuery;
+    client.readyForQuery = false;
+    return new Promise((resolve, reject) => {
+        // -----------------------------------------------------------------------
+        // State machine
+        // -----------------------------------------------------------------------
+        /**
+         * Total expected commandComplete events:
+         * - Transactional: BEGIN + N queries + COMMIT = N + 2
+         * - Non-transactional: N queries
+         */
+        const totalCommands = transactional ? queries.length + 2 : queries.length;
+        /**
+         * Expected readyForQuery events:
+         * - Transactional: 1 (single Sync at end)
+         * - Non-transactional: N (one Sync per query)
+         */
+        const expectedRfq = transactional ? 1 : queries.length;
+        // Results array: one Result per query (not counting BEGIN/COMMIT)
+        const results = [];
+        for (let i = 0; i < queries.length; i++) {
+            results.push(new Result(undefined, client._types));
+        }
+        // commandComplete counter — tracks position across all commands
+        let commandIndex = 0;
+        // readyForQuery counter
+        let rfqCount = 0;
+        // First error encountered
+        let pipelineError = null;
+        // For non-transactional mode: per-query error tracking
+        const queryErrors = new Array(queries.length).fill(null);
+        // Timeout handle
+        let timeoutHandle;
+        // Whether cleanup has already been performed
+        let cleaned = false;
+        /**
+         * Map commandComplete index to the corresponding results[] index.
+         * In transactional mode: index 0 = BEGIN, 1..N = queries, N+1 = COMMIT
+         * In non-transactional mode: index maps 1:1
+         */
+        function commandToQueryIndex(cmdIdx) {
+            if (transactional) {
+                if (cmdIdx === 0 || cmdIdx === totalCommands - 1)
+                    return null;
+                return cmdIdx - 1;
+            }
+            return cmdIdx;
+        }
+        /** Get the current query-results index for row data */
+        function currentQueryIndex() {
+            return commandToQueryIndex(commandIndex);
+        }
+        // -----------------------------------------------------------------------
+        // Cleanup: restore listeners
+        // -----------------------------------------------------------------------
+        function cleanup() {
+            if (cleaned)
+                return;
+            cleaned = true;
+            if (timeoutHandle !== undefined) {
+                clearTimeout(timeoutHandle);
+                timeoutHandle = undefined;
+            }
+            // Remove our listeners
+            detachListeners(connection);
+            // Restore original listeners
+            restoreListeners(connection, savedListeners);
+            // Restore readyForQuery so the Client can process its queue
+            client.readyForQuery = savedReadyForQuery;
+        }
+        // -----------------------------------------------------------------------
+        // Finalize: called on final readyForQuery
+        // -----------------------------------------------------------------------
+        function finalize() {
+            cleanup();
+            if (transactional && pipelineError) {
+                reject(pipelineError);
+                return;
+            }
+            if (!transactional && pipelineError) {
+                // In non-transactional mode, attach partial results to the error
+                const partialResults = [];
+                for (let i = 0; i < queries.length; i++) {
+                    const qErr = queryErrors[i];
+                    if (qErr) {
+                        partialResults.push({ status: 'error', error: qErr });
+                    }
+                    else {
+                        try {
+                            const q = queries[i];
+                            partialResults.push({ status: 'ok', value: q.transform(results[i]) });
+                        }
+                        catch (transformErr) {
+                            partialResults.push({ status: 'error', error: transformErr });
+                        }
+                    }
+                }
+                pipelineError.results = partialResults;
+                reject(pipelineError);
+                return;
+            }
+            // All succeeded — transform results
+            try {
+                const transformed = [];
+                for (let i = 0; i < queries.length; i++) {
+                    const q = queries[i];
+                    transformed.push(q.transform(results[i]));
+                }
+                resolve(transformed);
+            }
+            catch (err) {
+                reject(err);
+            }
+        }
+        // -----------------------------------------------------------------------
+        // Event handlers
+        // -----------------------------------------------------------------------
+        function onParseComplete() {
+            // No action needed — anonymous prepared statements
+        }
+        function onBindComplete() {
+            // No action needed
+        }
+        function onNoData() {
+            // DML without RETURNING — no RowDescription follows. Fine.
+        }
+        function onRowDescription(msg) {
+            const qIdx = currentQueryIndex();
+            if (qIdx !== null && qIdx >= 0 && qIdx < results.length) {
+                results[qIdx].addFields(msg.fields);
+            }
+        }
+        function onDataRow(msg) {
+            const qIdx = currentQueryIndex();
+            if (qIdx !== null && qIdx >= 0 && qIdx < results.length) {
+                const result = results[qIdx];
+                const row = result.parseRow(msg.fields);
+                result.addRow(row);
+            }
+        }
+        function onCommandComplete(msg) {
+            const qIdx = currentQueryIndex();
+            if (qIdx !== null && qIdx >= 0 && qIdx < results.length) {
+                results[qIdx].addCommandComplete(msg);
+            }
+            commandIndex++;
+        }
+        function onEmptyQuery() {
+            // Treat like a commandComplete with no data
+            commandIndex++;
+        }
+        function onErrorMessage(msg) {
+            const wrapped = wrapPgError(msg);
+            const error = wrapped instanceof Error ? wrapped : new Error(String(wrapped));
+            if (transactional) {
+                // In transactional mode, the first error aborts everything.
+                // Postgres marks the transaction as aborted; subsequent commands
+                // until Sync all return errors which we absorb.
+                if (!pipelineError) {
+                    const qIdx = currentQueryIndex();
+                    if (qIdx !== null && qIdx >= 0 && qIdx < queries.length) {
+                        error.failedIndex = qIdx;
+                        error.failedTag = queries[qIdx].tag;
+                    }
+                    pipelineError = error;
+                }
+            }
+            else {
+                // Non-transactional: record per-query error
+                const qIdx = currentQueryIndex();
+                if (qIdx !== null && qIdx >= 0 && qIdx < queries.length) {
+                    queryErrors[qIdx] = error;
+                    if (!pipelineError) {
+                        error.failedIndex = qIdx;
+                        error.failedTag = queries[qIdx].tag;
+                        pipelineError = error;
+                    }
+                }
+            }
+            // Advance command index past the failed command
+            commandIndex++;
+        }
+        function onPortalSuspended() {
+            // We don't use row-limited portals
+        }
+        function onReadyForQuery() {
+            rfqCount++;
+            if (rfqCount >= expectedRfq) {
+                finalize();
+            }
+        }
+        // -----------------------------------------------------------------------
+        // Attach our listeners
+        // -----------------------------------------------------------------------
+        connection.on('parseComplete', onParseComplete);
+        connection.on('bindComplete', onBindComplete);
+        connection.on('noData', onNoData);
+        connection.on('rowDescription', onRowDescription);
+        connection.on('dataRow', onDataRow);
+        connection.on('commandComplete', onCommandComplete);
+        connection.on('emptyQuery', onEmptyQuery);
+        connection.on('errorMessage', onErrorMessage);
+        connection.on('portalSuspended', onPortalSuspended);
+        connection.on('readyForQuery', onReadyForQuery);
+        connection.on('notice', () => { });
+        connection.on('copyInResponse', () => { });
+        connection.on('copyData', () => { });
+        // -----------------------------------------------------------------------
+        // Timeout
+        // -----------------------------------------------------------------------
+        if (timeout && timeout > 0) {
+            timeoutHandle = setTimeout(() => {
+                pipelineError = new Error(`[turbine] Pipeline timed out after ${timeout}ms`);
+                if (connection.stream.destroy) {
+                    connection.stream.destroy(pipelineError);
+                }
+                cleanup();
+                reject(pipelineError);
+            }, timeout);
+        }
+        // -----------------------------------------------------------------------
+        // Send protocol messages — all in one TCP flush
+        // -----------------------------------------------------------------------
+        try {
+            if (connection.stream.cork) {
+                connection.stream.cork();
+            }
+            if (transactional) {
+                // ---- Transactional mode ----
+                // BEGIN + N×(parse+bind+describe+execute) + COMMIT + sync
+                // One sync = one ReadyForQuery
+                // BEGIN
+                connection.parse({ text: 'BEGIN', name: '' });
+                connection.bind({ portal: '', statement: '', values: [], valueMapper: prepareValue });
+                connection.execute({ portal: '', rows: 0 });
+                // Each query
+                for (const q of queries) {
+                    connection.parse({ text: q.sql, name: '' });
+                    connection.bind({
+                        portal: '',
+                        statement: '',
+                        values: q.params,
+                        valueMapper: prepareValue,
+                    });
+                    connection.describe({ type: 'P', name: '' });
+                    connection.execute({ portal: '', rows: 0 });
+                }
+                // COMMIT
+                connection.parse({ text: 'COMMIT', name: '' });
+                connection.bind({ portal: '', statement: '', values: [], valueMapper: prepareValue });
+                connection.execute({ portal: '', rows: 0 });
+                // Single Sync
+                connection.sync();
+            }
+            else {
+                // ---- Non-transactional mode ----
+                // N×(parse+bind+describe+execute+sync)
+                // Each sync = one ReadyForQuery
+                // All messages still go in one cork/uncork (one TCP flush)
+                for (const q of queries) {
+                    connection.parse({ text: q.sql, name: '' });
+                    connection.bind({
+                        portal: '',
+                        statement: '',
+                        values: q.params,
+                        valueMapper: prepareValue,
+                    });
+                    connection.describe({ type: 'P', name: '' });
+                    connection.execute({ portal: '', rows: 0 });
+                    connection.sync();
+                }
+            }
+            if (connection.stream.uncork) {
+                connection.stream.uncork();
+            }
+        }
+        catch (err) {
+            cleanup();
+            reject(err);
+        }
+    });
+}
+// ---------------------------------------------------------------------------
+// Capability detection
+// ---------------------------------------------------------------------------
+/**
+ * Check whether a pool client supports the extended-query pipeline protocol.
+ *
+ * Returns true if the client has a Connection object with the required wire
+ * protocol methods (parse, bind, describe, execute, sync) and is an EventEmitter.
+ *
+ * Returns false for HTTP-based drivers (Neon HTTP, Vercel Postgres), mock pools,
+ * and any pool that doesn't expose pg internals.
+ */
+export function supportsExtendedPipeline(poolClient) {
+    if (!poolClient || typeof poolClient !== 'object')
+        return false;
+    const client = poolClient;
+    const conn = client.connection;
+    if (!conn || typeof conn !== 'object')
+        return false;
+    const c = conn;
+    return (typeof c.parse === 'function' &&
+        typeof c.bind === 'function' &&
+        typeof c.describe === 'function' &&
+        typeof c.execute === 'function' &&
+        typeof c.sync === 'function' &&
+        typeof c.on === 'function');
+}
+//# sourceMappingURL=pipeline-submittable.js.map

package/dist/pipeline.d.ts

@@ -7,21 +7,41 @@
  * How it works:
  *   1. Each query method (findUnique, count, etc.) can produce a DeferredQuery descriptor
  *      containing the SQL, params, and a transform function.
- *   2. pipeline() collects these descriptors, executes them in a single Postgres
- *      pipeline/transaction, and maps each result through its transform.
+ *   2. pipeline() collects these descriptors, checks whether the underlying pool client
+ *      supports the extended-query pipeline protocol, and either:
+ *      (a) executes them via real Postgres pipeline protocol (one TCP flush), or
+ *      (b) falls back to sequential execution on a single connection.
  *
- * In the production Turbine engine, this would go through the Rust proxy which uses
- * actual Postgres pipeline protocol (libpq PQpipelineEnter). For the TS SDK prototype,
- * we simulate it by running queries concurrently on a single connection or via
- * a multi-statement batch.
+ * Real pipeline mode uses `src/pipeline-submittable.ts` which drives the pg Connection's
+ * wire-protocol methods (parse/bind/describe/execute/sync) directly with listener-swap.
+ *
+ * Sequential fallback covers HTTP-based drivers (Neon HTTP, Vercel Postgres, Cloudflare
+ * Hyperdrive), mock pools in tests, and any pool that doesn't expose pg internals.
  */
 import type pg from 'pg';
 import type { DeferredQuery } from './query.js';
+export interface PipelineOptions {
+    /**
+     * Whether to wrap the pipeline in a transaction (default: true).
+     *
+     * - `true` (default): All queries execute atomically within BEGIN/COMMIT.
+     *   If any query fails, the entire batch is rolled back.
+     *
+     * - `false`: Each query is independent. A failure in one query does NOT
+     *   affect others. On partial failure, a `PipelineError` is thrown with
+     *   per-query results in `.results`.
+     */
+    transactional?: boolean;
+    /** Timeout in milliseconds. If exceeded, the connection is destroyed. */
+    timeout?: number;
+}
 /**
  * Execute multiple deferred queries in a single batch.
  *
- * Uses a single connection from the pool and runs all queries within
- * a transaction to guarantee consistency and minimize round-trips.
+ * On pg.Pool-backed connections with the standard TCP driver, this uses the
+ * real Postgres extended-query pipeline protocol for true 1-RTT execution.
+ * On HTTP-based drivers (Neon HTTP, Vercel Postgres, etc.) or mock pools,
+ * it falls back to sequential execution on a single connection.
  *
  * @example
  * ```ts
@@ -32,7 +52,15 @@ import type { DeferredQuery } from './query.js';
  * ]);
  * ```
  */
-export declare function executePipeline<T extends readonly DeferredQuery<unknown>[]>(pool: pg.Pool, queries: T): Promise<PipelineResults<T>>;
+export declare function executePipeline<T extends readonly DeferredQuery<unknown>[]>(pool: pg.Pool, queries: T, options?: PipelineOptions): Promise<PipelineResults<T>>;
+/**
+ * Check whether a pool supports the real pipeline protocol.
+ * Call this to determine at runtime whether pipelines will use the fast path
+ * or fall back to sequential execution.
+ *
+ * Note: This acquires and immediately releases a connection to inspect it.
+ */
+export declare function pipelineSupported(pool: pg.Pool): Promise<boolean>;
 /**
  * Extract the result types from a tuple of DeferredQuery objects.
  * If you pass [DeferredQuery<User>, DeferredQuery<number>, DeferredQuery<Post[]>],

package/dist/pipeline.js

@@ -7,23 +7,71 @@
  * How it works:
  *   1. Each query method (findUnique, count, etc.) can produce a DeferredQuery descriptor
  *      containing the SQL, params, and a transform function.
- *   2. pipeline() collects these descriptors, executes them in a single Postgres
- *      pipeline/transaction, and maps each result through its transform.
+ *   2. pipeline() collects these descriptors, checks whether the underlying pool client
+ *      supports the extended-query pipeline protocol, and either:
+ *      (a) executes them via real Postgres pipeline protocol (one TCP flush), or
+ *      (b) falls back to sequential execution on a single connection.
  *
- * In the production Turbine engine, this would go through the Rust proxy which uses
- * actual Postgres pipeline protocol (libpq PQpipelineEnter). For the TS SDK prototype,
- * we simulate it by running queries concurrently on a single connection or via
- * a multi-statement batch.
+ * Real pipeline mode uses `src/pipeline-submittable.ts` which drives the pg Connection's
+ * wire-protocol methods (parse/bind/describe/execute/sync) directly with listener-swap.
+ *
+ * Sequential fallback covers HTTP-based drivers (Neon HTTP, Vercel Postgres, Cloudflare
+ * Hyperdrive), mock pools in tests, and any pool that doesn't expose pg internals.
  */
 import { wrapPgError } from './errors.js';
+import { runPipelined, supportsExtendedPipeline } from './pipeline-submittable.js';
+/**
+ * Execute queries sequentially on an already-acquired connection.
+ * This is the fallback path for clients that don't support the extended-query
+ * pipeline protocol (HTTP drivers, mocks, etc.).
+ *
+ * The caller is responsible for acquiring the client and releasing it after
+ * this function completes (in the finally block).
+ */
+async function runSequential(client, queries, options = {}) {
+    const { transactional = true } = options;
+    try {
+        if (transactional) {
+            await client.query('BEGIN');
+        }
+        const results = [];
+        for (const q of queries) {
+            let raw;
+            try {
+                raw = await client.query(q.sql, q.params);
+            }
+            catch (err) {
+                throw wrapPgError(err);
+            }
+            results.push(q.transform(raw));
+        }
+        if (transactional) {
+            await client.query('COMMIT');
+        }
+        return results;
+    }
+    catch (err) {
+        if (transactional) {
+            try {
+                await client.query('ROLLBACK');
+            }
+            catch {
+                // Best-effort rollback
+            }
+        }
+        throw err;
+    }
+}
 // ---------------------------------------------------------------------------
-// Pipeline executor
+// Pipeline executor (public)
 // ---------------------------------------------------------------------------
 /**
  * Execute multiple deferred queries in a single batch.
  *
- * Uses a single connection from the pool and runs all queries within
- * a transaction to guarantee consistency and minimize round-trips.
+ * On pg.Pool-backed connections with the standard TCP driver, this uses the
+ * real Postgres extended-query pipeline protocol for true 1-RTT execution.
+ * On HTTP-based drivers (Neon HTTP, Vercel Postgres, etc.) or mock pools,
+ * it falls back to sequential execution on a single connection.
  *
  * @example
  * ```ts
@@ -34,43 +82,47 @@ import { wrapPgError } from './errors.js';
  * ]);
  * ```
  */
-export async function executePipeline(pool, queries) {
+export async function executePipeline(pool, queries, options) {
     if (queries.length === 0) {
         return [];
     }
-    // Acquire a single connection for the entire batch
+    // Acquire a single client — reused for both capability check and execution
     const client = await pool.connect();
     try {
-        // Wrap in a transaction for consistency
-        await client.query('BEGIN');
-        // Execute all queries on the same connection — in sequence on a single
-        // connection this avoids pool checkout overhead, and the Postgres server
-        // processes them as a tight batch.
-        //
-        // Execute queries sequentially on the same connection to avoid the
-        // "already executing a query" deprecation in pg@8. This is still faster
-        // than separate pool checkouts because we skip N-1 acquire/release cycles.
-        // Future: use actual Postgres pipeline protocol for true pipelining.
-        const results = [];
-        for (const q of queries) {
-            let raw;
-            try {
-                raw = await client.query(q.sql, q.params);
-            }
-            catch (err) {
-                throw wrapPgError(err);
-            }
-            results.push(q.transform(raw));
+        if (supportsExtendedPipeline(client)) {
+            // Real pipeline path — uses extended-query protocol wire methods
+            const pipelineOptions = {
+                transactional: options?.transactional ?? true,
+                timeout: options?.timeout,
+            };
+            const results = await runPipelined(client, queries, pipelineOptions);
+            return results;
         }
-        await client.query('COMMIT');
-        return results;
-    }
-    catch (err) {
-        await client.query('ROLLBACK');
-        throw err;
+        // Sequential fallback — reuses the same client
+        return await runSequential(client, queries, options);
     }
     finally {
         client.release();
     }
 }
+/**
+ * Check whether a pool supports the real pipeline protocol.
+ * Call this to determine at runtime whether pipelines will use the fast path
+ * or fall back to sequential execution.
+ *
+ * Note: This acquires and immediately releases a connection to inspect it.
+ */
+export async function pipelineSupported(pool) {
+    let client;
+    try {
+        client = await pool.connect();
+        return supportsExtendedPipeline(client);
+    }
+    catch {
+        return false;
+    }
+    finally {
+        client?.release();
+    }
+}
 //# sourceMappingURL=pipeline.js.map