pgserve 1.1.5-rc.1 → 1.1.6

package/.husky/pre-commit

@@ -1,2 +1,2 @@
-pnpm exec lint-staged
-pnpm deadcode
+bunx lint-staged
+bun run deadcode

package/bun.lock

@@ -17,23 +17,23 @@
         "pg": "^8.16.3",
       },
       "optionalDependencies": {
-        "@embedded-postgres/darwin-arm64": "17.7.0-beta.15",
-        "@embedded-postgres/darwin-x64": "17.7.0-beta.15",
-        "@embedded-postgres/linux-x64": "17.7.0-beta.15",
-        "@embedded-postgres/windows-x64": "17.7.0-beta.15",
+        "@embedded-postgres/darwin-arm64": "18.2.0-beta.16",
+        "@embedded-postgres/darwin-x64": "18.2.0-beta.16",
+        "@embedded-postgres/linux-x64": "18.2.0-beta.16",
+        "@embedded-postgres/windows-x64": "18.2.0-beta.16",
       },
     },
   },
   "packages": {
     "@electric-sql/pglite": ["@electric-sql/pglite@0.2.17", "", {}, "sha512-qEpKRT2oUaWDH6tjRxLHjdzMqRUGYDnGZlKrnL4dJ77JVMcP2Hpo3NYnOSPKdZdeec57B6QPprCUFg0picx5Pw=="],
 
-    "@embedded-postgres/darwin-arm64": ["@embedded-postgres/darwin-arm64@17.7.0-beta.15", "", { "os": "darwin", "cpu": "arm64" }, "sha512-yIo9ACgWI3U84lZpRD2G3ELzVM7fXCaTWCpH+9+lLeEj7+F9eTWYsG3DE/fcsPcDcDDxhLTqX71p9oFeX/KOZA=="],
+    "@embedded-postgres/darwin-arm64": ["@embedded-postgres/darwin-arm64@18.2.0-beta.16", "", { "os": "darwin", "cpu": "arm64" }, "sha512-wnswaF+uDvGeitqajJ8v8xOG4ttFrzixElwKNe2MIxBXSLWPV3xhi6tBY0Sjw8Lmiu6UG9vNLFZSjHPrIeokBg=="],
 
-    "@embedded-postgres/darwin-x64": ["@embedded-postgres/darwin-x64@17.7.0-beta.15", "", { "os": "darwin", "cpu": "x64" }, "sha512-KXXBbiyJ8fNYNQt9ondhVUayQ0+qCi6Uoma23Oa40xoyzXY6kRNjXwlGB7Bvf9ZV8PsiBdlnEv/fSpYeRcDjSA=="],
+    "@embedded-postgres/darwin-x64": ["@embedded-postgres/darwin-x64@18.2.0-beta.16", "", { "os": "darwin", "cpu": "x64" }, "sha512-u9WtTPxRuO0uOny5IniXHSDaLmtOujwzDoExIV/jFT0Fu8SzpX7wdoPbsSPBLgyQWdr/nPA77K9QI4r6P1/fKA=="],
 
-    "@embedded-postgres/linux-x64": ["@embedded-postgres/linux-x64@17.7.0-beta.15", "", { "os": "linux", "cpu": "x64" }, "sha512-HeaxSHsw6ccVh8l5iC4OgXqvaaCGWnnZR9CpgNgrAfnKPPGiEhUPBmO2XhEsFQIhc+ad/+36h0NTvKo4bdi40w=="],
+    "@embedded-postgres/linux-x64": ["@embedded-postgres/linux-x64@18.2.0-beta.16", "", { "os": "linux", "cpu": "x64" }, "sha512-BIt485ioL8/AwDgw37IcdraOfRgHNDOtGM6Hh63vnNaUAG4Z0qtJd5zXS5fr2wZTEsYHyC5PC60k7zkCRZXSzg=="],
 
-    "@embedded-postgres/windows-x64": ["@embedded-postgres/windows-x64@17.7.0-beta.15", "", { "os": "win32", "cpu": "x64" }, "sha512-Oq11yyKxISjefuYdKljcp3Q+uxx237zn9YpP9hO43+6Feorq7USuMIDqk5ofLSQ30FAnVyTqaIQK8ZIjW+tQXQ=="],
+    "@embedded-postgres/windows-x64": ["@embedded-postgres/windows-x64@18.2.0-beta.16", "", { "os": "win32", "cpu": "x64" }, "sha512-Sj6GhCZrvtMwchATEtWuEmexEBWpRNMHPTUHsqPuyDrHX/XgKfpIxz2/AMHa4sp7SZ0JOHGouH8AFIVsWQrQsQ=="],
 
     "@emnapi/core": ["@emnapi/core@1.7.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.1.0", "tslib": "^2.4.0" } }, "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg=="],
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pgserve",
-  "version": "1.1.5-rc.1",
+  "version": "1.1.6",
   "description": "Embedded PostgreSQL server with true concurrent connections - zero config, auto-provision databases",
   "main": "src/index.js",
   "type": "module",
@@ -41,10 +41,10 @@
   },
   "homepage": "https://github.com/namastexlabs/pgserve#readme",
   "optionalDependencies": {
-    "@embedded-postgres/darwin-arm64": "17.7.0-beta.15",
-    "@embedded-postgres/darwin-x64": "17.7.0-beta.15",
-    "@embedded-postgres/linux-x64": "17.7.0-beta.15",
-    "@embedded-postgres/windows-x64": "17.7.0-beta.15"
+    "@embedded-postgres/darwin-arm64": "18.2.0-beta.16",
+    "@embedded-postgres/darwin-x64": "18.2.0-beta.16",
+    "@embedded-postgres/linux-x64": "18.2.0-beta.16",
+    "@embedded-postgres/windows-x64": "18.2.0-beta.16"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.2.17",

package/src/cluster.js

@@ -23,6 +23,17 @@ const SSL_REQUEST_CODE = 80877103;
 const GSSAPI_REQUEST_CODE = 80877104;
 const CANCEL_REQUEST_CODE = 80877102;
 
+/**
+ * Attempt to write a pending buffer to a target socket.
+ * Returns remaining unwritten bytes, or null if fully flushed.
+ */
+function flushPending(target, pending) {
+  const written = target.write(pending);
+  if (written === pending.byteLength) return null;
+  if (written === 0) return pending;
+  return pending.subarray(written);
+}
+
 // Stats collection constants
 const WORKER_STATS_TIMEOUT_MS = 10000; // Worker stats older than this are considered stale
 const WORKER_STATS_REPORT_INTERVAL_MS = 4000; // How often workers report stats to primary
@@ -100,8 +111,14 @@ class ClusterRouter extends EventEmitter {
         },
         drain(socket) {
           const state = router.socketState.get(socket);
-          if (state?.pgSocket) {
-            state.pgSocket.resume?.();
+          if (!state) return;
+          // Flush any pending PG→Client data
+          if (state.pendingToClient) {
+            state.pendingToClient = flushPending(socket, state.pendingToClient);
+          }
+          // If fully flushed, resume reading from PostgreSQL
+          if (!state.pendingToClient && state.pgSocket) {
+            state.pgSocket.resume();
           }
         }
       }
@@ -182,7 +199,9 @@ class ClusterRouter extends EventEmitter {
       buffer: null,
       pgSocket: null,
       dbName: null,
-      handshakeComplete: false
+      handshakeComplete: false,
+      pendingToPg: null,
+      pendingToClient: null
     });
     this.connections.add(socket);
     this.connectionStats.totalConnected++;
@@ -197,7 +216,17 @@ class ClusterRouter extends EventEmitter {
 
     // If handshake complete, forward to PostgreSQL
     if (state.handshakeComplete && state.pgSocket) {
-      state.pgSocket.write(data);
+      // If there's already pending data, append to it
+      if (state.pendingToPg) {
+        state.pendingToPg = Buffer.concat([state.pendingToPg, data]);
+        return;
+      }
+      const written = state.pgSocket.write(data);
+      if (written < data.byteLength) {
+        // Partial write — buffer remainder and pause client
+        state.pendingToPg = written === 0 ? Buffer.from(data) : Buffer.from(data.subarray(written));
+        socket.pause();
+      }
       return;
     }
 
@@ -250,50 +279,47 @@ class ClusterRouter extends EventEmitter {
 
       const router = this;
 
-      // Connect to PRIMARY's PostgreSQL using Bun.connect()
-      if (this.pgSocketPath) {
-        state.pgSocket = await Bun.connect({
-          unix: this.pgSocketPath,
-          socket: {
-            data(_pgSocket, pgData) {
-              socket.write(pgData);
-            },
-            open(pgSocket) {
-              pgSocket.write(startupMessage);
-              state.handshakeComplete = true;
-            },
-            close(_pgSocket) {
-              socket.end();
-            },
-            error(_pgSocket, error) {
-              router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
-              socket.end();
-            },
-            drain(_pgSocket) {}
+      // Shared handler for pgSocket (used by both unix and TCP paths)
+      const pgHandler = {
+        data(_pgSocket, pgData) {
+          // Forward PostgreSQL response to client with backpressure
+          if (state.pendingToClient) {
+            state.pendingToClient = Buffer.concat([state.pendingToClient, pgData]);
+            return;
           }
-        });
-      } else {
-        state.pgSocket = await Bun.connect({
-          hostname: '127.0.0.1',
-          port: this.pgPort,
-          socket: {
-            data(_pgSocket, pgData) {
-              socket.write(pgData);
-            },
-            open(pgSocket) {
-              pgSocket.write(startupMessage);
-              state.handshakeComplete = true;
-            },
-            close(_pgSocket) {
-              socket.end();
-            },
-            error(_pgSocket, error) {
-              router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
-              socket.end();
-            },
-            drain(_pgSocket) {}
+          const written = socket.write(pgData);
+          if (written < pgData.byteLength) {
+            state.pendingToClient = written === 0 ? Buffer.from(pgData) : Buffer.from(pgData.subarray(written));
+            _pgSocket.pause();
           }
-        });
+        },
+        open(pgSocket) {
+          pgSocket.write(startupMessage);
+          state.handshakeComplete = true;
+        },
+        close(_pgSocket) {
+          socket.end();
+        },
+        error(_pgSocket, error) {
+          router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
+          socket.end();
+        },
+        drain(_pgSocket) {
+          // Flush any pending Client→PG data
+          if (state.pendingToPg) {
+            state.pendingToPg = flushPending(_pgSocket, state.pendingToPg);
+          }
+          // If fully flushed, resume reading from client
+          if (!state.pendingToPg) {
+            socket.resume();
+          }
+        }
+      };
+
+      if (this.pgSocketPath) {
+        state.pgSocket = await Bun.connect({ unix: this.pgSocketPath, socket: pgHandler });
+      } else {
+        state.pgSocket = await Bun.connect({ hostname: '127.0.0.1', port: this.pgPort, socket: pgHandler });
       }
     } catch (error) {
       this.logger.error({ dbName, err: error }, 'Connection error');
@@ -306,8 +332,10 @@ class ClusterRouter extends EventEmitter {
    */
   handleSocketClose(socket) {
     const state = this.socketState.get(socket);
-    if (state?.pgSocket) {
-      state.pgSocket.end();
+    if (state) {
+      state.pendingToPg = null;
+      state.pendingToClient = null;
+      if (state.pgSocket) state.pgSocket.end();
     }
     this.connections.delete(socket);
     this.socketState.delete(socket);
@@ -334,8 +362,10 @@ class ClusterRouter extends EventEmitter {
     if (error.code !== 'ECONNRESET') {
       this.logger.error({ err: error, dbName: state?.dbName }, 'Socket error');
     }
-    if (state?.pgSocket) {
-      state.pgSocket.end();
+    if (state) {
+      state.pendingToPg = null;
+      state.pendingToClient = null;
+      if (state.pgSocket) state.pgSocket.end();
     }
     this.connections.delete(socket);
     this.socketState.delete(socket);

package/src/postgres.js

@@ -66,7 +66,7 @@ async function downloadPostgresBinaries() {
 
   const platformKey = getPlatformKey();
   const pkgName = `@embedded-postgres/${platformKey}`;
-  const pkgVersion = '17.7.0-beta.15';
+  const pkgVersion = '18.2.0-beta.16';
 
   console.log(`[pgserve] PostgreSQL binaries not found.`);
   console.log(`[pgserve] Downloading ${pkgName}@${pkgVersion}...`);

package/src/router.js

@@ -28,6 +28,17 @@ const SSL_REQUEST_CODE = 80877103;
 const GSSAPI_REQUEST_CODE = 80877104;
 const CANCEL_REQUEST_CODE = 80877102;
 
+/**
+ * Attempt to write a pending buffer to a target socket.
+ * Returns remaining unwritten bytes, or null if fully flushed.
+ */
+function flushPending(target, pending) {
+  const written = target.write(pending);
+  if (written === pending.byteLength) return null;
+  if (written === 0) return pending;
+  return pending.subarray(written);
+}
+
 /**
  * Multi-Tenant Router Server
  */
@@ -175,12 +186,17 @@ export class MultiTenantRouter extends EventEmitter {
         error(socket, error) {
           router.handleSocketError(socket, error);
         },
-        // Called when socket is ready to receive more data
+        // Called when client socket is ready to receive more data
         drain(socket) {
-          // Resume reading from PostgreSQL if backpressure cleared
           const state = router.socketState.get(socket);
-          if (state?.pgSocket) {
-            state.pgSocket.resume?.();
+          if (!state) return;
+          // Flush any pending PG→Client data
+          if (state.pendingToClient) {
+            state.pendingToClient = flushPending(socket, state.pendingToClient);
+          }
+          // If fully flushed, resume reading from PostgreSQL
+          if (!state.pendingToClient && state.pgSocket) {
+            state.pgSocket.resume();
           }
         }
       }
@@ -214,7 +230,9 @@ export class MultiTenantRouter extends EventEmitter {
       buffer: null,
       pgSocket: null,
       dbName: null,
-      handshakeComplete: false
+      handshakeComplete: false,
+      pendingToPg: null,
+      pendingToClient: null
     });
 
     // Track connection
@@ -231,9 +249,16 @@ export class MultiTenantRouter extends EventEmitter {
 
     // If handshake complete, forward to PostgreSQL
     if (state.handshakeComplete && state.pgSocket) {
+      // If there's already pending data, append to it
+      if (state.pendingToPg) {
+        state.pendingToPg = Buffer.concat([state.pendingToPg, data]);
+        return;
+      }
       const written = state.pgSocket.write(data);
-      if (written === 0) {
-        // Backpressure - Bun handles this automatically
+      if (written < data.byteLength) {
+        // Partial write — buffer remainder and pause client
+        state.pendingToPg = written === 0 ? Buffer.from(data) : Buffer.from(data.subarray(written));
+        socket.pause();
       }
       return;
     }
@@ -300,56 +325,47 @@ export class MultiTenantRouter extends EventEmitter {
       const socketPath = this.pgManager.getSocketPath();
       const router = this;
 
-      if (socketPath) {
-        // Unix socket connection (Linux/macOS) - ~30% faster than TCP
-        state.pgSocket = await Bun.connect({
-          unix: socketPath,
-          socket: {
-            data(_pgSocket, pgData) {
-              // Forward PostgreSQL response to client
-              socket.write(pgData);
-            },
-            open(pgSocket) {
-              // Send buffered startup message to PostgreSQL
-              pgSocket.write(startupMessage);
-              state.handshakeComplete = true;
-            },
-            close(_pgSocket) {
-              // PostgreSQL closed - close client
-              socket.end();
-            },
-            error(_pgSocket, error) {
-              router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
-              socket.end();
-            },
-            drain(_pgSocket) {
-              // Ready for more data
-            }
+      // Shared handler for pgSocket (used by both unix and TCP paths)
+      const pgHandler = {
+        data(_pgSocket, pgData) {
+          // Forward PostgreSQL response to client with backpressure
+          if (state.pendingToClient) {
+            state.pendingToClient = Buffer.concat([state.pendingToClient, pgData]);
+            return;
           }
-        });
-      } else {
-        // TCP fallback (Windows)
-        state.pgSocket = await Bun.connect({
-          hostname: '127.0.0.1',
-          port: this.pgPort,
-          socket: {
-            data(_pgSocket, pgData) {
-              socket.write(pgData);
-            },
-            open(pgSocket) {
-              pgSocket.write(startupMessage);
-              state.handshakeComplete = true;
-            },
-            close(_pgSocket) {
-              socket.end();
-            },
-            error(_pgSocket, error) {
-              router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
-              socket.end();
-            },
-            drain(_pgSocket) {}
+          const written = socket.write(pgData);
+          if (written < pgData.byteLength) {
+            state.pendingToClient = written === 0 ? Buffer.from(pgData) : Buffer.from(pgData.subarray(written));
+            _pgSocket.pause();
+          }
+        },
+        open(pgSocket) {
+          pgSocket.write(startupMessage);
+          state.handshakeComplete = true;
+        },
+        close(_pgSocket) {
+          socket.end();
+        },
+        error(_pgSocket, error) {
+          router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
+          socket.end();
+        },
+        drain(_pgSocket) {
+          // Flush any pending Client→PG data
+          if (state.pendingToPg) {
+            state.pendingToPg = flushPending(_pgSocket, state.pendingToPg);
+          }
+          // If fully flushed, resume reading from client
+          if (!state.pendingToPg) {
+            socket.resume();
           }
-        });
+        }
+      };
+
+      if (socketPath) {
+        state.pgSocket = await Bun.connect({ unix: socketPath, socket: pgHandler });
+      } else {
+        state.pgSocket = await Bun.connect({ hostname: '127.0.0.1', port: this.pgPort, socket: pgHandler });
       }
 
       this.emit('connection', { dbName, socket });
@@ -365,8 +381,10 @@ export class MultiTenantRouter extends EventEmitter {
    */
   handleSocketClose(socket) {
     const state = this.socketState.get(socket);
-    if (state?.pgSocket) {
-      state.pgSocket.end();
+    if (state) {
+      state.pendingToPg = null;
+      state.pendingToClient = null;
+      if (state.pgSocket) state.pgSocket.end();
     }
     this.connections.delete(socket);
     this.socketState.delete(socket);
@@ -381,8 +399,10 @@ export class MultiTenantRouter extends EventEmitter {
     if (error.code !== 'ECONNRESET') {
       this.logger.error({ err: error, dbName: state?.dbName }, 'Socket error');
     }
-    if (state?.pgSocket) {
-      state.pgSocket.end();
+    if (state) {
+      state.pendingToPg = null;
+      state.pendingToClient = null;
+      if (state.pgSocket) state.pgSocket.end();
     }
     this.connections.delete(socket);
     this.socketState.delete(socket);

package/tests/backpressure.test.js

@@ -0,0 +1,167 @@
+/**
+ * Backpressure / Large Message Regression Tests
+ *
+ * Reproduces the deadlock from issue #14: TCP proxy drops bytes when
+ * socket buffers are full, causing PostgreSQL to wait forever for the
+ * remainder of a truncated wire protocol message.
+ */
+
+import { startMultiTenantServer } from '../src/index.js';
+import pg from 'pg';
+import { test, expect } from 'bun:test';
+import fs from 'fs';
+
+const { Client } = pg;
+
+const TEST_PORT = 15433;
+const testDataDir = './test-data-backpressure';
+
+function cleanup() {
+  if (fs.existsSync(testDataDir)) {
+    fs.rmSync(testDataDir, { recursive: true, force: true });
+  }
+}
+
+/** Create a connected pg.Client */
+async function connect(dbName) {
+  const client = new Client({
+    host: '127.0.0.1',
+    port: TEST_PORT,
+    database: dbName,
+    user: 'postgres',
+    password: 'postgres',
+  });
+  await client.connect();
+  return client;
+}
+
+test('Large INSERT (~360KB payload) does not deadlock', async () => {
+  cleanup();
+  const router = await startMultiTenantServer({
+    port: TEST_PORT,
+    baseDir: testDataDir,
+    logLevel: 'warn',
+  });
+
+  let client;
+  try {
+    client = await connect('bp_insert');
+    await client.query('CREATE TABLE big (id SERIAL PRIMARY KEY, payload TEXT)');
+
+    // ~360KB of text — exceeds typical socket buffer size
+    const bigPayload = 'x'.repeat(360_000);
+    await client.query('INSERT INTO big (payload) VALUES ($1)', [bigPayload]);
+
+    const res = await client.query('SELECT length(payload) AS len FROM big');
+    expect(Number(res.rows[0].len)).toBe(360000);
+  } finally {
+    if (client) await client.end().catch(() => {});
+    await router.stop();
+    cleanup();
+  }
+}, 30_000);
+
+test('Large SELECT result (500KB+) does not deadlock', async () => {
+  cleanup();
+  const router = await startMultiTenantServer({
+    port: TEST_PORT,
+    baseDir: testDataDir,
+    logLevel: 'warn',
+  });
+
+  let client;
+  try {
+    client = await connect('bp_select');
+    await client.query('CREATE TABLE chunks (id SERIAL PRIMARY KEY, data TEXT)');
+
+    // Insert many rows that sum to >500KB
+    const chunkSize = 10_000;
+    const numChunks = 60; // 60 * 10KB = 600KB total
+    const chunk = 'y'.repeat(chunkSize);
+
+    for (let i = 0; i < numChunks; i++) {
+      await client.query('INSERT INTO chunks (data) VALUES ($1)', [chunk]);
+    }
+
+    // Fetch all rows in a single result set (PG→Client backpressure)
+    const res = await client.query('SELECT * FROM chunks');
+    expect(res.rows.length).toBe(numChunks);
+    expect(res.rows[0].data.length).toBe(chunkSize);
+  } finally {
+    if (client) await client.end().catch(() => {});
+    await router.stop();
+    cleanup();
+  }
+}, 30_000);
+
+test('Large single query with multi-value INSERT (500KB+)', async () => {
+  cleanup();
+  const router = await startMultiTenantServer({
+    port: TEST_PORT,
+    baseDir: testDataDir,
+    logLevel: 'warn',
+  });
+
+  let client;
+  try {
+    client = await connect('bp_multivalue');
+    await client.query('CREATE TABLE items (id INT, val TEXT)');
+
+    // Build a single INSERT with many value tuples to produce a large wire message
+    const rowCount = 500;
+    const rowValue = 'z'.repeat(1_000); // 1KB per row → ~500KB total
+    const values = [];
+    const params = [];
+    for (let i = 0; i < rowCount; i++) {
+      values.push(`($${i * 2 + 1}, $${i * 2 + 2})`);
+      params.push(i, rowValue);
+    }
+
+    const sql = `INSERT INTO items (id, val) VALUES ${values.join(', ')}`;
+    await client.query(sql, params);
+
+    const res = await client.query('SELECT count(*)::int AS cnt FROM items');
+    expect(res.rows[0].cnt).toBe(rowCount);
+  } finally {
+    if (client) await client.end().catch(() => {});
+    await router.stop();
+    cleanup();
+  }
+}, 30_000);
+
+test('Concurrent large operations (5 clients x 300KB)', async () => {
+  cleanup();
+  const router = await startMultiTenantServer({
+    port: TEST_PORT,
+    baseDir: testDataDir,
+    logLevel: 'warn',
+  });
+
+  try {
+    const numClients = 5;
+    const payloadSize = 300_000;
+    const payload = 'c'.repeat(payloadSize);
+
+    // Run all clients concurrently
+    const results = await Promise.all(
+      Array.from({ length: numClients }, async (_, i) => {
+        const dbName = `bp_concurrent_${i}`;
+        const client = await connect(dbName);
+        await client.query('CREATE TABLE stress (id SERIAL PRIMARY KEY, data TEXT)');
+        await client.query('INSERT INTO stress (data) VALUES ($1)', [payload]);
+
+        const res = await client.query('SELECT length(data) AS len FROM stress');
+        await client.end();
+        return parseInt(res.rows[0].len, 10);
+      })
+    );
+
+    // All clients should have successfully stored the full payload
+    for (const len of results) {
+      expect(len).toBe(payloadSize);
+    }
+  } finally {
+    await router.stop();
+    cleanup();
+  }
+}, 60_000);