pgserve 0.1.5 → 1.0.1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pgserve",
-  "version": "0.1.5",
-  "description": "Multi-instance PostgreSQL embedded server using PGlite - zero config, auto-port allocation, perfect for development and embedded apps",
+  "version": "1.0.1",
+  "description": "Embedded PostgreSQL server with true concurrent connections - zero config, auto-provision databases",
   "main": "src/index.js",
   "type": "module",
   "bin": {
@@ -12,14 +12,12 @@
     "test": "node --test tests/**/*.test.js"
   },
   "keywords": [
-    "pglite",
     "postgresql",
     "embedded",
     "database",
-    "wasm",
-    "electron",
+    "postgres",
     "development",
-    "multi-instance"
+    "multi-tenant"
   ],
   "author": "Namastex Labs <labs@namastex.com>",
   "license": "MIT",
@@ -32,14 +30,19 @@
   },
   "homepage": "https://github.com/namastexlabs/pgserve#readme",
   "dependencies": {
-    "@electric-sql/pglite": "^0.3.14",
-    "@electric-sql/pglite-socket": "^0.0.19",
+    "pg": "^8.16.3",
     "pino": "^10.1.0",
     "pino-pretty": "^13.1.2"
   },
+  "optionalDependencies": {
+    "@embedded-postgres/linux-x64": "17.7.0-beta.15",
+    "@embedded-postgres/darwin-arm64": "17.7.0-beta.15",
+    "@embedded-postgres/darwin-x64": "17.7.0-beta.15",
+    "@embedded-postgres/win32-x64": "17.7.0-beta.15"
+  },
   "devDependencies": {
-    "better-sqlite3": "^11.10.0",
-    "pg": "^8.16.3"
+    "better-sqlite3": "^11.7.0",
+    "@electric-sql/pglite": "^0.2.17"
   },
   "engines": {
     "node": ">=18.0.0"

package/src/cluster.js

@@ -0,0 +1,322 @@
+/**
+ * Cluster Mode for pgserve
+ *
+ * Architecture:
+ * - PRIMARY process: Runs single embedded PostgreSQL instance
+ * - WORKER processes: Only run TCP routing to PRIMARY's PostgreSQL
+ *
+ * This enables multi-core scaling (3-5x throughput on multi-core systems)
+ * while maintaining a single PostgreSQL instance.
+ */
+
+import cluster from 'cluster';
+import os from 'os';
+import net from 'net';
+import pg from 'pg';
+import pino from 'pino';
+import { PostgresManager } from './postgres.js';
+import { extractDatabaseNameFromSocket } from './protocol.js';
+import { EventEmitter } from 'events';
+
+/**
+ * ClusterRouter - Lightweight TCP router for worker processes
+ * Does NOT start PostgreSQL - connects to PRIMARY's PostgreSQL via Unix socket
+ */
+class ClusterRouter extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this.port = options.port || 5432;
+    this.host = options.host || '127.0.0.1';
+    this.pgSocketPath = options.pgSocketPath; // From PRIMARY
+    this.pgPort = options.pgPort;
+    this.pgUser = options.pgUser || 'postgres';
+    this.pgPassword = options.pgPassword || 'postgres';
+    this.autoProvision = options.autoProvision !== false;
+    this.maxConnections = options.maxConnections || 1000;
+
+    this.logger = pino({ level: options.logLevel || 'info' });
+    this.adminClient = null;
+    this.server = null;
+    this.connections = new Set();
+    this.setMaxListeners(this.maxConnections + 10);
+  }
+
+  optimizeSocket(socket) {
+    socket.setNoDelay(true);
+    socket.setKeepAlive(true, 60000);
+    socket.setTimeout(0);
+  }
+
+  async start() {
+    // Admin connection for auto-provisioning databases
+    if (this.autoProvision) {
+      let connectionConfig;
+      if (this.pgSocketPath) {
+        // pg library expects socket DIRECTORY as host, it appends .s.PGSQL.<port>
+        // Socket path format: /tmp/pgserve-sock-xxx/.s.PGSQL.<port>
+        // Extract directory by removing the socket file suffix
+        const socketDir = this.pgSocketPath.replace(/\/\.s\.PGSQL\.\d+$/, '');
+        connectionConfig = {
+          host: socketDir,
+          port: this.pgPort,
+          database: 'postgres',
+          user: this.pgUser,
+          password: this.pgPassword
+        };
+      } else {
+        connectionConfig = {
+          host: '127.0.0.1',
+          port: this.pgPort,
+          database: 'postgres',
+          user: this.pgUser,
+          password: this.pgPassword
+        };
+      }
+
+      this.adminClient = new pg.Client(connectionConfig);
+      await this.adminClient.connect();
+    }
+
+    return new Promise((resolve, reject) => {
+      this.server = net.createServer({
+        allowHalfOpen: false,
+        pauseOnConnect: true
+      }, (socket) => this.handleConnection(socket));
+
+      this.server.maxConnections = this.maxConnections;
+
+      this.server.on('error', (error) => {
+        this.logger.error({ err: error }, 'Router error');
+        this.emit('error', error);
+      });
+
+      this.server.listen(this.port, this.host, () => {
+        this.emit('listening');
+        resolve();
+      });
+    });
+  }
+
+  async createDatabase(dbName) {
+    if (!this.autoProvision || !this.adminClient) return;
+
+    try {
+      const result = await this.adminClient.query(
+        'SELECT 1 FROM pg_database WHERE datname = $1',
+        [dbName]
+      );
+
+      if (result.rows.length === 0) {
+        await this.adminClient.query(`CREATE DATABASE "${dbName}"`);
+      }
+    } catch (error) {
+      // Ignore "already exists" (race condition between workers)
+      if (!error.message.includes('already exists')) {
+        this.logger.error({ database: dbName, err: error }, 'Failed to create database');
+      }
+    }
+  }
+
+  async handleConnection(socket) {
+    this.connections.add(socket);
+    this.optimizeSocket(socket);
+
+    let dbName = null;
+    let pgSocket = null;
+
+    try {
+      const { dbName: extractedDbName, buffered } = await extractDatabaseNameFromSocket(socket);
+      dbName = extractedDbName;
+
+      await this.createDatabase(dbName);
+
+      // Connect to PRIMARY's PostgreSQL
+      if (this.pgSocketPath) {
+        pgSocket = net.connect({ path: this.pgSocketPath });
+      } else {
+        pgSocket = net.connect({ host: '127.0.0.1', port: this.pgPort });
+      }
+
+      await new Promise((resolve, reject) => {
+        pgSocket.once('connect', resolve);
+        pgSocket.once('error', reject);
+      });
+
+      this.optimizeSocket(pgSocket);
+      pgSocket.write(buffered);
+      socket.resume();
+
+      // Bidirectional pipe
+      socket.pipe(pgSocket);
+      pgSocket.pipe(socket);
+
+      const cleanup = () => {
+        this.connections.delete(socket);
+        if (pgSocket && !pgSocket.destroyed) pgSocket.destroy();
+        if (socket && !socket.destroyed) socket.destroy();
+      };
+
+      socket.once('close', cleanup);
+      socket.once('error', cleanup);
+      pgSocket.once('close', () => {
+        if (socket && !socket.destroyed) socket.destroy();
+      });
+      pgSocket.once('error', cleanup);
+
+    } catch (error) {
+      this.logger.error({ dbName, err: error }, 'Connection error');
+      if (pgSocket && !pgSocket.destroyed) pgSocket.destroy();
+      socket.destroy();
+      this.connections.delete(socket);
+    }
+  }
+
+  async stop() {
+    for (const socket of this.connections) {
+      socket.end();
+    }
+    this.connections.clear();
+
+    if (this.adminClient) {
+      await this.adminClient.end();
+    }
+
+    if (this.server) {
+      await new Promise((resolve) => this.server.close(resolve));
+    }
+  }
+}
+
+/**
+ * Start pgserve in cluster mode
+ */
+export async function startClusterServer(options = {}) {
+  const numWorkers = options.workers || os.cpus().length;
+  const port = options.port || 5432;
+  const host = options.host || '127.0.0.1';
+  const pgPort = options.pgPort || (port + 1000);
+
+  if (cluster.isPrimary) {
+    console.log(`[pgserve] Cluster mode: ${numWorkers} workers`);
+
+    // PRIMARY: Start our embedded PostgreSQL (single instance)
+    const logger = pino({ level: options.logLevel || 'info' });
+    const pgManager = new PostgresManager({
+      dataDir: options.baseDir,
+      port: pgPort,
+      logger: logger.child({ component: 'postgres' })
+    });
+
+    await pgManager.start();
+    const pgSocketPath = pgManager.getSocketPath();
+
+    console.log(`[pgserve] Embedded PostgreSQL started`);
+    console.log(`[pgserve] Socket: ${pgSocketPath || `TCP port ${pgPort}`}`);
+
+    const workers = new Map();
+
+    // Fork workers with PostgreSQL connection info
+    for (let i = 0; i < numWorkers; i++) {
+      const worker = cluster.fork({
+        PGSERVE_WORKER: 'true',
+        PGSERVE_PORT: String(port),
+        PGSERVE_HOST: host,
+        PGSERVE_PG_SOCKET: pgSocketPath || '',
+        PGSERVE_PG_PORT: String(pgPort),
+        PGSERVE_PG_USER: 'postgres',
+        PGSERVE_PG_PASSWORD: 'postgres',
+        PGSERVE_LOG_LEVEL: options.logLevel || 'info',
+        PGSERVE_AUTO_PROVISION: options.autoProvision !== false ? 'true' : 'false'
+      });
+      workers.set(worker.id, worker);
+    }
+
+    // Restart dead workers
+    cluster.on('exit', (worker, code, signal) => {
+      console.log(`[pgserve] Worker ${worker.id} died (${signal || code}), restarting...`);
+      workers.delete(worker.id);
+
+      const newWorker = cluster.fork({
+        PGSERVE_WORKER: 'true',
+        PGSERVE_PORT: String(port),
+        PGSERVE_HOST: host,
+        PGSERVE_PG_SOCKET: pgSocketPath || '',
+        PGSERVE_PG_PORT: String(pgPort),
+        PGSERVE_PG_USER: 'postgres',
+        PGSERVE_PG_PASSWORD: 'postgres',
+        PGSERVE_LOG_LEVEL: options.logLevel || 'info',
+        PGSERVE_AUTO_PROVISION: options.autoProvision !== false ? 'true' : 'false'
+      });
+      workers.set(newWorker.id, newWorker);
+    });
+
+    // Wait for workers to be ready
+    let readyCount = 0;
+    await new Promise((resolve) => {
+      cluster.on('message', (worker, message) => {
+        if (message.type === 'ready') {
+          readyCount++;
+          if (readyCount === numWorkers) resolve();
+        }
+      });
+    });
+
+    console.log(`[pgserve] All ${numWorkers} workers ready`);
+    console.log(`[pgserve] Listening on ${host}:${port}`);
+
+    return {
+      workers,
+      pgPort,
+      pgSocketPath,
+      stop: async () => {
+        console.log('[pgserve] Stopping cluster...');
+        for (const worker of workers.values()) {
+          worker.send({ type: 'shutdown' });
+        }
+        await new Promise((resolve) => {
+          const check = setInterval(() => {
+            if (workers.size === 0) {
+              clearInterval(check);
+              resolve();
+            }
+          }, 100);
+        });
+        await pgManager.stop();
+        console.log('[pgserve] Cluster stopped');
+      },
+      getStats: () => ({
+        workers: workers.size,
+        pids: Array.from(workers.values()).map(w => w.process.pid)
+      })
+    };
+  } else {
+    // WORKER: Only run TCP routing, connect to PRIMARY's PostgreSQL
+    const router = new ClusterRouter({
+      port: parseInt(process.env.PGSERVE_PORT) || 5432,
+      host: process.env.PGSERVE_HOST || '127.0.0.1',
+      pgSocketPath: process.env.PGSERVE_PG_SOCKET || null,
+      pgPort: parseInt(process.env.PGSERVE_PG_PORT) || 6432,
+      pgUser: process.env.PGSERVE_PG_USER || 'postgres',
+      pgPassword: process.env.PGSERVE_PG_PASSWORD || 'postgres',
+      logLevel: process.env.PGSERVE_LOG_LEVEL || 'info',
+      autoProvision: process.env.PGSERVE_AUTO_PROVISION === 'true'
+    });
+
+    await router.start();
+
+    // Tell PRIMARY we're ready
+    process.send({ type: 'ready' });
+
+    // Handle shutdown
+    process.on('message', async (message) => {
+      if (message.type === 'shutdown') {
+        await router.stop();
+        process.exit(0);
+      }
+    });
+
+    return router;
+  }
+}
+
+export default startClusterServer;

package/src/index.js

@@ -1,177 +1,14 @@
 /**
- * pgserve - PostgreSQL embedded server using PGlite
+ * pgserve - Embedded PostgreSQL Server
  *
- * Multi-tenant PostgreSQL router using PGlite
- * Single port, auto-provisioning, perfect for multi-user apps and AI agents
+ * True concurrent connections, zero config, auto-provision databases.
+ * Uses embedded-postgres (real PostgreSQL binaries).
  */
 
-// Multi-tenant mode (NEW - recommended)
+// Main exports
 export { MultiTenantRouter, startMultiTenantServer } from './router.js';
-export { InstancePool } from './pool.js';
+export { PostgresManager } from './postgres.js';
+export { SyncManager } from './sync.js';
 
-// Legacy single-instance mode (backwards compatible)
-import { startServer as _startServer, stopServer as _stopServer } from './server.js';
-import { allocatePort, getPortRangeInfo } from './ports.js';
-import {
-  findInstanceByDataDir,
-  findInstanceByPort,
-  listInstances,
-  cleanupStaleInstances
-} from './registry.js';
-import { autoDetect as _autoDetect } from './detector.js';
-
-/**
- * Start a new PGlite server instance
- *
- * @param {Object} options
- * @param {string} options.dataDir - Data directory for the database
- * @param {number} [options.port] - Specific port (optional, auto-allocated if not provided)
- * @param {boolean} [options.autoPort=true] - Auto-allocate port if specified port is unavailable
- * @param {string} [options.logLevel='info'] - Log level (error, warn, info, debug)
- * @returns {Promise<Object>} Server instance
- */
-export async function startServer({ dataDir, port, autoPort = true, logLevel = 'info' }) {
-  // Allocate port (checks for existing instance, reuses if running)
-  const allocatedPort = await allocatePort(dataDir, port);
-
-  if (port && allocatedPort !== port && !autoPort) {
-    throw new Error(
-      `Port ${port} unavailable and autoPort is disabled. ` +
-        `Use autoPort: true or choose a different port.`
-    );
-  }
-
-  return _startServer({ dataDir, port: allocatedPort, logLevel });
-}
-
-/**
- * Stop a running server instance
- *
- * @param {Object} options
- * @param {string} [options.dataDir] - Data directory of the instance to stop
- * @param {number} [options.port] - Port of the instance to stop
- */
-export async function stopServer({ dataDir, port }) {
-  return _stopServer({ dataDir, port });
-}
-
-/**
- * Get an existing instance or start a new one
- *
- * This is the recommended way to start a server, as it prevents
- * duplicate instances for the same data directory.
- *
- * @param {Object} options
- * @param {string} options.dataDir - Data directory for the database
- * @param {number} [options.port] - Preferred port (auto-allocated if unavailable)
- * @param {boolean} [options.autoPort=true] - Auto-allocate port
- * @param {string} [options.logLevel='info'] - Log level
- * @returns {Promise<Object>} Server instance (existing or new)
- */
-export async function getOrStart({ dataDir, port, autoPort = true, logLevel = 'info' }) {
-  // Check if instance already running
-  const existing = findInstanceByDataDir(dataDir);
-
-  if (existing) {
-    // Verify process is still alive
-    try {
-      process.kill(existing.pid, 0);
-      console.log(`✅ Using existing instance on port ${existing.port}`);
-
-      return {
-        port: existing.port,
-        dataDir,
-        pid: existing.pid,
-        connectionUrl: `postgresql://localhost:${existing.port}`,
-        existing: true
-      };
-    } catch {
-      // Process dead, cleanup and start new
-      console.log('🔄 Stale instance found, starting fresh...');
-    }
-  }
-
-  // Start new instance
-  return startServer({ dataDir, port, autoPort, logLevel });
-}
-
-/**
- * Auto-detect database configuration
- *
- * Tries external PostgreSQL first, falls back to embedded PGlite
- *
- * @param {Object} options
- * @param {string} [options.externalUrl] - External PostgreSQL URL to try first
- * @param {string} options.embeddedDataDir - Data directory for embedded fallback
- * @param {number} [options.embeddedPort] - Preferred port for embedded server
- * @param {number} [options.timeout=5000] - Timeout for external connection test
- * @returns {Promise<Object>} Database configuration
- */
-export async function autoDetect({
-  externalUrl,
-  embeddedDataDir,
-  embeddedPort,
-  timeout = 5000
-}) {
-  return _autoDetect({ externalUrl, embeddedDataDir, embeddedPort, timeout });
-}
-
-/**
- * List all running instances
- *
- * @returns {Array<Object>} Array of instance info
- */
-export function list() {
-  return listInstances();
-}
-
-/**
- * Find instance by data directory
- *
- * @param {string} dataDir - Data directory path
- * @returns {Object|null} Instance info or null
- */
-export function findByDataDir(dataDir) {
-  return findInstanceByDataDir(dataDir);
-}
-
-/**
- * Find instance by port
- *
- * @param {number} port - Port number
- * @returns {Object|null} Instance info or null
- */
-export function findByPort(port) {
-  return findInstanceByPort(port);
-}
-
-/**
- * Get port range information
- *
- * @returns {Object} Port range stats
- */
-export function portInfo() {
-  return getPortRangeInfo();
-}
-
-/**
- * Cleanup stale instances (dead processes)
- *
- * @returns {number} Number of instances cleaned up
- */
-export function cleanup() {
-  return cleanupStaleInstances();
-}
-
-// Export all functions
-export default {
-  startServer,
-  stopServer,
-  getOrStart,
-  autoDetect,
-  list,
-  findByDataDir,
-  findByPort,
-  portInfo,
-  cleanup
-};
+// Default export
+export { startMultiTenantServer as default } from './router.js';