pgserve 0.1.4 → 1.0.1

package/src/postgres.js

@@ -0,0 +1,479 @@
+/**
+ * PostgreSQL Manager (Direct Binary Execution)
+ *
+ * Manages an embedded PostgreSQL instance with true concurrent connections.
+ * Directly executes PostgreSQL binaries from embedded-postgres packages,
+ * bypassing the embedded-postgres library's locale-dependent initialization.
+ *
+ * Features:
+ * - Uses embedded-postgres binaries (auto-downloaded via npm)
+ * - Memory mode (default) or persistent storage
+ * - True concurrent connections (native PostgreSQL process forking)
+ * - Auto-provision databases on demand
+ * - No locale dependency (works on any system)
+ */
+
+import { spawn } from 'child_process';
+import os from 'os';
+import path from 'path';
+import fs from 'fs';
+import crypto from 'crypto';
+import net from 'net';
+
+// Resolve binary paths from embedded-postgres platform packages
+function getBinaryPaths() {
+  const platform = os.platform();
+  const arch = os.arch();
+
+  let pkgName;
+  if (platform === 'linux' && arch === 'x64') {
+    pkgName = '@embedded-postgres/linux-x64';
+  } else if (platform === 'darwin' && arch === 'arm64') {
+    pkgName = '@embedded-postgres/darwin-arm64';
+  } else if (platform === 'darwin' && arch === 'x64') {
+    pkgName = '@embedded-postgres/darwin-x64';
+  } else if (platform === 'win32' && arch === 'x64') {
+    pkgName = '@embedded-postgres/win32-x64';
+  } else {
+    throw new Error(`Unsupported platform: ${platform}-${arch}`);
+  }
+
+  // Find the package in node_modules
+  const possiblePaths = [
+    path.join(process.cwd(), 'node_modules', pkgName, 'native', 'bin'),
+    path.join(import.meta.dirname, '..', 'node_modules', pkgName, 'native', 'bin'),
+  ];
+
+  for (const binDir of possiblePaths) {
+    const initdb = path.join(binDir, platform === 'win32' ? 'initdb.exe' : 'initdb');
+    const postgres = path.join(binDir, platform === 'win32' ? 'postgres.exe' : 'postgres');
+    if (fs.existsSync(initdb) && fs.existsSync(postgres)) {
+      return { initdb, postgres, binDir };
+    }
+  }
+
+  throw new Error(`Could not find PostgreSQL binaries. Please run: npm install ${pkgName}`);
+}
+
+export class PostgresManager {
+  constructor(options = {}) {
+    this.dataDir = options.dataDir || null; // null = memory mode (temp dir)
+    this.port = options.port || 5433; // Internal PG port (router listens on different port)
+    this.user = options.user || 'postgres';
+    this.password = options.password || 'postgres';
+    this.logger = options.logger;
+    this.process = null;
+    this.databaseDir = null;
+    this.persistent = !!options.dataDir;
+    this.createdDatabases = new Set();
+    this.binaries = null;
+    this.creatingDatabases = new Map(); // Track in-progress creations
+    this.socketDir = null; // Unix socket directory for faster local connections
+    this.adminPool = null; // Connection pool for database admin operations
+
+    // Sync/Replication options (for async sync to real PostgreSQL)
+    this.syncEnabled = options.syncEnabled || false;
+    this.syncManager = null; // Will be set via setSyncManager()
+  }
+
+  /**
+   * Set the SyncManager for async replication
+   * Called after PostgresManager is created but before start()
+   * @param {SyncManager} syncManager
+   */
+  setSyncManager(syncManager) {
+    this.syncManager = syncManager;
+    this.syncEnabled = !!syncManager;
+  }
+
+  /**
+   * Start the embedded PostgreSQL instance
+   */
+  async start() {
+    // Get binary paths
+    this.binaries = getBinaryPaths();
+
+    // Make binaries executable
+    await fs.promises.chmod(this.binaries.initdb, '755');
+    await fs.promises.chmod(this.binaries.postgres, '755');
+
+    // Determine data directory
+    if (this.persistent) {
+      this.databaseDir = this.dataDir;
+      // Ensure directory exists
+      if (!fs.existsSync(this.databaseDir)) {
+        fs.mkdirSync(this.databaseDir, { recursive: true });
+      }
+    } else {
+      // Memory mode: use temp directory with unique suffix
+      this.databaseDir = path.join(os.tmpdir(), `pgserve-${process.pid}-${Date.now()}`);
+      // Clean up if exists from a previous failed run
+      if (fs.existsSync(this.databaseDir)) {
+        fs.rmSync(this.databaseDir, { recursive: true, force: true });
+      }
+    }
+
+    // Create Unix socket directory (Linux/macOS only, Windows uses TCP)
+    if (os.platform() !== 'win32') {
+      this.socketDir = path.join(os.tmpdir(), `pgserve-sock-${process.pid}-${Date.now()}`);
+      if (!fs.existsSync(this.socketDir)) {
+        fs.mkdirSync(this.socketDir, { recursive: true, mode: 0o700 });
+      }
+    }
+
+    this.logger.info({
+      databaseDir: this.databaseDir,
+      persistent: this.persistent,
+      port: this.port
+    }, 'Starting embedded PostgreSQL');
+
+    // Check if data directory is already initialized
+    const pgVersionFile = path.join(this.databaseDir, 'PG_VERSION');
+    if (!fs.existsSync(pgVersionFile)) {
+      await this._runInitDb();
+    } else {
+      this.logger.debug({ databaseDir: this.databaseDir }, 'Using existing data directory');
+    }
+
+    // Start PostgreSQL server
+    await this._startPostgres();
+
+    // Initialize admin connection pool (for database creation operations)
+    await this._initAdminPool();
+
+    this.logger.info({
+      databaseDir: this.databaseDir,
+      port: this.port,
+      socketDir: this.socketDir,
+      persistent: this.persistent
+    }, 'PostgreSQL started successfully');
+
+    return this;
+  }
+
+  /**
+   * Run initdb to initialize the data directory
+   */
+  async _runInitDb() {
+    // Create password file
+    const randomId = crypto.randomBytes(6).toString('hex');
+    const passwordFile = path.join(os.tmpdir(), `pg-password-${randomId}`);
+    await fs.promises.writeFile(passwordFile, this.password + '\n');
+
+    this.logger.debug({ databaseDir: this.databaseDir }, 'Initializing PostgreSQL data directory');
+
+    return new Promise((resolve, reject) => {
+      const proc = spawn(this.binaries.initdb, [
+        `--pgdata=${this.databaseDir}`,
+        '--auth=password',
+        `--username=${this.user}`,
+        `--pwfile=${passwordFile}`,
+      ], {
+        env: { ...process.env, LC_ALL: 'C', LANG: 'C' }
+      });
+
+      let stdout = '';
+      let stderr = '';
+
+      proc.stdout.on('data', (data) => {
+        stdout += data.toString();
+      });
+
+      proc.stderr.on('data', (data) => {
+        stderr += data.toString();
+      });
+
+      proc.on('close', async (code) => {
+        // Clean up password file
+        try {
+          await fs.promises.unlink(passwordFile);
+        } catch (e) {
+          // Ignore
+        }
+
+        if (code === 0) {
+          this.logger.debug('initdb completed successfully');
+          resolve();
+        } else {
+          this.logger.error({ code, stdout, stderr }, 'initdb failed');
+          reject(new Error(`initdb failed with code ${code}: ${stderr || stdout}`));
+        }
+      });
+
+      proc.on('error', (err) => {
+        reject(new Error(`Failed to spawn initdb: ${err.message}`));
+      });
+    });
+  }
+
+  /**
+   * Initialize admin connection pool for database operations
+   * Uses Unix socket when available for faster connections
+   */
+  async _initAdminPool() {
+    const { default: pg } = await import('pg');
+
+    // Pool config - use Unix socket when available
+    const poolConfig = {
+      user: this.user,
+      password: this.password,
+      database: 'postgres',
+      max: 5, // Small pool - only for CREATE DATABASE operations
+      idleTimeoutMillis: 30000,
+      connectionTimeoutMillis: 5000,
+    };
+
+    // Use Unix socket for faster local connections (Linux/macOS)
+    // Note: pg library needs both host (socket dir) AND port to find the socket file
+    if (this.socketDir) {
+      poolConfig.host = this.socketDir;
+      poolConfig.port = this.port; // Required for Unix socket path construction
+    } else {
+      poolConfig.host = '127.0.0.1';
+      poolConfig.port = this.port;
+    }
+
+    this.adminPool = new pg.Pool(poolConfig);
+
+    // Verify pool is working
+    const client = await this.adminPool.connect();
+    client.release();
+
+    this.logger.debug({
+      host: poolConfig.host,
+      maxConnections: poolConfig.max
+    }, 'Admin connection pool initialized');
+  }
+
+  /**
+   * Start the PostgreSQL server process
+   */
+  async _startPostgres() {
+    return new Promise((resolve, reject) => {
+      // Build PostgreSQL arguments
+      const pgArgs = [
+        '-D', this.databaseDir,
+        '-p', this.port.toString(),
+      ];
+
+      // Enable Unix socket for faster local connections (Linux/macOS)
+      // Windows falls back to TCP only
+      if (this.socketDir) {
+        pgArgs.push('-k', this.socketDir);
+      } else {
+        pgArgs.push('-k', ''); // Disable Unix socket on Windows
+      }
+
+      // Add logical replication settings when sync is enabled
+      // These settings enable PostgreSQL's native WAL-based replication
+      // with ZERO hot path impact (handled by PostgreSQL's WAL writer process)
+      if (this.syncEnabled) {
+        pgArgs.push(
+          '-c', 'wal_level=logical',           // Enable logical decoding
+          '-c', 'max_replication_slots=10',    // Support multiple subscriptions
+          '-c', 'max_wal_senders=10',          // Parallel replication streams
+          '-c', 'wal_keep_size=512MB',         // Retain WAL for catchup
+        );
+        this.logger.info('Logical replication enabled for sync');
+      }
+
+      this.process = spawn(this.binaries.postgres, pgArgs, {
+        env: { ...process.env, LC_ALL: 'C', LANG: 'C' }
+      });
+
+      let started = false;
+      let startupOutput = '';
+
+      const onData = (data) => {
+        const message = data.toString();
+        startupOutput += message;
+        this.logger.debug({ pgOutput: message.trim() }, 'PostgreSQL output');
+
+        // Check for ready message
+        if (message.includes('database system is ready to accept connections') ||
+            message.includes('ready to accept connections')) {
+          started = true;
+          resolve();
+        }
+      };
+
+      this.process.stderr.on('data', onData);
+      this.process.stdout.on('data', onData);
+
+      this.process.on('close', (code) => {
+        if (!started) {
+          reject(new Error(`PostgreSQL exited with code ${code} before starting: ${startupOutput}`));
+        }
+        this.process = null;
+      });
+
+      this.process.on('error', (err) => {
+        reject(new Error(`Failed to spawn postgres: ${err.message}`));
+      });
+
+      // Timeout after 30 seconds
+      setTimeout(() => {
+        if (!started) {
+          reject(new Error(`PostgreSQL startup timed out after 30s. Output: ${startupOutput}`));
+        }
+      }, 30000);
+    });
+  }
+
+  /**
+   * Create a database if it doesn't exist
+   * Uses a promise-based lock to prevent race conditions
+   * @param {string} dbName - Database name to create
+   */
+  async createDatabase(dbName) {
+    // Skip if already created this session
+    if (this.createdDatabases.has(dbName)) {
+      return;
+    }
+
+    // Skip 'postgres' database - it always exists
+    if (dbName === 'postgres') {
+      this.createdDatabases.add(dbName);
+      return;
+    }
+
+    // Check if creation is already in progress for this database
+    // If so, wait for it to complete
+    if (this.creatingDatabases.has(dbName)) {
+      await this.creatingDatabases.get(dbName);
+      return;
+    }
+
+    // Create a promise that other concurrent requests will wait on
+    let resolveCreation;
+    const creationPromise = new Promise((resolve) => {
+      resolveCreation = resolve;
+    });
+    this.creatingDatabases.set(dbName, creationPromise);
+
+    // Use pooled connection for faster database creation
+    let createError = null;
+    const client = await this.adminPool.connect();
+    try {
+      await client.query(`CREATE DATABASE ${client.escapeIdentifier(dbName)}`);
+      this.createdDatabases.add(dbName);
+      this.logger.info({ dbName }, 'Database created');
+
+      // Trigger async sync setup (non-blocking, doesn't affect hot path)
+      if (this.syncManager) {
+        this.syncManager.setupDatabaseSync(dbName)
+          .catch(err => this.logger.warn({ dbName, err: err.message }, 'Sync setup failed (non-fatal)'));
+      }
+    } catch (error) {
+      // Database might already exist (from previous persistent session or race condition)
+      // 42P04 = duplicate_database, 23505 = unique_violation
+      if (error.code === '42P04' || error.code === '23505') {
+        this.createdDatabases.add(dbName);
+        this.logger.debug({ dbName }, 'Database already exists');
+      } else {
+        createError = error;
+      }
+    } finally {
+      client.release();
+      // Signal completion to waiting requests
+      this.creatingDatabases.delete(dbName);
+      resolveCreation();
+    }
+
+    if (createError) {
+      throw new Error(`Failed to create database '${dbName}': ${createError.message}`);
+    }
+  }
+
+  /**
+   * Check if a database exists
+   * @param {string} dbName - Database name to check
+   */
+  async databaseExists(dbName) {
+    return this.createdDatabases.has(dbName);
+  }
+
+  /**
+   * Stop the PostgreSQL instance
+   */
+  async stop() {
+    // Close admin pool first
+    if (this.adminPool) {
+      await this.adminPool.end();
+      this.adminPool = null;
+    }
+
+    if (this.process) {
+      this.logger.info('Stopping PostgreSQL');
+
+      return new Promise((resolve) => {
+        this.process.on('close', () => {
+          this.process = null;
+
+          // Clean up temp directory in memory mode
+          if (!this.persistent && this.databaseDir) {
+            try {
+              fs.rmSync(this.databaseDir, { recursive: true, force: true });
+              this.logger.debug({ databaseDir: this.databaseDir }, 'Cleaned up temp directory');
+            } catch (error) {
+              this.logger.warn({ error: error.message }, 'Failed to clean up temp directory');
+            }
+          }
+
+          // Clean up socket directory
+          if (this.socketDir) {
+            try {
+              fs.rmSync(this.socketDir, { recursive: true, force: true });
+              this.logger.debug({ socketDir: this.socketDir }, 'Cleaned up socket directory');
+            } catch (error) {
+              this.logger.warn({ error: error.message }, 'Failed to clean up socket directory');
+            }
+          }
+
+          resolve();
+        });
+
+        // Send SIGINT for graceful shutdown
+        this.process.kill('SIGINT');
+
+        // Force kill after 5 seconds
+        setTimeout(() => {
+          if (this.process) {
+            this.process.kill('SIGKILL');
+          }
+        }, 5000);
+      });
+    }
+  }
+
+  /**
+   * Get the Unix socket path for PostgreSQL connections
+   * Returns null on Windows (use TCP instead)
+   */
+  getSocketPath() {
+    if (!this.socketDir) return null;
+    return path.join(this.socketDir, `.s.PGSQL.${this.port}`);
+  }
+
+  /**
+   * Get connection URL for a specific database
+   * @param {string} dbName - Database name
+   */
+  getConnectionUrl(dbName = 'postgres') {
+    return `postgresql://${this.user}:${this.password}@127.0.0.1:${this.port}/${dbName}`;
+  }
+
+  /**
+   * Get manager stats
+   */
+  getStats() {
+    return {
+      port: this.port,
+      databaseDir: this.databaseDir,
+      socketDir: this.socketDir,
+      socketPath: this.getSocketPath(),
+      persistent: this.persistent,
+      databases: Array.from(this.createdDatabases)
+    };
+  }
+}

package/src/protocol.js

@@ -11,7 +11,9 @@
  */
 
 const PROTOCOL_VERSION_3 = 196608;
-const SSL_REQUEST_CODE = 80877103; // PostgreSQL SSL negotiation request
+const SSL_REQUEST_CODE = 80877103;    // PostgreSQL SSL negotiation request
+const GSSAPI_REQUEST_CODE = 80877104; // PostgreSQL GSSAPI encryption request
+const CANCEL_REQUEST_CODE = 80877102; // PostgreSQL cancel request
 const DATABASE_KEY = Buffer.from('database\0');
 
 /**
@@ -132,36 +134,59 @@ export function extractDatabaseName(data) {
   }
 }
 
+// Pre-allocated buffer pool for startup message parsing (avoids allocation per connection)
+const STARTUP_BUFFER_SIZE = 8192; // Max startup message is typically < 1KB
+const bufferPool = [];
+const MAX_POOL_SIZE = 100;
+
+function acquireBuffer() {
+  return bufferPool.pop() || Buffer.allocUnsafe(STARTUP_BUFFER_SIZE);
+}
+
+function releaseBuffer(buf) {
+  if (bufferPool.length < MAX_POOL_SIZE) {
+    bufferPool.push(buf);
+  }
+}
+
 /**
  * Read startup message from socket and buffer it
+ * OPTIMIZED: Uses pre-allocated buffer pool to avoid allocation per connection
  *
  * @param {net.Socket} socket - TCP socket
  * @returns {Promise<{message: Buffer, allData: Buffer}>} Startup message and all buffered data
  */
 export async function readStartupMessage(socket) {
   return new Promise((resolve, reject) => {
-    let buffer = Buffer.alloc(0);
+    const buffer = acquireBuffer();
+    let offset = 0;
     let expectedLength = null;
     let resolved = false;
 
     const onData = (chunk) => {
       if (resolved) return;
 
-      buffer = Buffer.concat([buffer, chunk]);
+      // Copy chunk into pre-allocated buffer (avoids Buffer.concat allocation)
+      const copyLen = Math.min(chunk.length, STARTUP_BUFFER_SIZE - offset);
+      chunk.copy(buffer, offset, 0, copyLen);
+      offset += copyLen;
 
       // Read expected length from first 4 bytes
-      if (expectedLength === null && buffer.length >= 4) {
+      if (expectedLength === null && offset >= 4) {
         expectedLength = buffer.readInt32BE(0);
       }
 
       // Check if we have full message
-      if (expectedLength !== null && buffer.length >= expectedLength) {
+      if (expectedLength !== null && offset >= expectedLength) {
         resolved = true;
         socket.removeListener('data', onData);
         socket.removeListener('error', onError);
 
-        const message = buffer.slice(0, expectedLength);
-        resolve({ message, allData: buffer });
+        // Create result buffers (need to copy since we're reusing pool buffer)
+        const message = Buffer.from(buffer.subarray(0, expectedLength));
+        const allData = Buffer.from(buffer.subarray(0, offset));
+        releaseBuffer(buffer);
+        resolve({ message, allData });
       }
     };
 
@@ -170,6 +195,7 @@ export async function readStartupMessage(socket) {
       resolved = true;
       socket.removeListener('data', onData);
       socket.removeListener('error', onError);
+      releaseBuffer(buffer);
       reject(error);
     };
 
@@ -179,14 +205,15 @@ export async function readStartupMessage(socket) {
     // Resume socket AFTER listeners are set up (prevents race condition)
     socket.resume();
 
-    // Timeout after 5 seconds
+    // Timeout after 2 seconds (reduced from 5s for faster probe connection handling)
     setTimeout(() => {
       if (resolved) return;
       resolved = true;
       socket.removeListener('data', onData);
       socket.removeListener('error', onError);
+      releaseBuffer(buffer);
       reject(new Error('Timeout reading startup message'));
-    }, 5000);
+    }, 2000);
   });
 }
 
@@ -199,7 +226,7 @@ export async function readStartupMessage(socket) {
 export async function extractDatabaseNameFromSocket(socket) {
   let { message, allData } = await readStartupMessage(socket);
 
-  // Check if this is an SSL request
+  // Check if this is a protocol negotiation request (SSL, GSSAPI, Cancel)
   if (message.length >= 8) {
     const version = message.readInt32BE(4);
 
@@ -211,6 +238,18 @@ export async function extractDatabaseNameFromSocket(socket) {
       const result = await readStartupMessage(socket);
       message = result.message;
       allData = result.allData;
+    } else if (version === GSSAPI_REQUEST_CODE) {
+      // Respond with 'N' (no GSSAPI support)
+      socket.write(Buffer.from('N'));
+
+      // Read the actual startup message
+      const result = await readStartupMessage(socket);
+      message = result.message;
+      allData = result.allData;
+    } else if (version === CANCEL_REQUEST_CODE) {
+      // Cancel request - PGlite doesn't support query cancellation
+      // Just close gracefully (cancel requests don't expect a response)
+      throw new Error('Cancel request received (not supported)');
     }
   }