pgserve 0.1.5 → 1.0.2

package/src/index.js

@@ -1,177 +1,16 @@
 /**
- * pgserve - PostgreSQL embedded server using PGlite
+ * pgserve - Embedded PostgreSQL Server
  *
- * Multi-tenant PostgreSQL router using PGlite
- * Single port, auto-provisioning, perfect for multi-user apps and AI agents
+ * True concurrent connections, zero config, auto-provision databases.
+ * Uses embedded-postgres (real PostgreSQL binaries).
  */
 
-// Multi-tenant mode (NEW - recommended)
+// Main exports
 export { MultiTenantRouter, startMultiTenantServer } from './router.js';
-export { InstancePool } from './pool.js';
+export { PostgresManager } from './postgres.js';
+export { SyncManager } from './sync.js';
+export { RestoreManager } from './restore.js';
+export { Dashboard } from './dashboard.js';
 
-// Legacy single-instance mode (backwards compatible)
-import { startServer as _startServer, stopServer as _stopServer } from './server.js';
-import { allocatePort, getPortRangeInfo } from './ports.js';
-import {
-  findInstanceByDataDir,
-  findInstanceByPort,
-  listInstances,
-  cleanupStaleInstances
-} from './registry.js';
-import { autoDetect as _autoDetect } from './detector.js';
-
-/**
- * Start a new PGlite server instance
- *
- * @param {Object} options
- * @param {string} options.dataDir - Data directory for the database
- * @param {number} [options.port] - Specific port (optional, auto-allocated if not provided)
- * @param {boolean} [options.autoPort=true] - Auto-allocate port if specified port is unavailable
- * @param {string} [options.logLevel='info'] - Log level (error, warn, info, debug)
- * @returns {Promise<Object>} Server instance
- */
-export async function startServer({ dataDir, port, autoPort = true, logLevel = 'info' }) {
-  // Allocate port (checks for existing instance, reuses if running)
-  const allocatedPort = await allocatePort(dataDir, port);
-
-  if (port && allocatedPort !== port && !autoPort) {
-    throw new Error(
-      `Port ${port} unavailable and autoPort is disabled. ` +
-        `Use autoPort: true or choose a different port.`
-    );
-  }
-
-  return _startServer({ dataDir, port: allocatedPort, logLevel });
-}
-
-/**
- * Stop a running server instance
- *
- * @param {Object} options
- * @param {string} [options.dataDir] - Data directory of the instance to stop
- * @param {number} [options.port] - Port of the instance to stop
- */
-export async function stopServer({ dataDir, port }) {
-  return _stopServer({ dataDir, port });
-}
-
-/**
- * Get an existing instance or start a new one
- *
- * This is the recommended way to start a server, as it prevents
- * duplicate instances for the same data directory.
- *
- * @param {Object} options
- * @param {string} options.dataDir - Data directory for the database
- * @param {number} [options.port] - Preferred port (auto-allocated if unavailable)
- * @param {boolean} [options.autoPort=true] - Auto-allocate port
- * @param {string} [options.logLevel='info'] - Log level
- * @returns {Promise<Object>} Server instance (existing or new)
- */
-export async function getOrStart({ dataDir, port, autoPort = true, logLevel = 'info' }) {
-  // Check if instance already running
-  const existing = findInstanceByDataDir(dataDir);
-
-  if (existing) {
-    // Verify process is still alive
-    try {
-      process.kill(existing.pid, 0);
-      console.log(`✅ Using existing instance on port ${existing.port}`);
-
-      return {
-        port: existing.port,
-        dataDir,
-        pid: existing.pid,
-        connectionUrl: `postgresql://localhost:${existing.port}`,
-        existing: true
-      };
-    } catch {
-      // Process dead, cleanup and start new
-      console.log('🔄 Stale instance found, starting fresh...');
-    }
-  }
-
-  // Start new instance
-  return startServer({ dataDir, port, autoPort, logLevel });
-}
-
-/**
- * Auto-detect database configuration
- *
- * Tries external PostgreSQL first, falls back to embedded PGlite
- *
- * @param {Object} options
- * @param {string} [options.externalUrl] - External PostgreSQL URL to try first
- * @param {string} options.embeddedDataDir - Data directory for embedded fallback
- * @param {number} [options.embeddedPort] - Preferred port for embedded server
- * @param {number} [options.timeout=5000] - Timeout for external connection test
- * @returns {Promise<Object>} Database configuration
- */
-export async function autoDetect({
-  externalUrl,
-  embeddedDataDir,
-  embeddedPort,
-  timeout = 5000
-}) {
-  return _autoDetect({ externalUrl, embeddedDataDir, embeddedPort, timeout });
-}
-
-/**
- * List all running instances
- *
- * @returns {Array<Object>} Array of instance info
- */
-export function list() {
-  return listInstances();
-}
-
-/**
- * Find instance by data directory
- *
- * @param {string} dataDir - Data directory path
- * @returns {Object|null} Instance info or null
- */
-export function findByDataDir(dataDir) {
-  return findInstanceByDataDir(dataDir);
-}
-
-/**
- * Find instance by port
- *
- * @param {number} port - Port number
- * @returns {Object|null} Instance info or null
- */
-export function findByPort(port) {
-  return findInstanceByPort(port);
-}
-
-/**
- * Get port range information
- *
- * @returns {Object} Port range stats
- */
-export function portInfo() {
-  return getPortRangeInfo();
-}
-
-/**
- * Cleanup stale instances (dead processes)
- *
- * @returns {number} Number of instances cleaned up
- */
-export function cleanup() {
-  return cleanupStaleInstances();
-}
-
-// Export all functions
-export default {
-  startServer,
-  stopServer,
-  getOrStart,
-  autoDetect,
-  list,
-  findByDataDir,
-  findByPort,
-  portInfo,
-  cleanup
-};
+// Default export
+export { startMultiTenantServer as default } from './router.js';

package/src/postgres.js

@@ -0,0 +1,478 @@
+/**
+ * PostgreSQL Manager (Direct Binary Execution)
+ *
+ * Manages an embedded PostgreSQL instance with true concurrent connections.
+ * Directly executes PostgreSQL binaries from embedded-postgres packages,
+ * bypassing the embedded-postgres library's locale-dependent initialization.
+ *
+ * Features:
+ * - Uses embedded-postgres binaries (auto-downloaded via npm)
+ * - Memory mode (default) or persistent storage
+ * - True concurrent connections (native PostgreSQL process forking)
+ * - Auto-provision databases on demand
+ * - No locale dependency (works on any system)
+ */
+
+import { spawn } from 'child_process';
+import os from 'os';
+import path from 'path';
+import fs from 'fs';
+import crypto from 'crypto';
+
+// Resolve binary paths from embedded-postgres platform packages
+function getBinaryPaths() {
+  const platform = os.platform();
+  const arch = os.arch();
+
+  let pkgName;
+  if (platform === 'linux' && arch === 'x64') {
+    pkgName = '@embedded-postgres/linux-x64';
+  } else if (platform === 'darwin' && arch === 'arm64') {
+    pkgName = '@embedded-postgres/darwin-arm64';
+  } else if (platform === 'darwin' && arch === 'x64') {
+    pkgName = '@embedded-postgres/darwin-x64';
+  } else if (platform === 'win32' && arch === 'x64') {
+    pkgName = '@embedded-postgres/win32-x64';
+  } else {
+    throw new Error(`Unsupported platform: ${platform}-${arch}`);
+  }
+
+  // Find the package in node_modules
+  const possiblePaths = [
+    path.join(process.cwd(), 'node_modules', pkgName, 'native', 'bin'),
+    path.join(import.meta.dirname, '..', 'node_modules', pkgName, 'native', 'bin'),
+  ];
+
+  for (const binDir of possiblePaths) {
+    const initdb = path.join(binDir, platform === 'win32' ? 'initdb.exe' : 'initdb');
+    const postgres = path.join(binDir, platform === 'win32' ? 'postgres.exe' : 'postgres');
+    if (fs.existsSync(initdb) && fs.existsSync(postgres)) {
+      return { initdb, postgres, binDir };
+    }
+  }
+
+  throw new Error(`Could not find PostgreSQL binaries. Please run: npm install ${pkgName}`);
+}
+
+export class PostgresManager {
+  constructor(options = {}) {
+    this.dataDir = options.dataDir || null; // null = memory mode (temp dir)
+    this.port = options.port || 5433; // Internal PG port (router listens on different port)
+    this.user = options.user || 'postgres';
+    this.password = options.password || 'postgres';
+    this.logger = options.logger;
+    this.process = null;
+    this.databaseDir = null;
+    this.persistent = !!options.dataDir;
+    this.createdDatabases = new Set();
+    this.binaries = null;
+    this.creatingDatabases = new Map(); // Track in-progress creations
+    this.socketDir = null; // Unix socket directory for faster local connections
+    this.adminPool = null; // Connection pool for database admin operations
+
+    // Sync/Replication options (for async sync to real PostgreSQL)
+    this.syncEnabled = options.syncEnabled || false;
+    this.syncManager = null; // Will be set via setSyncManager()
+  }
+
+  /**
+   * Set the SyncManager for async replication
+   * Called after PostgresManager is created but before start()
+   * @param {SyncManager} syncManager
+   */
+  setSyncManager(syncManager) {
+    this.syncManager = syncManager;
+    this.syncEnabled = !!syncManager;
+  }
+
+  /**
+   * Start the embedded PostgreSQL instance
+   */
+  async start() {
+    // Get binary paths
+    this.binaries = getBinaryPaths();
+
+    // Make binaries executable
+    await fs.promises.chmod(this.binaries.initdb, '755');
+    await fs.promises.chmod(this.binaries.postgres, '755');
+
+    // Determine data directory
+    if (this.persistent) {
+      this.databaseDir = this.dataDir;
+      // Ensure directory exists
+      if (!fs.existsSync(this.databaseDir)) {
+        fs.mkdirSync(this.databaseDir, { recursive: true });
+      }
+    } else {
+      // Memory mode: use temp directory with unique suffix
+      this.databaseDir = path.join(os.tmpdir(), `pgserve-${process.pid}-${Date.now()}`);
+      // Clean up if exists from a previous failed run
+      if (fs.existsSync(this.databaseDir)) {
+        fs.rmSync(this.databaseDir, { recursive: true, force: true });
+      }
+    }
+
+    // Create Unix socket directory (Linux/macOS only, Windows uses TCP)
+    if (os.platform() !== 'win32') {
+      this.socketDir = path.join(os.tmpdir(), `pgserve-sock-${process.pid}-${Date.now()}`);
+      if (!fs.existsSync(this.socketDir)) {
+        fs.mkdirSync(this.socketDir, { recursive: true, mode: 0o700 });
+      }
+    }
+
+    this.logger.info({
+      databaseDir: this.databaseDir,
+      persistent: this.persistent,
+      port: this.port
+    }, 'Starting embedded PostgreSQL');
+
+    // Check if data directory is already initialized
+    const pgVersionFile = path.join(this.databaseDir, 'PG_VERSION');
+    if (!fs.existsSync(pgVersionFile)) {
+      await this._runInitDb();
+    } else {
+      this.logger.debug({ databaseDir: this.databaseDir }, 'Using existing data directory');
+    }
+
+    // Start PostgreSQL server
+    await this._startPostgres();
+
+    // Initialize admin connection pool (for database creation operations)
+    await this._initAdminPool();
+
+    this.logger.info({
+      databaseDir: this.databaseDir,
+      port: this.port,
+      socketDir: this.socketDir,
+      persistent: this.persistent
+    }, 'PostgreSQL started successfully');
+
+    return this;
+  }
+
+  /**
+   * Run initdb to initialize the data directory
+   */
+  async _runInitDb() {
+    // Create password file
+    const randomId = crypto.randomBytes(6).toString('hex');
+    const passwordFile = path.join(os.tmpdir(), `pg-password-${randomId}`);
+    await fs.promises.writeFile(passwordFile, this.password + '\n');
+
+    this.logger.debug({ databaseDir: this.databaseDir }, 'Initializing PostgreSQL data directory');
+
+    return new Promise((resolve, reject) => {
+      const proc = spawn(this.binaries.initdb, [
+        `--pgdata=${this.databaseDir}`,
+        '--auth=password',
+        `--username=${this.user}`,
+        `--pwfile=${passwordFile}`,
+      ], {
+        env: { ...process.env, LC_ALL: 'C', LANG: 'C' }
+      });
+
+      let stdout = '';
+      let stderr = '';
+
+      proc.stdout.on('data', (data) => {
+        stdout += data.toString();
+      });
+
+      proc.stderr.on('data', (data) => {
+        stderr += data.toString();
+      });
+
+      proc.on('close', async (code) => {
+        // Clean up password file
+        try {
+          await fs.promises.unlink(passwordFile);
+        } catch {
+          // Ignore cleanup errors
+        }
+
+        if (code === 0) {
+          this.logger.debug('initdb completed successfully');
+          resolve();
+        } else {
+          this.logger.error({ code, stdout, stderr }, 'initdb failed');
+          reject(new Error(`initdb failed with code ${code}: ${stderr || stdout}`));
+        }
+      });
+
+      proc.on('error', (err) => {
+        reject(new Error(`Failed to spawn initdb: ${err.message}`));
+      });
+    });
+  }
+
+  /**
+   * Initialize admin connection pool for database operations
+   * Uses Unix socket when available for faster connections
+   */
+  async _initAdminPool() {
+    const { default: pg } = await import('pg');
+
+    // Pool config - use Unix socket when available
+    const poolConfig = {
+      user: this.user,
+      password: this.password,
+      database: 'postgres',
+      max: 5, // Small pool - only for CREATE DATABASE operations
+      idleTimeoutMillis: 30000,
+      connectionTimeoutMillis: 5000,
+    };
+
+    // Use Unix socket for faster local connections (Linux/macOS)
+    // Note: pg library needs both host (socket dir) AND port to find the socket file
+    if (this.socketDir) {
+      poolConfig.host = this.socketDir;
+      poolConfig.port = this.port; // Required for Unix socket path construction
+    } else {
+      poolConfig.host = '127.0.0.1';
+      poolConfig.port = this.port;
+    }
+
+    this.adminPool = new pg.Pool(poolConfig);
+
+    // Verify pool is working
+    const client = await this.adminPool.connect();
+    client.release();
+
+    this.logger.debug({
+      host: poolConfig.host,
+      maxConnections: poolConfig.max
+    }, 'Admin connection pool initialized');
+  }
+
+  /**
+   * Start the PostgreSQL server process
+   */
+  async _startPostgres() {
+    return new Promise((resolve, reject) => {
+      // Build PostgreSQL arguments
+      const pgArgs = [
+        '-D', this.databaseDir,
+        '-p', this.port.toString(),
+      ];
+
+      // Enable Unix socket for faster local connections (Linux/macOS)
+      // Windows falls back to TCP only
+      if (this.socketDir) {
+        pgArgs.push('-k', this.socketDir);
+      } else {
+        pgArgs.push('-k', ''); // Disable Unix socket on Windows
+      }
+
+      // Add logical replication settings when sync is enabled
+      // These settings enable PostgreSQL's native WAL-based replication
+      // with ZERO hot path impact (handled by PostgreSQL's WAL writer process)
+      if (this.syncEnabled) {
+        pgArgs.push(
+          '-c', 'wal_level=logical',           // Enable logical decoding
+          '-c', 'max_replication_slots=10',    // Support multiple subscriptions
+          '-c', 'max_wal_senders=10',          // Parallel replication streams
+          '-c', 'wal_keep_size=512MB',         // Retain WAL for catchup
+        );
+        this.logger.info('Logical replication enabled for sync');
+      }
+
+      this.process = spawn(this.binaries.postgres, pgArgs, {
+        env: { ...process.env, LC_ALL: 'C', LANG: 'C' }
+      });
+
+      let started = false;
+      let startupOutput = '';
+
+      const onData = (data) => {
+        const message = data.toString();
+        startupOutput += message;
+        this.logger.debug({ pgOutput: message.trim() }, 'PostgreSQL output');
+
+        // Check for ready message
+        if (message.includes('database system is ready to accept connections') ||
+            message.includes('ready to accept connections')) {
+          started = true;
+          resolve();
+        }
+      };
+
+      this.process.stderr.on('data', onData);
+      this.process.stdout.on('data', onData);
+
+      this.process.on('close', (code) => {
+        if (!started) {
+          reject(new Error(`PostgreSQL exited with code ${code} before starting: ${startupOutput}`));
+        }
+        this.process = null;
+      });
+
+      this.process.on('error', (err) => {
+        reject(new Error(`Failed to spawn postgres: ${err.message}`));
+      });
+
+      // Timeout after 30 seconds
+      setTimeout(() => {
+        if (!started) {
+          reject(new Error(`PostgreSQL startup timed out after 30s. Output: ${startupOutput}`));
+        }
+      }, 30000);
+    });
+  }
+
+  /**
+   * Create a database if it doesn't exist
+   * Uses a promise-based lock to prevent race conditions
+   * @param {string} dbName - Database name to create
+   */
+  async createDatabase(dbName) {
+    // Skip if already created this session
+    if (this.createdDatabases.has(dbName)) {
+      return;
+    }
+
+    // Skip 'postgres' database - it always exists
+    if (dbName === 'postgres') {
+      this.createdDatabases.add(dbName);
+      return;
+    }
+
+    // Check if creation is already in progress for this database
+    // If so, wait for it to complete
+    if (this.creatingDatabases.has(dbName)) {
+      await this.creatingDatabases.get(dbName);
+      return;
+    }
+
+    // Create a promise that other concurrent requests will wait on
+    let resolveCreation;
+    const creationPromise = new Promise((resolve) => {
+      resolveCreation = resolve;
+    });
+    this.creatingDatabases.set(dbName, creationPromise);
+
+    // Use pooled connection for faster database creation
+    let createError = null;
+    const client = await this.adminPool.connect();
+    try {
+      await client.query(`CREATE DATABASE ${client.escapeIdentifier(dbName)}`);
+      this.createdDatabases.add(dbName);
+      this.logger.info({ dbName }, 'Database created');
+
+      // Trigger async sync setup (non-blocking, doesn't affect hot path)
+      if (this.syncManager) {
+        this.syncManager.setupDatabaseSync(dbName)
+          .catch(err => this.logger.warn({ dbName, err: err.message }, 'Sync setup failed (non-fatal)'));
+      }
+    } catch (error) {
+      // Database might already exist (from previous persistent session or race condition)
+      // 42P04 = duplicate_database, 23505 = unique_violation
+      if (error.code === '42P04' || error.code === '23505') {
+        this.createdDatabases.add(dbName);
+        this.logger.debug({ dbName }, 'Database already exists');
+      } else {
+        createError = error;
+      }
+    } finally {
+      client.release();
+      // Signal completion to waiting requests
+      this.creatingDatabases.delete(dbName);
+      resolveCreation();
+    }
+
+    if (createError) {
+      throw new Error(`Failed to create database '${dbName}': ${createError.message}`);
+    }
+  }
+
+  /**
+   * Check if a database exists
+   * @param {string} dbName - Database name to check
+   */
+  async databaseExists(dbName) {
+    return this.createdDatabases.has(dbName);
+  }
+
+  /**
+   * Stop the PostgreSQL instance
+   */
+  async stop() {
+    // Close admin pool first
+    if (this.adminPool) {
+      await this.adminPool.end();
+      this.adminPool = null;
+    }
+
+    if (this.process) {
+      this.logger.info('Stopping PostgreSQL');
+
+      return new Promise((resolve) => {
+        this.process.on('close', () => {
+          this.process = null;
+
+          // Clean up temp directory in memory mode
+          if (!this.persistent && this.databaseDir) {
+            try {
+              fs.rmSync(this.databaseDir, { recursive: true, force: true });
+              this.logger.debug({ databaseDir: this.databaseDir }, 'Cleaned up temp directory');
+            } catch (error) {
+              this.logger.warn({ error: error.message }, 'Failed to clean up temp directory');
+            }
+          }
+
+          // Clean up socket directory
+          if (this.socketDir) {
+            try {
+              fs.rmSync(this.socketDir, { recursive: true, force: true });
+              this.logger.debug({ socketDir: this.socketDir }, 'Cleaned up socket directory');
+            } catch (error) {
+              this.logger.warn({ error: error.message }, 'Failed to clean up socket directory');
+            }
+          }
+
+          resolve();
+        });
+
+        // Send SIGINT for graceful shutdown
+        this.process.kill('SIGINT');
+
+        // Force kill after 5 seconds
+        setTimeout(() => {
+          if (this.process) {
+            this.process.kill('SIGKILL');
+          }
+        }, 5000);
+      });
+    }
+  }
+
+  /**
+   * Get the Unix socket path for PostgreSQL connections
+   * Returns null on Windows (use TCP instead)
+   */
+  getSocketPath() {
+    if (!this.socketDir) return null;
+    return path.join(this.socketDir, `.s.PGSQL.${this.port}`);
+  }
+
+  /**
+   * Get connection URL for a specific database
+   * @param {string} dbName - Database name
+   */
+  getConnectionUrl(dbName = 'postgres') {
+    return `postgresql://${this.user}:${this.password}@127.0.0.1:${this.port}/${dbName}`;
+  }
+
+  /**
+   * Get manager stats
+   */
+  getStats() {
+    return {
+      port: this.port,
+      databaseDir: this.databaseDir,
+      socketDir: this.socketDir,
+      socketPath: this.getSocketPath(),
+      persistent: this.persistent,
+      databases: Array.from(this.createdDatabases)
+    };
+  }
+}