pgserve 2.3.0 → 2.4.0

package/src/router.js

@@ -1,546 +0,0 @@
-/**
- * Multi-Tenant Router (TCP Proxy to Embedded PostgreSQL)
- *
- * Single TCP server that routes connections to an embedded PostgreSQL instance.
- * Extracts database name from PostgreSQL startup message, auto-creates database,
- * then proxies the connection to real PostgreSQL.
- *
- * Features:
- * - TRUE concurrent connections (native PostgreSQL)
- * - Auto-provision databases on first connection
- * - Zero configuration required
- * - Memory mode (default) or persistent storage
- *
- * PERFORMANCE: Uses Bun.listen() and Bun.connect() for 2-3x throughput improvement
- *
- * v2 NOTE: The MultiTenantRouter is the **direct-embed** path — callers that
- * spawn their own PostgresManager and bind a TCP port get a per-pid Unix
- * socket from `pgManager.getSocketPath()` (preserved by PR #24). The new
- * **daemon** path (`src/daemon.js`) binds a singleton control socket at
- * `$XDG_RUNTIME_DIR/pgserve/control.sock` and is the v2 default for the
- * `pgserve daemon` CLI subcommand. Both paths coexist; direct-embed callers
- * are not affected by daemon mode.
- */
-
-import fs from 'fs';
-import { PostgresManager } from './postgres.js';
-import { SyncManager } from './sync.js';
-import { RestoreManager } from './restore.js';
-import { Dashboard } from './dashboard.js';
-import { extractDatabaseName } from './protocol.js';
-import { EventEmitter } from 'events';
-import { createLogger } from './logger.js';
-
-// PostgreSQL protocol constants
-const PROTOCOL_VERSION_3 = 196608;
-const SSL_REQUEST_CODE = 80877103;
-const GSSAPI_REQUEST_CODE = 80877104;
-const CANCEL_REQUEST_CODE = 80877102;
-
-// Maximum size for the pre-handshake startup buffer. A legitimate PG
-// startup message is at most a few hundred bytes; anything approaching
-// 1 MiB is a runaway client or an attempted buffer-growth DoS. Bound
-// this to stop the proxy from accumulating gigabytes of orphaned data
-// when a client sends garbage and the handshake never completes.
-// (Issue #18 root cause #2 — unbounded growth at state.buffer.)
-const MAX_STARTUP_BUFFER_SIZE = 1024 * 1024; // 1 MiB
-
-/**
- * Attempt to write a pending buffer to a target socket.
- * Returns remaining unwritten bytes, or null if fully flushed.
- */
-function flushPending(target, pending) {
-  const written = target.write(pending);
-  if (written === pending.byteLength) return null;
-  if (written === 0) return pending;
-  return pending.subarray(written);
-}
-
-/**
- * Multi-Tenant Router Server
- */
-export class MultiTenantRouter extends EventEmitter {
-  constructor(options = {}) {
-    super();
-    this.port = options.port || 8432;
-    this.host = options.host || '127.0.0.1';
-    this.baseDir = options.baseDir || null; // null = memory mode
-    this.memoryMode = !options.baseDir;
-    this.maxConnections = options.maxConnections || 1000;
-    this.autoProvision = options.autoProvision !== false;
-
-    // Internal PostgreSQL port (different from router port)
-    this.pgPort = options.pgPort || (this.port + 1000);
-
-    // Pino logger (ultra-fast structured logging)
-    const logLevel = options.logLevel || 'info';
-    this.logger = options.logger || createLogger({ level: logLevel });
-
-    // Sync options (async replication to real PostgreSQL)
-    this.syncTo = options.syncTo || null;
-    this.syncDatabases = options.syncDatabases
-      ? options.syncDatabases.split(',').map(s => s.trim())
-      : [];
-    this.syncManager = null;
-
-    // PostgreSQL manager (with sync flag if needed)
-    this.pgManager = new PostgresManager({
-      dataDir: this.baseDir,
-      port: this.pgPort,
-      logger: this.logger.child({ component: 'postgres' }),
-      syncEnabled: !!this.syncTo,  // Enable logical replication if sync is configured
-      useRam: options.useRam,  // Use /dev/shm for true RAM storage (Linux only)
-      enablePgvector: options.enablePgvector  // Auto-enable pgvector extension on new databases
-    });
-
-    // TCP server
-    this.server = null;
-    this.connections = new Set();
-
-    // Performance: Reduce event listener overhead
-    this.setMaxListeners(this.maxConnections + 10);
-  }
-
-  /**
-   * Socket state storage for Bun TCP handler model
-   * Maps client socket to its state (buffer, pgSocket, dbName, etc.)
-   */
-  socketState = new WeakMap();
-
-  /**
-   * Start multi-tenant router
-   */
-  async start() {
-    // Initialize dashboard for informative CLI output
-    const dashboard = new Dashboard();
-    dashboard.showHeader({
-      port: this.port,
-      host: this.host,
-      memoryMode: this.memoryMode,
-      syncTo: this.syncTo
-    });
-
-    // Start PostgreSQL first
-    dashboard.stage('PostgreSQL binaries resolved');
-    await this.pgManager.start();
-    dashboard.stage('PostgreSQL started');
-
-    // Automatic restore from external PostgreSQL (if sync configured)
-    // This runs BEFORE SyncManager to restore data before enabling outbound sync
-    if (this.syncTo) {
-      const restoreManager = new RestoreManager({
-        sourceUrl: this.syncTo,
-        patterns: this.syncDatabases,
-        targetPort: this.pgPort,
-        targetSocketPath: this.pgManager.getSocketPath(),
-        logger: this.logger.child({ component: 'restore' }),
-        onProgress: (metrics) => dashboard.updateRestore(metrics)
-      });
-
-      // Start restore progress display
-      dashboard.startRestore(restoreManager.totalDatabases || 1);
-
-      const restoreResult = await restoreManager.restore(this.pgManager);
-
-      if (restoreResult.success) {
-        dashboard.completeRestore(restoreResult.metrics);
-        this.logger.info({
-          databases: restoreResult.metrics.databasesRestored,
-          tables: restoreResult.metrics.tablesRestored,
-          rows: restoreResult.metrics.rowsRestored,
-          bytes: restoreResult.metrics.bytesTransferred,
-          durationMs: restoreResult.metrics.endTime - restoreResult.metrics.startTime
-        }, 'Restored from external PostgreSQL');
-      } else if (restoreResult.skipped) {
-        // No progress to complete - was skipped
-      } else {
-        dashboard.cleanup();
-        this.logger.warn({ error: restoreResult.error }, 'Restore failed (continuing without restored data)');
-      }
-    }
-
-    // Initialize SyncManager if configured (async replication)
-    if (this.syncTo) {
-      this.syncManager = new SyncManager({
-        targetUrl: this.syncTo,
-        databases: this.syncDatabases,
-        sourcePort: this.pgPort,
-        sourceSocketPath: this.pgManager.getSocketPath(),
-        logLevel: this.logger.level
-      });
-
-      // Wire SyncManager to PostgresManager for database creation hooks
-      this.pgManager.setSyncManager(this.syncManager);
-
-      // Initialize sync connections (non-blocking, runs in background)
-      this.syncManager.initialize(this.pgManager)
-        .then(() => {
-          dashboard.stage('Sync manager initialized');
-          this.logger.info('Sync manager initialized');
-        })
-        .catch(err => this.logger.warn({ err: err.message }, 'Sync manager initialization failed (non-fatal)'));
-    }
-
-    // Create TCP server using Bun.listen() for 2-3x throughput
-    const router = this;
-    this.server = Bun.listen({
-      hostname: this.host,
-      port: this.port,
-      socket: {
-        // Called when data arrives on client socket
-        data(socket, data) {
-          router.handleSocketData(socket, data);
-        },
-        // Called when client connects
-        open(socket) {
-          router.handleSocketOpen(socket);
-        },
-        // Called when client disconnects
-        close(socket) {
-          router.handleSocketClose(socket);
-        },
-        // Called on socket error
-        error(socket, error) {
-          router.handleSocketError(socket, error);
-        },
-        // Called when client socket is ready to receive more data
-        drain(socket) {
-          const state = router.socketState.get(socket);
-          if (!state) return;
-          // Flush any pending PG→Client data
-          if (state.pendingToClient) {
-            state.pendingToClient = flushPending(socket, state.pendingToClient);
-          }
-          // If fully flushed, resume reading from PostgreSQL
-          if (!state.pendingToClient && state.pgSocket) {
-            state.pgSocket.resume();
-          }
-        }
-      }
-    });
-
-    const socketPath = this.pgManager.getSocketPath();
-
-    dashboard.stage('TCP server listening');
-    dashboard.showReady({ port: this.port, host: this.host });
-
-    this.logger.info({
-      host: this.host,
-      port: this.port,
-      pgPort: this.pgPort,
-      pgSocketPath: socketPath || '(TCP)',
-      baseDir: this.memoryMode ? '(in-memory)' : this.baseDir,
-      memoryMode: this.memoryMode,
-      autoProvision: this.autoProvision,
-      maxConnections: this.maxConnections
-    }, 'Multi-tenant router started');
-
-    this.emit('listening');
-  }
-
-  /**
-   * Handle socket open (Bun TCP handler)
-   */
-  handleSocketOpen(socket) {
-    // Initialize socket state
-    this.socketState.set(socket, {
-      buffer: null,
-      pgSocket: null,
-      dbName: null,
-      handshakeComplete: false,
-      // startupInProgress serializes processStartupMessage() against async
-      // reentrancy — without it, every data event fired while the previous
-      // processStartupMessage() is still awaiting createDatabase() would
-      // launch another async task on the same state, racing to overwrite
-      // state.pgSocket and leaking the losers (issue #18 root cause #1).
-      startupInProgress: false,
-      pendingToPg: null,
-      pendingToClient: null
-    });
-
-    // Track connection
-    this.connections.add(socket);
-  }
-
-  /**
-   * Handle socket data (Bun TCP handler)
-   * Handles both handshake and proxying phases
-   */
-  handleSocketData(socket, data) {
-    const state = this.socketState.get(socket);
-    if (!state) return;
-
-    // If handshake complete, forward to PostgreSQL
-    if (state.handshakeComplete && state.pgSocket) {
-      // If there's already pending data, append and re-pause.
-      // (Re-pause is defensive: client should already be paused from the
-      //  earlier partial-write, but kernel-buffered data can still arrive
-      //  before the pause takes effect — issue #18 root cause #3.)
-      if (state.pendingToPg) {
-        state.pendingToPg = Buffer.concat([state.pendingToPg, data]);
-        socket.pause();
-        return;
-      }
-      const written = state.pgSocket.write(data);
-      if (written < data.byteLength) {
-        // Partial write — buffer remainder and pause client
-        state.pendingToPg = written === 0 ? Buffer.from(data) : Buffer.from(data.subarray(written));
-        socket.pause();
-      }
-      return;
-    }
-
-    // Buffer data for startup message parsing.
-    // Bound the pre-handshake buffer so a client that never completes its
-    // startup (or sends garbage) cannot grow state.buffer without limit —
-    // the 74 GiB VmSize in the production deadlock report traces to this
-    // path (issue #18 root cause #2).
-    const incomingSize = state.buffer ? state.buffer.length + data.byteLength : data.byteLength;
-    if (incomingSize > MAX_STARTUP_BUFFER_SIZE) {
-      this.logger.warn(
-        { incomingSize, limit: MAX_STARTUP_BUFFER_SIZE },
-        'Pre-handshake buffer exceeded limit — closing connection'
-      );
-      socket.end();
-      return;
-    }
-    if (state.buffer) {
-      state.buffer = Buffer.concat([state.buffer, data]);
-    } else {
-      state.buffer = Buffer.from(data);
-    }
-
-    // Try to parse startup message
-    this.processStartupMessage(socket, state);
-  }
-
-  /**
-   * Process PostgreSQL startup message and establish proxy connection.
-   *
-   * Guarded against async reentrancy: multiple data events arriving while
-   * the first processStartupMessage() is still awaiting createDatabase()
-   * or Bun.connect() must not launch concurrent tasks on the same state —
-   * they would race to assign state.pgSocket, leaking the losing sockets
-   * and double-writing the startup message (issue #18 root cause #1).
-   */
-  async processStartupMessage(socket, state) {
-    if (state.startupInProgress) return;
-
-    const buffer = state.buffer;
-    if (!buffer || buffer.length < 8) return; // Need at least length + protocol
-
-    // Read message length (first 4 bytes, big-endian)
-    const messageLength = buffer.readUInt32BE(0);
-    if (buffer.length < messageLength) return; // Wait for complete message
-
-    // Read protocol version or request code (next 4 bytes)
-    const code = buffer.readUInt32BE(4);
-
-    // Handle SSL/GSSAPI/Cancel requests
-    if (code === SSL_REQUEST_CODE || code === GSSAPI_REQUEST_CODE) {
-      // Reject SSL/GSSAPI - send 'N' (not supported)
-      socket.write(Buffer.from('N'));
-      // Remove this request from buffer, wait for real startup
-      state.buffer = buffer.length > messageLength ? buffer.subarray(messageLength) : null;
-      if (state.buffer) await this.processStartupMessage(socket, state);
-      return;
-    }
-
-    if (code === CANCEL_REQUEST_CODE) {
-      // Cancel request - just close connection
-      socket.end();
-      return;
-    }
-
-    // Must be protocol version 3.0
-    if (code !== PROTOCOL_VERSION_3) {
-      this.logger.warn({ code }, 'Unsupported protocol version');
-      socket.end();
-      return;
-    }
-
-    // Extract database name from startup message
-    const startupMessage = buffer.subarray(0, messageLength);
-    const dbName = extractDatabaseName(startupMessage);
-    state.dbName = dbName;
-
-    // Claim the reentrancy guard BEFORE the first await so subsequent data
-    // events (buffered into state.buffer by handleSocketData) cannot launch
-    // a second async task on the same state.
-    state.startupInProgress = true;
-
-    try {
-      // Auto-provision database if needed
-      if (this.autoProvision) {
-        await this.pgManager.createDatabase(dbName);
-      }
-
-      // Connect to real PostgreSQL using Bun.connect()
-      const socketPath = this.pgManager.getSocketPath();
-      const router = this;
-
-      // Shared handler for pgSocket (used by both unix and TCP paths)
-      const pgHandler = {
-        data(_pgSocket, pgData) {
-          // Forward PostgreSQL response to client with backpressure.
-          // Re-pause defensively when pendingToClient already exists —
-          // kernel-buffered PG data can arrive before the earlier pause()
-          // takes effect (issue #18 root cause #3).
-          if (state.pendingToClient) {
-            state.pendingToClient = Buffer.concat([state.pendingToClient, pgData]);
-            _pgSocket.pause();
-            return;
-          }
-          const written = socket.write(pgData);
-          if (written < pgData.byteLength) {
-            state.pendingToClient = written === 0 ? Buffer.from(pgData) : Buffer.from(pgData.subarray(written));
-            _pgSocket.pause();
-          }
-        },
-        open(pgSocket) {
-          pgSocket.write(startupMessage);
-          state.handshakeComplete = true;
-        },
-        close(_pgSocket) {
-          socket.end();
-        },
-        error(_pgSocket, error) {
-          router.logger.error({ dbName, err: error }, 'PostgreSQL socket error');
-          socket.end();
-        },
-        drain(_pgSocket) {
-          // Flush any pending Client→PG data
-          if (state.pendingToPg) {
-            state.pendingToPg = flushPending(_pgSocket, state.pendingToPg);
-          }
-          // If fully flushed, resume reading from client
-          if (!state.pendingToPg) {
-            socket.resume();
-          }
-        }
-      };
-
-      // Safety net for issue #24: if socketPath points to a directory that was
-      // cleaned up (e.g. pgManager was stopped+started, or the PG subprocess
-      // exited unexpectedly and socketDir was reset to null but a stale cached
-      // path is still hanging around), fall back to TCP instead of Bun.connect
-      // hanging on a missing unix socket.
-      const useUnix = socketPath && fs.existsSync(socketPath);
-      if (useUnix) {
-        state.pgSocket = await Bun.connect({ unix: socketPath, socket: pgHandler });
-      } else {
-        if (socketPath && !useUnix) {
-          this.logger.warn({ socketPath, dbName }, 'Unix socket path stale — falling back to TCP');
-        }
-        state.pgSocket = await Bun.connect({ hostname: '127.0.0.1', port: this.pgPort, socket: pgHandler });
-      }
-
-      this.emit('connection', { dbName, socket });
-    } catch (error) {
-      this.logger.error({ dbName, err: error }, 'Connection error');
-      socket.end();
-      this.emit('connection-error', { error, dbName });
-    } finally {
-      // Release the reentrancy guard whether handshake succeeded or not.
-      // If it succeeded, handshakeComplete is now true and further data
-      // events will bypass processStartupMessage anyway (handleSocketData
-      // takes the handshakeComplete path). If it failed, socket.end()
-      // has been called and the connection is tearing down.
-      state.startupInProgress = false;
-    }
-  }
-
-  /**
-   * Handle socket close (Bun TCP handler)
-   */
-  handleSocketClose(socket) {
-    const state = this.socketState.get(socket);
-    if (state) {
-      state.pendingToPg = null;
-      state.pendingToClient = null;
-      if (state.pgSocket) state.pgSocket.end();
-    }
-    this.connections.delete(socket);
-    this.socketState.delete(socket);
-  }
-
-  /**
-   * Handle socket error (Bun TCP handler)
-   */
-  handleSocketError(socket, error) {
-    const state = this.socketState.get(socket);
-    // Only log non-connection-reset errors
-    if (error.code !== 'ECONNRESET') {
-      this.logger.error({ err: error, dbName: state?.dbName }, 'Socket error');
-    }
-    if (state) {
-      state.pendingToPg = null;
-      state.pendingToClient = null;
-      if (state.pgSocket) state.pgSocket.end();
-    }
-    this.connections.delete(socket);
-    this.socketState.delete(socket);
-  }
-
-  /**
-   * Stop router (graceful shutdown)
-   */
-  async stop() {
-    this.logger.info('Stopping multi-tenant router');
-
-    // Close all connections gracefully
-    const activeConns = this.connections.size;
-    for (const socket of this.connections) {
-      socket.end();
-    }
-    this.connections.clear();
-
-    // Close TCP server (Bun.listen returns a server with stop() method)
-    if (this.server) {
-      this.server.stop();
-    }
-
-    // Stop SyncManager first (before PostgreSQL)
-    if (this.syncManager) {
-      await this.syncManager.stop();
-    }
-
-    // Stop PostgreSQL
-    await this.pgManager.stop();
-
-    this.logger.info({
-      activeConnections: activeConns
-    }, 'Router stopped');
-
-    this.emit('stopped');
-  }
-
-  /**
-   * Get router stats
-   */
-  getStats() {
-    return {
-      port: this.port,
-      host: this.host,
-      pgPort: this.pgPort,
-      activeConnections: this.connections.size,
-      postgres: this.pgManager.getStats()
-    };
-  }
-
-  /**
-   * List all databases
-   */
-  listDatabases() {
-    return this.pgManager.getStats().databases;
-  }
-}
-
-/**
- * Start multi-tenant router (convenience function)
- */
-export async function startMultiTenantServer(options = {}) {
-  const router = new MultiTenantRouter(options);
-  await router.start();
-  return router;
-}

package/src/sdk.js

@@ -1,137 +0,0 @@
-/**
- * Public SDK helpers for applications that want to consume the singleton
- * pgserve daemon without shelling out themselves.
- *
- * The intended flow is:
- *   1. App calls ensureDaemon() during install/startup.
- *   2. App connects with daemonClientOptions().
- *   3. pgserve derives the app identity from the Unix-socket peer creds and
- *      routes it to that app's fingerprinted database.
- */
-
-import { spawn } from 'child_process';
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-import {
-  isProcessAlive,
-  resolveControlSocketDir,
-  resolveControlSocketPath,
-  resolveLibpqCompatPath,
-  resolvePidLockPath,
-} from './daemon.js';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-export function probeDaemon({ controlSocketDir = resolveControlSocketDir() } = {}) {
-  const socketPath = resolveControlSocketPath(controlSocketDir);
-  const libpqSocketPath = resolveLibpqCompatPath(controlSocketDir);
-  const pidLockPath = resolvePidLockPath(controlSocketDir);
-  const socketPresent = fs.existsSync(socketPath);
-  const libpqSocketPresent = fs.existsSync(libpqSocketPath);
-  let pid = null;
-
-  try {
-    const raw = fs.readFileSync(pidLockPath, 'utf8').trim();
-    const parsed = Number.parseInt(raw, 10);
-    if (Number.isInteger(parsed) && parsed > 0) pid = parsed;
-  } catch {
-    // Missing/unreadable pid file means no live daemon can be trusted.
-  }
-
-  const pidAlive = pid !== null && isProcessAlive(pid);
-  const running = pidAlive && socketPresent && libpqSocketPresent;
-  return {
-    running,
-    pid: pidAlive ? pid : null,
-    socketPresent,
-    libpqSocketPresent,
-    controlSocketDir,
-    controlSocketPath: socketPath,
-    libpqSocketPath,
-    pidLockPath,
-    reason: running ? null : explainProbeMiss({ pid, pidAlive, socketPresent, libpqSocketPresent }),
-  };
-}
-
-function explainProbeMiss({ pid, pidAlive, socketPresent, libpqSocketPresent }) {
-  if (pid === null && !socketPresent && !libpqSocketPresent) return 'no daemon';
-  if (pid !== null && !pidAlive) return 'stale pid';
-  if (!socketPresent) return 'control socket missing';
-  if (!libpqSocketPresent) return 'libpq socket missing';
-  return 'not running';
-}
-
-export function daemonClientOptions({
-  controlSocketDir = resolveControlSocketDir(),
-  database = 'postgres',
-  username = 'postgres',
-} = {}) {
-  return {
-    host: controlSocketDir,
-    port: 5432,
-    database,
-    username,
-    password: '',
-  };
-}
-
-export function buildDaemonArgs({
-  dataDir,
-  ram = false,
-  logLevel,
-  noProvision = false,
-  listens = [],
-  pgvector = false,
-} = {}) {
-  const args = ['daemon'];
-  if (dataDir) args.push('--data', dataDir);
-  if (ram) args.push('--ram');
-  if (logLevel) args.push('--log', logLevel);
-  if (noProvision) args.push('--no-provision');
-  if (pgvector) args.push('--pgvector');
-  for (const listen of Array.isArray(listens) ? listens : [listens]) {
-    if (listen) args.push('--listen', String(listen));
-  }
-  return args;
-}
-
-export async function ensureDaemon(options = {}) {
-  const controlSocketDir = options.controlSocketDir || resolveControlSocketDir();
-  const initial = probeDaemon({ controlSocketDir });
-  if (initial.running) return initial;
-
-  const bin = options.bin || resolveBundledCliBin();
-  const env = { ...process.env, ...envForControlSocketDir(controlSocketDir), ...(options.env || {}) };
-  const child = spawn(bin, buildDaemonArgs(options), {
-    detached: true,
-    stdio: 'ignore',
-    env,
-  });
-  child.unref();
-
-  const timeoutMs = options.timeoutMs || 16000;
-  const deadline = Date.now() + timeoutMs;
-  while (Date.now() < deadline) {
-    const state = probeDaemon({ controlSocketDir });
-    if (state.running) return state;
-    await new Promise((resolve) => setTimeout(resolve, 250));
-  }
-
-  const state = probeDaemon({ controlSocketDir });
-  const err = new Error(`pgserve daemon did not become ready within ${timeoutMs}ms (${state.reason})`);
-  err.code = 'EPGSERVE_DAEMON_TIMEOUT';
-  err.state = state;
-  throw err;
-}
-
-export function resolveBundledCliBin() {
-  return path.join(__dirname, '..', 'bin', 'pgserve-wrapper.cjs');
-}
-
-function envForControlSocketDir(controlSocketDir) {
-  if (path.basename(controlSocketDir) !== 'pgserve') {
-    throw new Error('ensureDaemon: controlSocketDir must be a pgserve runtime directory ending in /pgserve');
-  }
-  return { XDG_RUNTIME_DIR: path.dirname(controlSocketDir) };
-}