pgserve 0.1.5 → 1.0.2

package/eslint.config.js

@@ -0,0 +1,54 @@
+import unusedImports from 'eslint-plugin-unused-imports';
+
+export default [
+  {
+    files: ['src/**/*.js', 'bin/**/*.js', 'tests/**/*.js'],
+    plugins: {
+      'unused-imports': unusedImports,
+    },
+    languageOptions: {
+      ecmaVersion: 2022,
+      sourceType: 'module',
+      globals: {
+        console: 'readonly',
+        process: 'readonly',
+        Buffer: 'readonly',
+        setTimeout: 'readonly',
+        clearTimeout: 'readonly',
+        setInterval: 'readonly',
+        clearInterval: 'readonly',
+        URL: 'readonly',
+        __dirname: 'readonly',
+      },
+    },
+    rules: {
+      // Dead imports - ERROR
+      'unused-imports/no-unused-imports': 'error',
+      'unused-imports/no-unused-vars': [
+        'error',
+        {
+          vars: 'all',
+          varsIgnorePattern: '^_',
+          args: 'after-used',
+          argsIgnorePattern: '^_',
+        },
+      ],
+
+      // Code quality
+      'no-unused-vars': 'off', // Handled by unused-imports
+      'no-console': 'off', // We use console in CLI
+      'no-undef': 'error',
+      'no-unreachable': 'error',
+      'no-constant-condition': 'warn',
+      'no-empty': 'warn',
+      'no-duplicate-case': 'error',
+      'no-fallthrough': 'warn',
+      'eqeqeq': ['warn', 'always', { null: 'ignore' }],
+      'no-var': 'error',
+      'prefer-const': 'warn',
+    },
+  },
+  {
+    ignores: ['node_modules/', 'data/', 'test-data-*/', '.genie/'],
+  },
+];

package/knip.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://unpkg.com/knip@5/schema.json",
+  "entry": ["src/index.js", "bin/pglite-server.js"],
+  "project": ["src/**/*.js", "bin/**/*.js"],
+  "ignore": ["tests/**", "helpers/**"],
+  "ignoreDependencies": ["pino-pretty"],
+  "ignoreExportsUsedInFile": true
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pgserve",
-  "version": "0.1.5",
-  "description": "Multi-instance PostgreSQL embedded server using PGlite - zero config, auto-port allocation, perfect for development and embedded apps",
+  "version": "1.0.2",
+  "description": "Embedded PostgreSQL server with true concurrent connections - zero config, auto-provision databases",
   "main": "src/index.js",
   "type": "module",
   "bin": {
@@ -9,17 +9,19 @@
   },
   "scripts": {
     "bench": "node tests/benchmarks/runner.js",
-    "test": "node --test tests/**/*.test.js"
+    "test": "node --test tests/**/*.test.js",
+    "lint": "eslint src/ bin/",
+    "lint:fix": "eslint src/ bin/ --fix",
+    "deadcode": "knip",
+    "prepare": "husky"
   },
   "keywords": [
-    "pglite",
     "postgresql",
     "embedded",
     "database",
-    "wasm",
-    "electron",
+    "postgres",
     "development",
-    "multi-instance"
+    "multi-tenant"
   ],
   "author": "Namastex Labs <labs@namastex.com>",
   "license": "MIT",
@@ -32,16 +34,35 @@
   },
   "homepage": "https://github.com/namastexlabs/pgserve#readme",
   "dependencies": {
-    "@electric-sql/pglite": "^0.3.14",
-    "@electric-sql/pglite-socket": "^0.0.19",
+    "pg": "^8.16.3",
+    "pg-copy-streams": "^7.0.0",
     "pino": "^10.1.0",
     "pino-pretty": "^13.1.2"
   },
+  "optionalDependencies": {
+    "@embedded-postgres/darwin-arm64": "17.7.0-beta.15",
+    "@embedded-postgres/darwin-x64": "17.7.0-beta.15",
+    "@embedded-postgres/linux-x64": "17.7.0-beta.15",
+    "@embedded-postgres/win32-x64": "17.7.0-beta.15"
+  },
   "devDependencies": {
-    "better-sqlite3": "^11.10.0",
-    "pg": "^8.16.3"
+    "@electric-sql/pglite": "^0.2.17",
+    "better-sqlite3": "^11.7.0",
+    "eslint": "^9.39.1",
+    "eslint-plugin-unused-imports": "^4.3.0",
+    "husky": "^9.1.7",
+    "knip": "^5.71.0",
+    "lint-staged": "^16.2.7"
   },
   "engines": {
     "node": ">=18.0.0"
+  },
+  "lint-staged": {
+    "src/**/*.js": [
+      "eslint --fix"
+    ],
+    "bin/**/*.js": [
+      "eslint --fix"
+    ]
   }
 }

package/src/cluster.js

@@ -0,0 +1,320 @@
+/**
+ * Cluster Mode for pgserve
+ *
+ * Architecture:
+ * - PRIMARY process: Runs single embedded PostgreSQL instance
+ * - WORKER processes: Only run TCP routing to PRIMARY's PostgreSQL
+ *
+ * This enables multi-core scaling (3-5x throughput on multi-core systems)
+ * while maintaining a single PostgreSQL instance.
+ */
+
+import cluster from 'cluster';
+import os from 'os';
+import net from 'net';
+import pg from 'pg';
+import pino from 'pino';
+import { PostgresManager } from './postgres.js';
+import { extractDatabaseNameFromSocket } from './protocol.js';
+import { EventEmitter } from 'events';
+
+/**
+ * ClusterRouter - Lightweight TCP router for worker processes
+ * Does NOT start PostgreSQL - connects to PRIMARY's PostgreSQL via Unix socket
+ */
+class ClusterRouter extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this.port = options.port || 5432;
+    this.host = options.host || '127.0.0.1';
+    this.pgSocketPath = options.pgSocketPath; // From PRIMARY
+    this.pgPort = options.pgPort;
+    this.pgUser = options.pgUser || 'postgres';
+    this.pgPassword = options.pgPassword || 'postgres';
+    this.autoProvision = options.autoProvision !== false;
+    this.maxConnections = options.maxConnections || 1000;
+
+    this.logger = pino({ level: options.logLevel || 'info' });
+    this.adminClient = null;
+    this.server = null;
+    this.connections = new Set();
+    this.setMaxListeners(this.maxConnections + 10);
+  }
+
+  optimizeSocket(socket) {
+    socket.setNoDelay(true);
+    socket.setKeepAlive(true, 60000);
+    socket.setTimeout(0);
+  }
+
+  async start() {
+    // Admin connection for auto-provisioning databases
+    if (this.autoProvision) {
+      let connectionConfig;
+      if (this.pgSocketPath) {
+        // pg library expects socket DIRECTORY as host, it appends .s.PGSQL.<port>
+        // Socket path format: /tmp/pgserve-sock-xxx/.s.PGSQL.<port>
+        // Extract directory by removing the socket file suffix
+        const socketDir = this.pgSocketPath.replace(/\/\.s\.PGSQL\.\d+$/, '');
+        connectionConfig = {
+          host: socketDir,
+          port: this.pgPort,
+          database: 'postgres',
+          user: this.pgUser,
+          password: this.pgPassword
+        };
+      } else {
+        connectionConfig = {
+          host: '127.0.0.1',
+          port: this.pgPort,
+          database: 'postgres',
+          user: this.pgUser,
+          password: this.pgPassword
+        };
+      }
+
+      this.adminClient = new pg.Client(connectionConfig);
+      await this.adminClient.connect();
+    }
+
+    return new Promise((resolve, _reject) => {
+      this.server = net.createServer({
+        allowHalfOpen: false,
+        pauseOnConnect: true
+      }, (socket) => this.handleConnection(socket));
+
+      this.server.maxConnections = this.maxConnections;
+
+      this.server.on('error', (error) => {
+        this.logger.error({ err: error }, 'Router error');
+        this.emit('error', error);
+      });
+
+      this.server.listen(this.port, this.host, () => {
+        this.emit('listening');
+        resolve();
+      });
+    });
+  }
+
+  async createDatabase(dbName) {
+    if (!this.autoProvision || !this.adminClient) return;
+
+    try {
+      const result = await this.adminClient.query(
+        'SELECT 1 FROM pg_database WHERE datname = $1',
+        [dbName]
+      );
+
+      if (result.rows.length === 0) {
+        await this.adminClient.query(`CREATE DATABASE "${dbName}"`);
+      }
+    } catch (error) {
+      // Ignore "already exists" (race condition between workers)
+      if (!error.message.includes('already exists')) {
+        this.logger.error({ database: dbName, err: error }, 'Failed to create database');
+      }
+    }
+  }
+
+  async handleConnection(socket) {
+    this.connections.add(socket);
+    this.optimizeSocket(socket);
+
+    let dbName = null;
+    let pgSocket = null;
+
+    try {
+      const { dbName: extractedDbName, buffered } = await extractDatabaseNameFromSocket(socket);
+      dbName = extractedDbName;
+
+      await this.createDatabase(dbName);
+
+      // Connect to PRIMARY's PostgreSQL
+      if (this.pgSocketPath) {
+        pgSocket = net.connect({ path: this.pgSocketPath });
+      } else {
+        pgSocket = net.connect({ host: '127.0.0.1', port: this.pgPort });
+      }
+
+      await new Promise((resolve, reject) => {
+        pgSocket.once('connect', resolve);
+        pgSocket.once('error', reject);
+      });
+
+      this.optimizeSocket(pgSocket);
+      pgSocket.write(buffered);
+      socket.resume();
+
+      // Bidirectional pipe
+      socket.pipe(pgSocket);
+      pgSocket.pipe(socket);
+
+      const cleanup = () => {
+        this.connections.delete(socket);
+        if (pgSocket && !pgSocket.destroyed) pgSocket.destroy();
+        if (socket && !socket.destroyed) socket.destroy();
+      };
+
+      socket.once('close', cleanup);
+      socket.once('error', cleanup);
+      pgSocket.once('close', () => {
+        if (socket && !socket.destroyed) socket.destroy();
+      });
+      pgSocket.once('error', cleanup);
+
+    } catch (error) {
+      this.logger.error({ dbName, err: error }, 'Connection error');
+      if (pgSocket && !pgSocket.destroyed) pgSocket.destroy();
+      socket.destroy();
+      this.connections.delete(socket);
+    }
+  }
+
+  async stop() {
+    for (const socket of this.connections) {
+      socket.end();
+    }
+    this.connections.clear();
+
+    if (this.adminClient) {
+      await this.adminClient.end();
+    }
+
+    if (this.server) {
+      await new Promise((resolve) => this.server.close(resolve));
+    }
+  }
+}
+
+/**
+ * Start pgserve in cluster mode
+ */
+export async function startClusterServer(options = {}) {
+  const numWorkers = options.workers || os.cpus().length;
+  const port = options.port || 5432;
+  const host = options.host || '127.0.0.1';
+  const pgPort = options.pgPort || (port + 1000);
+
+  if (cluster.isPrimary) {
+    console.log(`[pgserve] Cluster mode: ${numWorkers} workers`);
+
+    // PRIMARY: Start our embedded PostgreSQL (single instance)
+    const logger = pino({ level: options.logLevel || 'info' });
+    const pgManager = new PostgresManager({
+      dataDir: options.baseDir,
+      port: pgPort,
+      logger: logger.child({ component: 'postgres' })
+    });
+
+    await pgManager.start();
+    const pgSocketPath = pgManager.getSocketPath();
+
+    console.log(`[pgserve] Embedded PostgreSQL started`);
+    console.log(`[pgserve] Socket: ${pgSocketPath || `TCP port ${pgPort}`}`);
+
+    const workers = new Map();
+
+    // Fork workers with PostgreSQL connection info
+    for (let i = 0; i < numWorkers; i++) {
+      const worker = cluster.fork({
+        PGSERVE_WORKER: 'true',
+        PGSERVE_PORT: String(port),
+        PGSERVE_HOST: host,
+        PGSERVE_PG_SOCKET: pgSocketPath || '',
+        PGSERVE_PG_PORT: String(pgPort),
+        PGSERVE_PG_USER: 'postgres',
+        PGSERVE_PG_PASSWORD: 'postgres',
+        PGSERVE_LOG_LEVEL: options.logLevel || 'info',
+        PGSERVE_AUTO_PROVISION: options.autoProvision !== false ? 'true' : 'false'
+      });
+      workers.set(worker.id, worker);
+    }
+
+    // Restart dead workers
+    cluster.on('exit', (worker, code, signal) => {
+      console.log(`[pgserve] Worker ${worker.id} died (${signal || code}), restarting...`);
+      workers.delete(worker.id);
+
+      const newWorker = cluster.fork({
+        PGSERVE_WORKER: 'true',
+        PGSERVE_PORT: String(port),
+        PGSERVE_HOST: host,
+        PGSERVE_PG_SOCKET: pgSocketPath || '',
+        PGSERVE_PG_PORT: String(pgPort),
+        PGSERVE_PG_USER: 'postgres',
+        PGSERVE_PG_PASSWORD: 'postgres',
+        PGSERVE_LOG_LEVEL: options.logLevel || 'info',
+        PGSERVE_AUTO_PROVISION: options.autoProvision !== false ? 'true' : 'false'
+      });
+      workers.set(newWorker.id, newWorker);
+    });
+
+    // Wait for workers to be ready
+    let readyCount = 0;
+    await new Promise((resolve) => {
+      cluster.on('message', (worker, message) => {
+        if (message.type === 'ready') {
+          readyCount++;
+          if (readyCount === numWorkers) resolve();
+        }
+      });
+    });
+
+    console.log(`[pgserve] All ${numWorkers} workers ready`);
+    console.log(`[pgserve] Listening on ${host}:${port}`);
+
+    return {
+      workers,
+      pgPort,
+      pgSocketPath,
+      stop: async () => {
+        console.log('[pgserve] Stopping cluster...');
+        for (const worker of workers.values()) {
+          worker.send({ type: 'shutdown' });
+        }
+        await new Promise((resolve) => {
+          const check = setInterval(() => {
+            if (workers.size === 0) {
+              clearInterval(check);
+              resolve();
+            }
+          }, 100);
+        });
+        await pgManager.stop();
+        console.log('[pgserve] Cluster stopped');
+      },
+      getStats: () => ({
+        workers: workers.size,
+        pids: Array.from(workers.values()).map(w => w.process.pid)
+      })
+    };
+  } else {
+    // WORKER: Only run TCP routing, connect to PRIMARY's PostgreSQL
+    const router = new ClusterRouter({
+      port: parseInt(process.env.PGSERVE_PORT) || 5432,
+      host: process.env.PGSERVE_HOST || '127.0.0.1',
+      pgSocketPath: process.env.PGSERVE_PG_SOCKET || null,
+      pgPort: parseInt(process.env.PGSERVE_PG_PORT) || 6432,
+      pgUser: process.env.PGSERVE_PG_USER || 'postgres',
+      pgPassword: process.env.PGSERVE_PG_PASSWORD || 'postgres',
+      logLevel: process.env.PGSERVE_LOG_LEVEL || 'info',
+      autoProvision: process.env.PGSERVE_AUTO_PROVISION === 'true'
+    });
+
+    await router.start();
+
+    // Tell PRIMARY we're ready
+    process.send({ type: 'ready' });
+
+    // Handle shutdown
+    process.on('message', async (message) => {
+      if (message.type === 'shutdown') {
+        await router.stop();
+        process.exit(0);
+      }
+    });
+
+    return router;
+  }
+}

package/src/dashboard.js

@@ -0,0 +1,211 @@
+/**
+ * Dashboard - Informative CLI startup display
+ *
+ * Hybrid approach:
+ * - Scrolling stages (preserved history)
+ * - In-place progress updates (only during restore)
+ * - Non-TTY fallback (works in pipes/CI)
+ *
+ * Zero external dependencies - pure Node.js ANSI codes
+ */
+
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+// Get package version
+let version = '1.0.0';
+try {
+  const __dirname = dirname(fileURLToPath(import.meta.url));
+  const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf8'));
+  version = pkg.version;
+} catch {
+  // Fallback version
+}
+
+// ANSI escape codes
+const ANSI = {
+  CLEAR_LINE: '\x1B[2K',
+  MOVE_UP: (n) => `\x1B[${n}A`,
+  MOVE_DOWN: (n) => `\x1B[${n}B`,
+  HIDE_CURSOR: '\x1B[?25l',
+  SHOW_CURSOR: '\x1B[?25h',
+  GREEN: '\x1B[32m',
+  YELLOW: '\x1B[33m',
+  CYAN: '\x1B[36m',
+  DIM: '\x1B[2m',
+  RESET: '\x1B[0m',
+  BOLD: '\x1B[1m'
+};
+
+/**
+ * Dashboard class for CLI output
+ */
+export class Dashboard {
+  constructor(options = {}) {
+    this.enabled = process.stdout.isTTY && !process.env.NO_COLOR;
+    this.updateInterval = options.updateInterval || 200; // Throttle updates
+    this.lastUpdate = 0;
+    this.progressLines = 0; // Track lines to overwrite
+    this.restoreStartTime = 0;
+    this.config = options.config || {};
+  }
+
+  /**
+   * Show the startup header
+   */
+  showHeader(config = {}) {
+    const mode = config.memoryMode ? 'In-memory' : 'Persistent';
+    const port = config.port || 5432;
+    const host = config.host || '127.0.0.1';
+    const syncTo = config.syncTo ? ` → ${this._maskUrl(config.syncTo)}` : '';
+
+    console.log('');
+    console.log(`${ANSI.BOLD}pgserve v${version}${ANSI.RESET} - Embedded PostgreSQL Server`);
+    console.log(`${ANSI.DIM}MODE: ${mode}  |  PORT: ${port}  |  HOST: ${host}${syncTo}${ANSI.RESET}`);
+    console.log('');
+  }
+
+  /**
+   * Log a stage completion
+   */
+  stage(name, status = 'done') {
+    const icon = status === 'done' ? `${ANSI.GREEN}[✓]${ANSI.RESET}` :
+                 status === 'error' ? `${ANSI.YELLOW}[✗]${ANSI.RESET}` :
+                 `${ANSI.DIM}[○]${ANSI.RESET}`;
+    console.log(`${icon} ${name}`);
+  }
+
+  /**
+   * Start restore progress section
+   */
+  startRestore(totalDatabases, totalTables = 0, totalBytes = 0) {
+    this.restoreStartTime = Date.now();
+    this.totalDatabases = totalDatabases;
+    this.totalTables = totalTables || totalDatabases * 10; // Estimate
+    this.totalBytes = totalBytes;
+
+    console.log('');
+    console.log(`${ANSI.CYAN}Restoring from external PostgreSQL...${ANSI.RESET}`);
+
+    if (this.enabled) {
+      // Reserve lines for progress
+      console.log('  Databases:  0/0   [░░░░░░░░░░░░░░░░]   0%');
+      console.log('  Tables:     0/0   [░░░░░░░░░░░░░░░░]   0%');
+      console.log('  Speed:      0.0 MB/s  |  ETA: calculating...');
+      this.progressLines = 3;
+      process.stdout.write(ANSI.HIDE_CURSOR);
+    }
+  }
+
+  /**
+   * Update restore progress (in-place)
+   */
+  updateRestore(metrics) {
+    if (!this.enabled) return;
+
+    // Throttle updates
+    const now = Date.now();
+    if (now - this.lastUpdate < this.updateInterval) return;
+    this.lastUpdate = now;
+
+    const {
+      databasesRestored = 0,
+      totalDatabases = this.totalDatabases || 1,
+      tablesRestored = 0,
+      totalTables = this.totalTables || 1,
+      bytesTransferred = 0
+    } = metrics;
+
+    // Calculate throughput and ETA
+    const elapsed = (now - this.restoreStartTime) / 1000;
+    const throughputMBps = elapsed > 0 ? (bytesTransferred / 1024 / 1024) / elapsed : 0;
+    const bytesRemaining = this.totalBytes - bytesTransferred;
+    const eta = throughputMBps > 0 ? Math.ceil(bytesRemaining / (throughputMBps * 1024 * 1024)) : 0;
+
+    // Format progress
+    const dbPct = Math.round((databasesRestored / totalDatabases) * 100);
+    const tablePct = Math.round((tablesRestored / totalTables) * 100);
+    const dbBar = this._progressBar(databasesRestored, totalDatabases);
+    const tableBar = this._progressBar(tablesRestored, totalTables);
+    const etaStr = eta > 0 ? `~${eta}s` : 'finishing...';
+
+    // Move up and overwrite
+    process.stdout.write(ANSI.MOVE_UP(this.progressLines));
+
+    process.stdout.write(ANSI.CLEAR_LINE);
+    console.log(`  Databases:  ${String(databasesRestored).padStart(2)}/${totalDatabases}   ${dbBar}  ${String(dbPct).padStart(3)}%`);
+
+    process.stdout.write(ANSI.CLEAR_LINE);
+    console.log(`  Tables:     ${String(tablesRestored).padStart(3)}/${totalTables}  ${tableBar}  ${String(tablePct).padStart(3)}%`);
+
+    process.stdout.write(ANSI.CLEAR_LINE);
+    console.log(`  Speed:      ${throughputMBps.toFixed(1)} MB/s  |  ETA: ${etaStr}`);
+  }
+
+  /**
+   * Complete restore progress (replace with summary)
+   */
+  completeRestore(metrics) {
+    if (this.enabled && this.progressLines > 0) {
+      // Move up and clear progress lines
+      process.stdout.write(ANSI.MOVE_UP(this.progressLines));
+      for (let i = 0; i < this.progressLines; i++) {
+        process.stdout.write(ANSI.CLEAR_LINE + '\n');
+      }
+      process.stdout.write(ANSI.MOVE_UP(this.progressLines));
+      process.stdout.write(ANSI.SHOW_CURSOR);
+      this.progressLines = 0;
+    }
+
+    const duration = ((metrics.endTime || Date.now()) - this.restoreStartTime) / 1000;
+    const mb = (metrics.bytesTransferred / 1024 / 1024).toFixed(1);
+
+    console.log(`${ANSI.GREEN}[✓]${ANSI.RESET} Restored ${metrics.databasesRestored} database${metrics.databasesRestored !== 1 ? 's' : ''} (${mb} MB in ${duration.toFixed(1)}s)`);
+    console.log('');
+  }
+
+  /**
+   * Show final ready message
+   */
+  showReady(config = {}) {
+    const port = config.port || 5432;
+    const host = config.host || '127.0.0.1';
+
+    console.log('');
+    console.log(`${ANSI.GREEN}${ANSI.BOLD}✨ READY${ANSI.RESET}: postgresql://${host}:${port}/<database>`);
+    console.log(`${ANSI.DIM}Press Ctrl+C to stop${ANSI.RESET}`);
+    console.log('');
+  }
+
+  /**
+   * Generate progress bar
+   */
+  _progressBar(current, total, width = 16) {
+    const pct = total > 0 ? current / total : 0;
+    const filled = Math.round(pct * width);
+    const empty = width - filled;
+    return '[' + '█'.repeat(filled) + '░'.repeat(empty) + ']';
+  }
+
+  /**
+   * Mask sensitive URL parts
+   */
+  _maskUrl(url) {
+    try {
+      const u = new URL(url);
+      return `${u.protocol}//${u.host}/${u.pathname.split('/')[1] || ''}`;
+    } catch {
+      return url.replace(/:[^:@]+@/, ':***@');
+    }
+  }
+
+  /**
+   * Cleanup (show cursor if hidden)
+   */
+  cleanup() {
+    if (this.enabled) {
+      process.stdout.write(ANSI.SHOW_CURSOR);
+    }
+  }
+}