pgserve 2.3.0 → 2.4.0

package/src/restore.js

@@ -1,574 +0,0 @@
-/**
- * RestoreManager - Automatic restore from external PostgreSQL on startup
- *
- * High-performance restore using:
- * - Parallel database restore (Promise.all)
- * - Native wire protocol for bulk COPY transfer (pg-wire.js)
- * - Unix sockets for local connections (~30% faster)
- * - Binary format COPY (~2x faster than text)
- *
- * 100% Bun-native: Uses PgWirePool for all database operations.
- */
-
-import { PgWirePool } from './pg-wire.js';
-import { createLogger } from './logger.js';
-
-/**
- * Match database name against patterns (supports wildcards)
- * Reused from sync.js for consistency
- * @param {string} dbName - Database name to check
- * @param {string[]} patterns - Array of patterns (supports * wildcard)
- * @returns {boolean}
- */
-function matchesPattern(dbName, patterns) {
-  if (!patterns || patterns.length === 0) return true; // No filter = restore all
-
-  return patterns.some(pattern => {
-    if (pattern.includes('*')) {
-      const regex = new RegExp('^' + pattern.replace(/\*/g, '.*') + '$');
-      return regex.test(dbName);
-    }
-    return dbName === pattern;
-  });
-}
-
-/**
- * RestoreManager - Handles automatic restore from external PostgreSQL
- */
-export class RestoreManager {
-  constructor(options = {}) {
-    this.sourceUrl = options.sourceUrl;              // External PostgreSQL URL
-    this.patterns = options.patterns || [];          // Database patterns ["myapp", "tenant_*"]
-    this.targetPort = options.targetPort;            // Local embedded PostgreSQL port
-    this.targetSocketPath = options.targetSocketPath; // Unix socket path (optional)
-
-    this.logger = options.logger || createLogger({ level: options.logLevel || 'info', component: 'restore' });
-
-    // Connection pools (lazy initialized)
-    this.sourcePool = null;
-
-    // Performance tuning - parallel restore limits
-    this.maxParallelDatabases = options.maxParallelDatabases || 4;
-    this.maxParallelTables = options.maxParallelTables || 8;
-
-    // Timeout handling
-    this.restoreTimeout = options.restoreTimeout || 60000; // 60s default
-
-    // Progress callback for dashboard
-    this.onProgress = options.onProgress || (() => {});
-
-    // Totals for progress tracking
-    this.totalDatabases = 0;
-    this.totalTables = 0;
-    this.totalBytes = 0;
-
-    // Metrics collection
-    this.metrics = {
-      startTime: 0,
-      endTime: 0,
-      databasesRestored: 0,
-      tablesRestored: 0,
-      rowsRestored: 0,
-      bytesTransferred: 0,
-      errors: []
-    };
-  }
-
-  /**
-   * Main entry point - restore databases from external PostgreSQL
-   * Called from router.js after pgManager.start(), before SyncManager
-   *
-   * @param {PostgresManager} pgManager - Local PostgreSQL manager
-   * @returns {Promise<Object>} Restore result with metrics
-   */
-  async restore(pgManager) {
-    if (!this.sourceUrl) {
-      return { skipped: true, reason: 'no sourceUrl configured' };
-    }
-
-    this.metrics.startTime = Date.now();
-    this.logger.info({ source: this.sourceUrl.replace(/:[^:@]+@/, ':***@') }, 'Starting automatic restore from external PostgreSQL');
-
-    try {
-      // Initialize connection to external PostgreSQL
-      const connected = await this._initSourcePool();
-      if (!connected) {
-        return { skipped: true, reason: 'external PostgreSQL unreachable' };
-      }
-
-      // Discover databases matching patterns
-      const databases = await this._discoverDatabases();
-
-      if (databases.length === 0) {
-        this.logger.info('No databases found matching sync patterns on external PostgreSQL');
-        return { skipped: true, reason: 'no matching databases found' };
-      }
-
-      this.logger.info({ count: databases.length, databases }, 'Found databases to restore');
-
-      // Set totals for progress tracking
-      this.totalDatabases = databases.length;
-
-      // Restore databases in parallel (with controlled concurrency)
-      await this._restoreDatabasesParallel(databases, pgManager);
-
-      this.metrics.endTime = Date.now();
-      const duration = this.metrics.endTime - this.metrics.startTime;
-
-      this.logger.info({
-        databasesRestored: this.metrics.databasesRestored,
-        tablesRestored: this.metrics.tablesRestored,
-        rowsRestored: this.metrics.rowsRestored,
-        bytesTransferred: this.metrics.bytesTransferred,
-        throughputMBps: ((this.metrics.bytesTransferred / 1024 / 1024) / (duration / 1000)).toFixed(2),
-        durationMs: duration,
-        errors: this.metrics.errors.length
-      }, 'Restore completed');
-
-      return {
-        success: true,
-        metrics: { ...this.metrics }
-      };
-
-    } catch (error) {
-      this.logger.error({ err: error }, 'Restore failed');
-      return { success: false, error: error.message };
-    } finally {
-      await this._closeSourcePool();
-    }
-  }
-
-  /**
-   * Initialize connection pool to external PostgreSQL
-   * @returns {Promise<boolean>} true if connected successfully
-   */
-  async _initSourcePool() {
-    try {
-      const url = new URL(this.sourceUrl);
-      this.sourcePool = new PgWirePool({
-        hostname: url.hostname,
-        port: parseInt(url.port) || 5432,
-        database: url.pathname.slice(1) || 'postgres',
-        username: url.username || 'postgres',
-        password: url.password || 'postgres',
-        max: 3  // Small pool - just for discovery
-      });
-
-      // Test connection
-      await this.sourcePool.query('SELECT 1');
-      this.logger.debug('Connected to external PostgreSQL');
-      return true;
-
-    } catch (error) {
-      if (error.code === 'ECONNREFUSED' || error.code === 'ETIMEDOUT' || error.code === 'ENOTFOUND') {
-        this.logger.warn({ err: error.message }, 'External PostgreSQL unreachable, skipping restore');
-        return false;
-      }
-      throw error;
-    }
-  }
-
-  /**
-   * Close source connection pool
-   */
-  async _closeSourcePool() {
-    if (this.sourcePool) {
-      await this.sourcePool.end();
-      this.sourcePool = null;
-    }
-  }
-
-  /**
-   * Discover databases on external PostgreSQL matching patterns
-   * @returns {Promise<string[]>} List of database names
-   */
-  async _discoverDatabases() {
-    const result = await this.sourcePool.query(`
-      SELECT datname FROM pg_database
-      WHERE datistemplate = false
-      AND datname NOT IN ('postgres', 'template0', 'template1')
-      ORDER BY datname
-    `);
-
-    // Filter by patterns
-    return result.rows
-      .map(r => r.datname)
-      .filter(name => matchesPattern(name, this.patterns));
-  }
-
-  /**
-   * Restore databases in parallel with controlled concurrency
-   * @param {string[]} databases - Database names to restore
-   * @param {PostgresManager} pgManager - Local PostgreSQL manager
-   */
-  async _restoreDatabasesParallel(databases, pgManager) {
-    // Batch databases to limit concurrency
-    const batches = [];
-    for (let i = 0; i < databases.length; i += this.maxParallelDatabases) {
-      batches.push(databases.slice(i, i + this.maxParallelDatabases));
-    }
-
-    for (const batch of batches) {
-      const results = await Promise.allSettled(
-        batch.map(dbName => this._restoreDatabase(dbName, pgManager))
-      );
-
-      // Track failures but continue
-      for (let i = 0; i < results.length; i++) {
-        if (results[i].status === 'rejected') {
-          this.metrics.errors.push({
-            database: batch[i],
-            error: results[i].reason.message
-          });
-        }
-      }
-    }
-
-    if (this.metrics.errors.length > 0) {
-      this.logger.warn({
-        failedCount: this.metrics.errors.length,
-        totalCount: databases.length
-      }, 'Some databases failed to restore');
-    }
-  }
-
-  /**
-   * Restore a single database: create DB, schema, data
-   * @param {string} dbName - Database name
-   * @param {PostgresManager} pgManager - Local PostgreSQL manager
-   */
-  async _restoreDatabase(dbName, pgManager) {
-    this.logger.info({ dbName }, 'Restoring database');
-    const startTime = Date.now();
-
-    // Step 1: Create database locally
-    await pgManager.createDatabase(dbName);
-
-    // Step 2: Create connection pools for this specific database
-    const sourceDbPool = await this._createSourceDbPool(dbName);
-    const targetDbPool = await this._createTargetDbPool(dbName);
-
-    try {
-      // Step 3: Restore schema (types, tables, indexes, FKs)
-      await this._restoreSchema(sourceDbPool, targetDbPool, dbName);
-
-      // Step 4: Discover tables and copy data in parallel
-      const tables = await this._discoverTables(sourceDbPool);
-      this.totalTables += tables.length; // Track total for progress
-      if (tables.length > 0) {
-        await this._restoreTablesParallel(sourceDbPool, targetDbPool, tables);
-      }
-
-      // Step 5: Restore sequences (after data for correct values)
-      await this._restoreSequences(sourceDbPool, targetDbPool);
-
-      this.metrics.databasesRestored++;
-      const duration = Date.now() - startTime;
-      this.logger.info({ dbName, durationMs: duration }, 'Database restored successfully');
-
-    } finally {
-      await sourceDbPool.end();
-      await targetDbPool.end();
-    }
-  }
-
-  /**
-   * Create connection pool to specific database on external PostgreSQL
-   * @param {string} dbName - Database name
-   * @returns {Promise<PgWirePool>}
-   */
-  async _createSourceDbPool(dbName) {
-    const url = new URL(this.sourceUrl);
-
-    return new PgWirePool({
-      hostname: url.hostname,
-      port: parseInt(url.port) || 5432,
-      database: dbName,
-      username: url.username || 'postgres',
-      password: url.password || 'postgres',
-      max: this.maxParallelTables
-    });
-  }
-
-  /**
-   * Create connection pool to specific database on local embedded PostgreSQL
-   * Uses Unix socket when available for ~30% faster connections
-   * @param {string} dbName - Database name
-   * @returns {Promise<PgWirePool>}
-   */
-  async _createTargetDbPool(dbName) {
-    const config = {
-      database: dbName,
-      username: 'postgres',
-      password: 'postgres',
-      max: this.maxParallelTables
-    };
-
-    // Prefer Unix socket for faster local connections
-    if (this.targetSocketPath) {
-      config.unix = this.targetSocketPath;
-    } else {
-      config.hostname = '127.0.0.1';
-      config.port = this.targetPort;
-    }
-
-    return new PgWirePool(config);
-  }
-
-  /**
-   * Restore schema: ENUMs, tables, indexes, foreign keys
-   * Order matters: types → tables → indexes → FKs
-   * @param {PgWirePool} sourcePool - External database pool
-   * @param {PgWirePool} targetPool - Local database pool
-   * @param {string} dbName - Database name (for logging)
-   */
-  async _restoreSchema(sourcePool, targetPool, dbName) {
-    // 1. Restore ENUM types
-    await this._restoreEnums(sourcePool, targetPool);
-
-    // 2. Restore tables (structure only, no data yet)
-    await this._restoreTables(sourcePool, targetPool);
-
-    this.logger.debug({ dbName }, 'Schema restored');
-  }
-
-  /**
-   * Restore ENUM types from external database
-   */
-  async _restoreEnums(sourcePool, targetPool) {
-    const result = await sourcePool.query(`
-      SELECT n.nspname as schema, t.typname as name,
-             array_agg(e.enumlabel ORDER BY e.enumsortorder) as values
-      FROM pg_type t
-      JOIN pg_enum e ON t.oid = e.enumtypid
-      JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace
-      WHERE n.nspname = 'public'
-      GROUP BY n.nspname, t.typname
-    `);
-
-    for (const enumType of result.rows) {
-      const values = enumType.values.map(v => `'${v.replace(/'/g, "''")}'`).join(', ');
-      const createSql = `CREATE TYPE "${enumType.name}" AS ENUM (${values})`;
-
-      try {
-        await targetPool.query(createSql);
-      } catch (err) {
-        if (err.code !== '42710') throw err; // 42710 = type already exists
-      }
-    }
-  }
-
-  /**
-   * Restore table structures from external database
-   */
-  async _restoreTables(sourcePool, targetPool) {
-    // Get table list
-    const tablesResult = await sourcePool.query(`
-      SELECT table_name FROM information_schema.tables
-      WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
-      ORDER BY table_name
-    `);
-
-    for (const row of tablesResult.rows) {
-      const tableName = row.table_name;
-      const createSql = await this._getTableCreateStatement(sourcePool, tableName);
-
-      try {
-        await targetPool.query(createSql);
-      } catch (err) {
-        if (err.code !== '42P07') throw err; // 42P07 = table already exists
-      }
-    }
-  }
-
-  /**
-   * Generate CREATE TABLE statement from information_schema
-   * @param {PgWirePool} sourcePool - Source database pool
-   * @param {string} tableName - Table name
-   * @returns {Promise<string>} CREATE TABLE SQL
-   */
-  async _getTableCreateStatement(sourcePool, tableName) {
-    // Get columns
-    const columnsResult = await sourcePool.query(`
-      SELECT column_name, data_type, udt_name, character_maximum_length,
-             column_default, is_nullable, numeric_precision, numeric_scale
-      FROM information_schema.columns
-      WHERE table_schema = 'public' AND table_name = $1
-      ORDER BY ordinal_position
-    `, [tableName]);
-
-    const columns = columnsResult.rows.map(col => {
-      let type = col.data_type;
-
-      // Handle special types
-      if (type === 'USER-DEFINED') {
-        type = `"${col.udt_name}"`; // ENUM or custom type
-      } else if (type === 'character varying' && col.character_maximum_length) {
-        type = `varchar(${col.character_maximum_length})`;
-      } else if (type === 'character' && col.character_maximum_length) {
-        type = `char(${col.character_maximum_length})`;
-      } else if (type === 'numeric' && col.numeric_precision) {
-        type = col.numeric_scale
-          ? `numeric(${col.numeric_precision},${col.numeric_scale})`
-          : `numeric(${col.numeric_precision})`;
-      } else if (type === 'ARRAY') {
-        type = `${col.udt_name.replace(/^_/, '')}[]`;
-      }
-
-      let colDef = `"${col.column_name}" ${type}`;
-
-      if (col.column_default) {
-        colDef += ` DEFAULT ${col.column_default}`;
-      }
-
-      if (col.is_nullable === 'NO') {
-        colDef += ' NOT NULL';
-      }
-
-      return colDef;
-    });
-
-    // Get primary key
-    const pkResult = await sourcePool.query(`
-      SELECT a.attname
-      FROM pg_index i
-      JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
-      WHERE i.indrelid = $1::regclass AND i.indisprimary
-      ORDER BY array_position(i.indkey, a.attnum)
-    `, [tableName]);
-
-    if (pkResult.rows.length > 0) {
-      const pkCols = pkResult.rows.map(r => `"${r.attname}"`).join(', ');
-      columns.push(`PRIMARY KEY (${pkCols})`);
-    }
-
-    return `CREATE TABLE "${tableName}" (\n  ${columns.join(',\n  ')}\n)`;
-  }
-
-  /**
-   * Discover tables in the database
-   * @param {PgWirePool} sourcePool - Source database pool
-   * @returns {Promise<string[]>} Table names
-   */
-  async _discoverTables(sourcePool) {
-    const result = await sourcePool.query(`
-      SELECT table_name FROM information_schema.tables
-      WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
-      ORDER BY table_name
-    `);
-
-    return result.rows.map(r => r.table_name);
-  }
-
-  /**
-   * Restore table data in parallel using COPY protocol
-   * @param {PgWirePool} sourcePool - Source database pool
-   * @param {PgWirePool} targetPool - Target database pool
-   * @param {string[]} tables - Table names
-   */
-  async _restoreTablesParallel(sourcePool, targetPool, tables) {
-    // Batch tables to limit concurrency
-    const batches = [];
-    for (let i = 0; i < tables.length; i += this.maxParallelTables) {
-      batches.push(tables.slice(i, i + this.maxParallelTables));
-    }
-
-    for (const batch of batches) {
-      await Promise.all(
-        batch.map(table => this._copyTableData(sourcePool, targetPool, table))
-      );
-    }
-  }
-
-  /**
-   * Copy table data using binary COPY protocol (high performance)
-   * Uses async generator pattern for streaming data between connections.
-   * @param {PgWirePool} sourcePool - Source database pool
-   * @param {PgWirePool} targetPool - Target database pool
-   * @param {string} tableName - Table name
-   */
-  async _copyTableData(sourcePool, targetPool, tableName) {
-    // Get row count first (for metrics)
-    const countResult = await sourcePool.query(
-      `SELECT COUNT(*)::int as count FROM "${tableName}"`
-    );
-    const rowCount = countResult.rows[0].count;
-
-    if (rowCount === 0) {
-      this.logger.debug({ tableName, rows: 0 }, 'Skipping empty table');
-      return;
-    }
-
-    // Get connections for COPY streaming
-    const sourceClient = await sourcePool.connect();
-    const targetClient = await targetPool.connect();
-
-    let bytesTransferred = 0;
-
-    try {
-      // Create byte-tracking wrapper for the async generator
-      async function* trackBytes(source) {
-        for await (const chunk of source) {
-          bytesTransferred += chunk.length;
-          yield chunk;
-        }
-      }
-
-      // Stream COPY: source → target using async generators
-      const copyToSql = `COPY "${tableName}" TO STDOUT WITH (FORMAT binary)`;
-      const copyFromSql = `COPY "${tableName}" FROM STDIN WITH (FORMAT binary)`;
-
-      const sourceStream = sourceClient.copyTo(copyToSql);
-      await targetClient.copyFrom(copyFromSql, trackBytes(sourceStream));
-
-      // Update metrics
-      this.metrics.bytesTransferred += bytesTransferred;
-      this.metrics.rowsRestored += rowCount;
-      this.metrics.tablesRestored++;
-
-      this.logger.debug({ tableName, rows: rowCount, bytes: bytesTransferred }, 'Table data copied');
-
-      // Emit progress for dashboard
-      this.onProgress({
-        databasesRestored: this.metrics.databasesRestored,
-        totalDatabases: this.totalDatabases,
-        tablesRestored: this.metrics.tablesRestored,
-        totalTables: this.totalTables,
-        bytesTransferred: this.metrics.bytesTransferred,
-        totalBytes: this.totalBytes
-      });
-
-    } finally {
-      sourcePool.release(sourceClient);
-      targetPool.release(targetClient);
-    }
-  }
-
-  /**
-   * Restore sequences to correct values (after data restore)
-   * @param {PgWirePool} sourcePool - Source database pool
-   * @param {PgWirePool} targetPool - Target database pool
-   */
-  async _restoreSequences(sourcePool, targetPool) {
-    // Get all sequences
-    const seqResult = await sourcePool.query(`
-      SELECT sequence_name FROM information_schema.sequences
-      WHERE sequence_schema = 'public'
-    `);
-
-    for (const seq of seqResult.rows) {
-      const seqName = seq.sequence_name;
-
-      // Get current value from source
-      const valueResult = await sourcePool.query(`SELECT last_value FROM "${seqName}"`);
-      const lastValue = valueResult.rows[0].last_value;
-
-      // Set on target
-      try {
-        await targetPool.query(`SELECT setval($1, $2, true)`, [seqName, lastValue]);
-      } catch (err) {
-        this.logger.warn({ sequence: seqName, err: err.message }, 'Failed to restore sequence');
-      }
-    }
-  }
-}