spindb 0.26.2 → 0.27.3

package/engines/mysql/restore.ts

@@ -199,47 +199,32 @@ async function getMysqlClientPath(binPath?: string): Promise<string> {
   )
 }
 
-export async function restoreBackup(
-  backupPath: string,
-  options: RestoreOptions,
-): Promise<RestoreResult> {
-  const {
-    port,
-    database,
-    user = engineDef.superuser,
-    validateVersion = true,
-    binPath,
-  } = options
+// Compatibility SQL to handle large row sizes and other edge cases
+// - innodb_default_row_format=DYNAMIC: Store long columns off-page to avoid row size limits
+// - innodb_strict_mode=OFF: Allow tables that might exceed row size limits in strict mode
+// - foreign_key_checks=0: Defer FK checks until after all tables are created
+// - unique_checks=0: Speed up bulk inserts
+const COMPAT_INIT_SQL = [
+  "SET GLOBAL innodb_default_row_format='dynamic';",
+  'SET SESSION innodb_strict_mode=OFF;',
+  "SET SESSION sql_mode='NO_ENGINE_SUBSTITUTION';",
+  'SET SESSION foreign_key_checks=0;',
+  'SET SESSION unique_checks=0;',
+  '',
+].join('\n')
 
-  // Validate version compatibility if requested
-  if (validateVersion) {
-    try {
-      await validateRestoreCompatibility({ dumpPath: backupPath })
-    } catch (error) {
-      // Re-throw SpinDBError, log and continue for other errors
-      if (error instanceof Error && error.name === 'SpinDBError') {
-        throw error
-      }
-      logDebug('Version validation failed, proceeding anyway', {
-        error: error instanceof Error ? error.message : String(error),
-      })
-    }
-  }
-
-  const mysql = await getMysqlClientPath(binPath)
-
-  // Detect format and check for wrong engine
-  const format = await detectBackupFormat(backupPath)
-  logDebug('Detected backup format', { format: format.format })
-  assertCompatibleFormat(format)
-
-  // Restore using mysql client
-  // CLI: mysql -h 127.0.0.1 -P {port} -u root {db} < {file}
-  // For compressed files: gunzip -c {file} | mysql ...
-
-  // Use plain spawn (no shell) so the executable path can be quoted when
-  // it contains spaces (e.g., 'C:\Program Files\...'). Using shell:true
-  // previously caused quoting issues on Windows.
+/**
+ * Internal restore function with optional compatibility mode
+ */
+function doRestore(
+  backupPath: string,
+  mysql: string,
+  port: number,
+  database: string,
+  user: string,
+  format: BackupFormat,
+  withCompatSettings: boolean,
+): Promise<RestoreResult & { rawStderr?: string }> {
   const spawnOptions: SpawnOptions = {
     stdio: ['pipe', 'pipe', 'pipe'],
   }
@@ -247,39 +232,68 @@ export async function restoreBackup(
   return new Promise((resolve, reject) => {
     const args = ['-h', '127.0.0.1', '-P', String(port), '-u', user, database]
 
-    logDebug('Restoring backup with mysql', { mysql, args, spawnOptions })
+    logDebug('Restoring backup with mysql', {
+      mysql,
+      args,
+      withCompatSettings,
+    })
 
     const proc = spawn(mysql, args, spawnOptions)
 
-    // Pipe backup file to stdin, decompressing if necessary
+    // Track whether we've already settled the promise to avoid duplicate rejections
+    let settled = false
     const fileStream = createReadStream(backupPath)
 
+    const rejectOnce = (err: Error) => {
+      if (settled) return
+      settled = true
+      fileStream.destroy()
+      proc.stdin?.end()
+      reject(err)
+    }
+
+    // Handle file read errors
+    fileStream.on('error', (err) => {
+      rejectOnce(new Error(`Failed to read backup file: ${err.message}`))
+    })
+
+    if (!proc.stdin) {
+      rejectOnce(
+        new Error(
+          'MySQL process stdin is not available, cannot restore backup',
+        ),
+      )
+      return
+    }
+
+    // Handle EPIPE errors on stdin - this happens when mysql exits due to SQL errors
+    // while we're still piping data. The actual error will be in stderr.
+    proc.stdin.on('error', (err) => {
+      // EPIPE is expected when the process exits early - don't reject here,
+      // let the 'close' event handle it with the actual error from stderr
+      if ((err as NodeJS.ErrnoException).code !== 'EPIPE') {
+        rejectOnce(new Error(`Failed to write to MySQL process: ${err.message}`))
+      }
+    })
+
+    // Prepend compatibility settings if requested
+    if (withCompatSettings) {
+      proc.stdin.write(COMPAT_INIT_SQL)
+      logDebug('Prepended compatibility settings to restore')
+    }
+
     if (format.format === 'compressed') {
       // Decompress gzipped file before piping to mysql
       const gunzip = createGunzip()
-      if (!proc.stdin) {
-        reject(
-          new Error(
-            'MySQL process stdin is not available, cannot restore backup',
-          ),
-        )
-        return
-      }
       fileStream.pipe(gunzip).pipe(proc.stdin)
 
       // Handle gunzip errors
       gunzip.on('error', (err) => {
-        reject(new Error(`Failed to decompress backup file: ${err.message}`))
+        fileStream.unpipe(gunzip)
+        gunzip.unpipe(proc.stdin!)
+        rejectOnce(new Error(`Failed to decompress backup file: ${err.message}`))
       })
     } else {
-      if (!proc.stdin) {
-        reject(
-          new Error(
-            'MySQL process stdin is not available, cannot restore backup',
-          ),
-        )
-        return
-      }
       fileStream.pipe(proc.stdin)
     }
 
@@ -294,18 +308,104 @@ export async function restoreBackup(
     })
 
     proc.on('close', (code) => {
+      if (settled) return
+      settled = true
+
       resolve({
         format: format.format,
         stdout,
         stderr,
+        rawStderr: stderr,
         code: code ?? undefined,
       })
     })
 
-    proc.on('error', reject)
+    proc.on('error', (err) => {
+      rejectOnce(err)
+    })
   })
 }
 
+/**
+ * Restore a MySQL backup to a database
+ *
+ * CLI equivalent: mysql -h 127.0.0.1 -P {port} -u root {db} < {file}
+ *
+ * Uses retry logic: if restore fails with ERROR 1118 (row size too large),
+ * automatically retries with compatibility settings that enable DYNAMIC row format.
+ */
+export async function restoreBackup(
+  backupPath: string,
+  options: RestoreOptions,
+): Promise<RestoreResult> {
+  const {
+    port,
+    database,
+    user = engineDef.superuser,
+    validateVersion = true,
+    binPath,
+  } = options
+
+  // Validate version compatibility if requested
+  if (validateVersion) {
+    try {
+      await validateRestoreCompatibility({ dumpPath: backupPath })
+    } catch (error) {
+      // Re-throw SpinDBError, log and continue for other errors
+      if (error instanceof Error && error.name === 'SpinDBError') {
+        throw error
+      }
+      logDebug('Version validation failed, proceeding anyway', {
+        error: error instanceof Error ? error.message : String(error),
+      })
+    }
+  }
+
+  const mysql = await getMysqlClientPath(binPath)
+
+  // Detect format and check for wrong engine
+  const format = await detectBackupFormat(backupPath)
+  logDebug('Detected backup format', { format: format.format })
+  assertCompatibleFormat(format)
+
+  // First attempt: try without compatibility settings
+  const result = await doRestore(backupPath, mysql, port, database, user, format, false)
+
+  // Check if restore succeeded
+  if (result.code === 0) {
+    return result
+  }
+
+  // Check if it failed with row size error (ERROR 1118)
+  // This happens when tables have too many VARCHAR columns for the default row format
+  const isRowSizeError = result.rawStderr?.includes('ERROR 1118') ||
+    result.rawStderr?.includes('Row size too large')
+
+  if (isRowSizeError) {
+    logDebug('Detected row size error, retrying with compatibility settings')
+
+    // Retry with compatibility settings
+    const retryResult = await doRestore(backupPath, mysql, port, database, user, format, true)
+
+    if (retryResult.code === 0) {
+      return {
+        ...retryResult,
+        stdout: retryResult.stdout || 'Restore succeeded with compatibility mode (DYNAMIC row format)',
+      }
+    }
+
+    // Still failed - report the retry error
+    const errorMatch = retryResult.rawStderr?.match(/^ERROR\s+\d+.*$/m)
+    const errorMessage = errorMatch ? errorMatch[0] : retryResult.rawStderr?.trim() || 'Unknown error'
+    throw new Error(`MySQL restore failed: ${errorMessage}`)
+  }
+
+  // Failed with a different error - report it
+  const errorMatch = result.rawStderr?.match(/^ERROR\s+\d+.*$/m)
+  const errorMessage = errorMatch ? errorMatch[0] : result.rawStderr?.trim() || 'Unknown error'
+  throw new Error(`MySQL restore failed: ${errorMessage}`)
+}
+
 /**
  * Parse a MySQL connection string
  *

package/engines/questdb/backup.ts

@@ -0,0 +1,217 @@
+/**
+ * QuestDB Backup Implementation
+ *
+ * QuestDB uses a custom backup format based on SQL exports.
+ * Since QuestDB uses PostgreSQL wire protocol, we can use psql for backup
+ * but QuestDB has its own SQL extensions for time-series data.
+ *
+ * Backup approach:
+ * - Export table schemas using SHOW CREATE TABLE
+ * - Export data using SELECT with proper ordering
+ * - Output as standard SQL statements
+ */
+
+import { writeFile, stat } from 'fs/promises'
+import { spawn } from 'child_process'
+import { configManager } from '../../core/config-manager'
+import { logDebug, logWarning } from '../../core/error-handler'
+import type {
+  ContainerConfig,
+  BackupOptions,
+  BackupResult,
+} from '../../types'
+
+/**
+ * Execute a query against QuestDB using psql (PostgreSQL protocol)
+ */
+async function executeQuery(
+  port: number,
+  database: string,
+  query: string,
+): Promise<string> {
+  // Try to find psql from config or PATH
+  let psqlPath = await configManager.getBinaryPath('psql')
+  if (!psqlPath) {
+    // Fall back to system psql
+    psqlPath = 'psql'
+  }
+
+  return new Promise((resolve, reject) => {
+    const args = [
+      '-h', '127.0.0.1',
+      '-p', String(port),
+      '-U', 'admin',
+      '-d', database,
+      '-t', // Tuples only (no headers)
+      '-A', // Unaligned output
+      '-c', query,
+    ]
+
+    const proc = spawn(psqlPath!, args, {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      env: { ...process.env, PGPASSWORD: 'quest' },
+    })
+
+    let stdout = ''
+    let stderr = ''
+
+    proc.stdout.on('data', (data: Buffer) => {
+      stdout += data.toString()
+    })
+    proc.stderr.on('data', (data: Buffer) => {
+      stderr += data.toString()
+    })
+
+    proc.on('close', (code) => {
+      if (code === 0) {
+        resolve(stdout.trim())
+      } else {
+        reject(new Error(`psql error: ${stderr || `exit code ${code}`}`))
+      }
+    })
+    proc.on('error', (err) => {
+      reject(new Error(`Failed to execute psql: ${err.message}`))
+    })
+  })
+}
+
+/**
+ * Create a backup of a QuestDB database
+ */
+export async function createBackup(
+  container: ContainerConfig,
+  outputPath: string,
+  options: BackupOptions,
+): Promise<BackupResult> {
+  const { port } = container
+  const database = options.database || container.database || 'qdb'
+
+  const lines: string[] = []
+  lines.push('-- QuestDB backup generated by SpinDB')
+  lines.push(`-- Database: ${database}`)
+  lines.push(`-- Date: ${new Date().toISOString()}`)
+  lines.push('')
+
+  try {
+    // Get list of tables
+    const tablesQuery = `SELECT table_name FROM tables() WHERE table_name NOT LIKE 'sys.%'`
+    const tablesResult = await executeQuery(port, database, tablesQuery)
+    const tables = tablesResult.split('\n').filter((t) => t.trim())
+
+    logDebug(`Found ${tables.length} tables in database ${database}`)
+
+    for (const table of tables) {
+      if (!table.trim()) continue
+
+      lines.push(`-- Table: ${table}`)
+      lines.push('')
+
+      // Get CREATE TABLE statement
+      try {
+        const createQuery = `SHOW CREATE TABLE "${table}"`
+        let createResult = await executeQuery(port, database, createQuery)
+        if (createResult) {
+          // QuestDB's SHOW CREATE TABLE uses single quotes for table names,
+          // but SQL requires double quotes for identifiers. Fix the quoting.
+          // Also, the output sometimes ends with ; already, avoid double ;;
+          createResult = createResult
+            .replace(/^CREATE TABLE '([^']+)'/, 'CREATE TABLE "$1"')
+            .replace(/;$/, '') // Remove trailing semicolon if present
+          lines.push(createResult + ';')
+          lines.push('')
+        }
+      } catch (error) {
+        logWarning(`Could not get CREATE TABLE for ${table}: ${error}`)
+        continue
+      }
+
+      // Export table data
+      try {
+        // Try to get column names using table_columns() function
+        // This ensures INSERT statements have explicit column names for reliability
+        let columns: string[] = []
+        let useExplicitColumns = false
+
+        try {
+          const columnsQuery = `SELECT column FROM table_columns('${table}')`
+          const columnsResult = await executeQuery(port, database, columnsQuery)
+          columns = columnsResult.split('\n').filter((c) => c.trim())
+          useExplicitColumns = columns.length > 0
+        } catch {
+          // table_columns() failed - will use SELECT * without explicit column names
+          logDebug(`Could not get columns for ${table}, using SELECT *`)
+        }
+
+        // Get the designated timestamp column (if any) for ordering
+        // QuestDB tables have a designated timestamp column that can have any name
+        let orderClause = ''
+        try {
+          const tsQuery = `SELECT designatedTimestamp FROM tables() WHERE table_name = '${table}'`
+          const tsResult = await executeQuery(port, database, tsQuery)
+          if (tsResult && tsResult.trim()) {
+            orderClause = ` ORDER BY "${tsResult.trim()}"`
+          }
+        } catch {
+          // No designated timestamp or query failed - export without ordering
+        }
+
+        // Build SELECT query - use explicit columns if available, otherwise SELECT *
+        const columnList = useExplicitColumns
+          ? columns.map((c) => `"${c}"`).join(', ')
+          : '*'
+        const dataQuery = `SELECT ${columnList} FROM "${table}"${orderClause}`
+        const dataResult = await executeQuery(port, database, dataQuery)
+
+        if (dataResult) {
+          const rows = dataResult.split('\n').filter((r) => r.trim())
+          lines.push(`-- Data for ${table}: ${rows.length} rows`)
+
+          // Generate INSERT statements
+          for (const row of rows) {
+            // Parse the pipe-delimited output and convert to INSERT
+            // Trim each value to handle Windows CRLF line endings
+            const values = row.split('|').map((v) => {
+              const trimmed = v.trim()
+              if (trimmed === '' || trimmed === 'null') return 'NULL'
+              // Check if value looks like a number (int or float)
+              if (/^-?\d+(\.\d+)?$/.test(trimmed)) {
+                return trimmed // Don't quote numbers
+              }
+              // Escape single quotes and wrap strings
+              return `'${trimmed.replace(/'/g, "''")}'`
+            })
+
+            // Use explicit column names if available, otherwise positional VALUES
+            if (useExplicitColumns) {
+              lines.push(
+                `INSERT INTO "${table}" (${columnList}) VALUES (${values.join(', ')});`,
+              )
+            } else {
+              lines.push(`INSERT INTO "${table}" VALUES (${values.join(', ')});`)
+            }
+          }
+          lines.push('')
+        }
+      } catch (error) {
+        logWarning(`Could not export data for ${table}: ${error}`)
+      }
+    }
+
+    // Write backup file
+    const content = lines.join('\n')
+    await writeFile(outputPath, content, 'utf-8')
+
+    // Get file size
+    const stats = await stat(outputPath)
+
+    return {
+      path: outputPath,
+      format: 'sql',
+      size: stats.size,
+    }
+  } catch (error) {
+    throw new Error(
+      `Failed to create QuestDB backup: ${error instanceof Error ? error.message : String(error)}`,
+    )
+  }
+}