spindb 0.37.2 → 0.38.0

package/engines/clickhouse/backup.ts

@@ -1,398 +0,0 @@
-/**
- * ClickHouse backup module
- * Supports SQL-based backups using clickhouse client
- *
- * ClickHouse backup formats:
- * - SQL: DDL + INSERT statements (portable, human-readable)
- * - Native: ClickHouse native format (faster, compact)
- */
-
-import { spawn } from 'child_process'
-import { stat, mkdir, writeFile } from 'fs/promises'
-import { existsSync, createWriteStream } from 'fs'
-import { dirname } from 'path'
-import { logDebug, logWarning } from '../../core/error-handler'
-import {
-  requireClickHousePath,
-  validateClickHouseIdentifier,
-  escapeClickHouseIdentifier,
-} from './cli-utils'
-import type { ContainerConfig, BackupOptions, BackupResult } from '../../types'
-
-/**
- * Execute a ClickHouse query and return the result
- */
-async function execClickHouseQuery(
-  clickhousePath: string,
-  port: number,
-  database: string,
-  query: string,
-): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const args = [
-      'client',
-      '--host',
-      '127.0.0.1',
-      '--port',
-      String(port),
-      '--database',
-      database,
-      '--query',
-      query,
-    ]
-
-    const proc = spawn(clickhousePath, args, {
-      stdio: ['pipe', 'pipe', 'pipe'],
-    })
-
-    let stdout = ''
-    let stderr = ''
-
-    proc.stdout.on('data', (data: Buffer) => {
-      stdout += data.toString()
-    })
-    proc.stderr.on('data', (data: Buffer) => {
-      stderr += data.toString()
-    })
-
-    proc.on('error', reject)
-
-    proc.on('close', (code) => {
-      if (code === 0 || code === null) {
-        resolve(stdout)
-      } else {
-        reject(
-          new Error(stderr || `clickhouse client exited with code ${code}`),
-        )
-      }
-    })
-  })
-}
-
-/**
- * Get list of tables in a database
- */
-async function getTables(
-  clickhousePath: string,
-  port: number,
-  database: string,
-): Promise<string[]> {
-  // Validate database identifier to prevent SQL injection
-  validateClickHouseIdentifier(database, 'database')
-  // Escape single quotes for use in string literal (WHERE clause)
-  const escapedDbLiteral = database.replace(/'/g, "''")
-
-  const result = await execClickHouseQuery(
-    clickhousePath,
-    port,
-    database,
-    `SELECT name FROM system.tables WHERE database = '${escapedDbLiteral}' ORDER BY name`,
-  )
-  return result
-    .trim()
-    .split('\n')
-    .filter((t) => t.trim())
-}
-
-/**
- * Get CREATE TABLE statement for a table
- * Returns portable SQL without database prefix for cross-database restore
- */
-async function getCreateTableStatement(
-  clickhousePath: string,
-  port: number,
-  database: string,
-  table: string,
-): Promise<string> {
-  // Validate and escape identifiers to prevent SQL injection
-  validateClickHouseIdentifier(database, 'database')
-  validateClickHouseIdentifier(table, 'table')
-  const escapedDb = escapeClickHouseIdentifier(database)
-  const escapedTable = escapeClickHouseIdentifier(table)
-
-  // Use TSVRaw format to get unescaped output (newlines as actual newlines, not \n)
-  const result = await execClickHouseQuery(
-    clickhousePath,
-    port,
-    database,
-    `SHOW CREATE TABLE ${escapedDb}.${escapedTable} FORMAT TSVRaw`,
-  )
-
-  // Strip database prefix from CREATE TABLE statement to make it portable
-  // e.g., "CREATE TABLE testdb.test_user" → "CREATE TABLE test_user"
-  // e.g., "CREATE TABLE `testdb`.`test_user`" → "CREATE TABLE `test_user`"
-  // This allows the backup to be restored to any target database
-  const createStmt = result.trim()
-  // Escape regex metacharacters in database name for safe interpolation
-  const escapedDatabase = database.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
-  // Match both quoted and unquoted database names: db.table or `db`.table or `db`.`table`
-  const dbPrefixPattern = new RegExp(
-    `(CREATE TABLE\\s+)\`?${escapedDatabase}\`?\\.`,
-    'i',
-  )
-  return createStmt.replace(dbPrefixPattern, '$1')
-}
-
-/**
- * Create a SQL backup (DDL + INSERT statements)
- */
-async function createSqlBackup(
-  container: ContainerConfig,
-  outputPath: string,
-  database: string,
-): Promise<BackupResult> {
-  const { port, version } = container
-
-  const clickhousePath = await requireClickHousePath(version)
-
-  const lines: string[] = []
-  lines.push('-- ClickHouse backup generated by SpinDB')
-  lines.push(`-- Date: ${new Date().toISOString()}`)
-  lines.push(`-- Database: ${database}`)
-  lines.push('')
-
-  // Get list of tables
-  const tables = await getTables(clickhousePath, port, database)
-  logDebug(`Found ${tables.length} tables to backup`)
-
-  for (const table of tables) {
-    lines.push(`-- Table: ${table}`)
-    lines.push('')
-
-    // Get CREATE TABLE statement
-    try {
-      const createStmt = await getCreateTableStatement(
-        clickhousePath,
-        port,
-        database,
-        table,
-      )
-      lines.push(createStmt + ';')
-      lines.push('')
-    } catch (error) {
-      logWarning(`Could not get CREATE TABLE for ${table}: ${error}`)
-      continue
-    }
-
-    // Export data using INSERT format
-    try {
-      // Validate and escape identifiers for SQL injection protection
-      // Note: table names from getTables are already validated, but escape for query safety
-      validateClickHouseIdentifier(table, 'table')
-      const escapedDb = escapeClickHouseIdentifier(database)
-      const escapedTable = escapeClickHouseIdentifier(table)
-
-      const data = await execClickHouseQuery(
-        clickhousePath,
-        port,
-        database,
-        `SELECT * FROM ${escapedDb}.${escapedTable} FORMAT SQLInsert`,
-      )
-      if (data.trim()) {
-        // SQLInsert format uses literal "table" as placeholder, replace ALL occurrences with actual table name
-        // Use global flag to handle multi-statement output
-        const insertData = data
-          .trim()
-          .replace(/INSERT INTO table \(/gi, `INSERT INTO ${escapedTable} (`)
-        lines.push(insertData)
-        lines.push('')
-      }
-    } catch (error) {
-      logWarning(`Could not export data for ${table}: ${error}`)
-    }
-  }
-
-  // Ensure output directory exists
-  const outputDir = dirname(outputPath)
-  if (!existsSync(outputDir)) {
-    await mkdir(outputDir, { recursive: true })
-  }
-
-  // Write to file
-  const content = lines.join('\n')
-  await writeFile(outputPath, content, 'utf-8')
-
-  const stats = await stat(outputPath)
-
-  return {
-    path: outputPath,
-    format: 'sql',
-    size: stats.size,
-  }
-}
-
-/**
- * Create a native format backup (faster, more compact)
- *
- * TODO: Enable native backup format when restore support is implemented.
- * Native format is ~10x faster and more compact than SQL, but requires
- * parsing the binary format for restore. SQL format is currently preferred
- * for portability and ease of restore via clickhouse client --multiquery.
- *
- * ARCHITECTURAL NOTE: The current implementation mixes text markers
- * (-- TABLE:, -- CREATE:, -- DATA:) with binary Native format data in
- * the same stream. Before enabling this feature, refactor to use a
- * structured container format for deterministic parsing during restore:
- * - Tar archive with separate files per table (schema.sql + data.native)
- * - Or length-prefixed binary sections
- * - Or separate metadata JSON file alongside pure binary data
- *
- * To enable: export this function, add 'native' to BACKUP_FORMATS in
- * config/backup-formats.ts, and implement native restore in restore.ts.
- *
- * @internal Kept for future native backup support
- */
-async function _createNativeBackup(
-  container: ContainerConfig,
-  outputPath: string,
-  database: string,
-): Promise<BackupResult> {
-  const { port, version } = container
-
-  const clickhousePath = await requireClickHousePath(version)
-
-  // Ensure output directory exists
-  const outputDir = dirname(outputPath)
-  if (!existsSync(outputDir)) {
-    await mkdir(outputDir, { recursive: true })
-  }
-
-  // Get list of tables
-  const tables = await getTables(clickhousePath, port, database)
-  logDebug(`Found ${tables.length} tables to backup`)
-
-  // Create a combined backup with all tables
-  const fileStream = createWriteStream(outputPath)
-
-  // Write header
-  const header = `-- ClickHouse native backup\n-- Database: ${database}\n-- Date: ${new Date().toISOString()}\n\n`
-  fileStream.write(header)
-
-  for (const table of tables) {
-    // Write table marker
-    fileStream.write(`\n-- TABLE: ${table}\n`)
-
-    // Get CREATE TABLE
-    try {
-      const createStmt = await getCreateTableStatement(
-        clickhousePath,
-        port,
-        database,
-        table,
-      )
-      fileStream.write(`-- CREATE:\n${createStmt};\n`)
-    } catch (error) {
-      logWarning(`Could not get CREATE TABLE for ${table}: ${error}`)
-      continue
-    }
-
-    // Export data in Native format (binary, fast)
-    fileStream.write(`-- DATA (Native format):\n`)
-
-    // Validate and escape identifiers for SQL injection protection
-    validateClickHouseIdentifier(table, 'table')
-    const escapedDb = escapeClickHouseIdentifier(database)
-    const escapedTable = escapeClickHouseIdentifier(table)
-
-    await new Promise<void>((resolve, reject) => {
-      const args = [
-        'client',
-        '--host',
-        '127.0.0.1',
-        '--port',
-        String(port),
-        '--database',
-        database,
-        '--query',
-        `SELECT * FROM ${escapedDb}.${escapedTable} FORMAT Native`,
-      ]
-
-      const proc = spawn(clickhousePath, args, {
-        stdio: ['ignore', 'pipe', 'pipe'],
-      })
-
-      let stderr = ''
-
-      proc.stdout.pipe(fileStream, { end: false })
-
-      proc.stderr.on('data', (data: Buffer) => {
-        stderr += data.toString()
-      })
-
-      proc.on('error', reject)
-
-      proc.on('close', (code) => {
-        if (code === 0 || code === null) {
-          resolve()
-        } else {
-          reject(
-            new Error(stderr || `clickhouse client exited with code ${code}`),
-          )
-        }
-      })
-    })
-
-    fileStream.write('\n-- END TABLE\n')
-  }
-
-  // Close file
-  await new Promise<void>((resolve, reject) => {
-    fileStream.end((err: Error | null) => {
-      if (err) reject(err)
-      else resolve()
-    })
-  })
-
-  const stats = await stat(outputPath)
-
-  return {
-    path: outputPath,
-    format: 'native',
-    size: stats.size,
-  }
-}
-
-/**
- * Create a backup
- *
- * Currently only supports SQL format (DDL + INSERT statements).
- * Native format support is planned but not yet implemented for restore.
- * See _createNativeBackup for the future native implementation.
- *
- * @param container - Container configuration
- * @param outputPath - Path to write backup file
- * @param options - Backup options (BackupOptions type)
- * @param options.format - Reserved for future use. Currently ignored; all backups
- *   use SQL format. When native format support is added, 'dump' will use
- *   _createNativeBackup for faster, more compact backups.
- */
-export async function createBackup(
-  container: ContainerConfig,
-  outputPath: string,
-  options: BackupOptions,
-): Promise<BackupResult> {
-  const database = options.database || container.database || 'default'
-
-  // Log when a non-SQL format is requested but not yet supported
-  if (options.format && options.format !== 'sql') {
-    logDebug(
-      `ClickHouse backup: format '${options.format}' requested but not yet supported. ` +
-        `Using SQL format instead. See _createNativeBackup for future native support.`,
-    )
-  }
-
-  // Currently only SQL format is supported
-  // Native format (_createNativeBackup) will be enabled when restore support is added
-  return createSqlBackup(container, outputPath, database)
-}
-
-/**
- * Create a backup for cloning purposes
- * Uses SQL format for reliability
- */
-export async function createCloneBackup(
-  container: ContainerConfig,
-  outputPath: string,
-): Promise<BackupResult> {
-  return createSqlBackup(container, outputPath, container.database || 'default')
-}

package/engines/clickhouse/binary-manager.ts

@@ -1,201 +0,0 @@
-/**
- * ClickHouse Binary Manager
- *
- * Handles downloading, extracting, and managing ClickHouse binaries from hostdb.
- * ClickHouse uses a single unified binary that handles server, client, and local modes.
- *
- * Extends BaseServerBinaryManager with ClickHouse-specific overrides:
- * - No Windows support (throws on Windows extraction)
- * - YY.MM version matching in verify()
- * - Flat archive handling (moves clickhouse binary to bin/)
- */
-
-import { existsSync } from 'fs'
-import { mkdir, readdir } from 'fs/promises'
-import { join } from 'path'
-import { exec } from 'child_process'
-import { promisify } from 'util'
-import { paths } from '../../config/paths'
-import { getBinaryUrl } from './binary-urls'
-import { normalizeVersion } from './version-maps'
-import { logDebug } from '../../core/error-handler'
-import { moveEntry } from '../../core/fs-error-utils'
-import {
-  BaseServerBinaryManager,
-  type ServerBinaryManagerConfig,
-} from '../../core/base-server-binary-manager'
-import {
-  Engine,
-  type Platform,
-  type Arch,
-  type ProgressCallback,
-} from '../../types'
-
-const execAsync = promisify(exec)
-
-class ClickHouseBinaryManager extends BaseServerBinaryManager {
-  protected readonly config: ServerBinaryManagerConfig = {
-    engine: Engine.ClickHouse,
-    engineName: 'clickhouse',
-    displayName: 'ClickHouse',
-    serverBinaryNames: ['clickhouse'],
-  }
-
-  protected getBinaryUrlFromModule(
-    version: string,
-    platform: Platform,
-    arch: Arch,
-  ): string {
-    return getBinaryUrl(version, platform, arch)
-  }
-
-  protected normalizeVersionFromModule(version: string): string {
-    return normalizeVersion(version)
-  }
-
-  protected parseVersionFromOutput(stdout: string): string | null {
-    // Extract version from output like "ClickHouse client version 25.12.3.21"
-    const match = stdout.match(/(\d+\.\d+\.\d+\.\d+)/)
-    return match?.[1] ?? null
-  }
-
-  /**
-   * ClickHouse does not have Windows binaries on hostdb.
-   * Override to throw a clear error.
-   */
-  protected override async extractWindowsBinaries(
-    _zipFile: string,
-    _binPath: string,
-    _tempDir: string,
-    _onProgress?: ProgressCallback,
-  ): Promise<void> {
-    throw new Error(
-      'ClickHouse binaries are not available for Windows. ' +
-        'ClickHouse is only supported on macOS and Linux.',
-    )
-  }
-
-  /**
-   * Override to handle ClickHouse's archive structure.
-   * ClickHouse archives may have a flat structure where the binary is at the root,
-   * not in a bin/ subdirectory. This moves the clickhouse binary to bin/.
-   */
-  protected override async moveExtractedEntries(
-    extractDir: string,
-    binPath: string,
-  ): Promise<void> {
-    const entries = await readdir(extractDir, { withFileTypes: true })
-
-    // Check for a clickhouse subdirectory
-    const clickhouseDir = entries.find(
-      (e) =>
-        e.isDirectory() &&
-        (e.name === 'clickhouse' || e.name.startsWith('clickhouse-')),
-    )
-
-    const sourceDir = clickhouseDir
-      ? join(extractDir, clickhouseDir.name)
-      : extractDir
-    const sourceEntries = clickhouseDir
-      ? await readdir(sourceDir, { withFileTypes: true })
-      : entries
-
-    // Check if source has a bin/ subdirectory
-    const hasBinDir = sourceEntries.some(
-      (e) => e.isDirectory() && e.name === 'bin',
-    )
-
-    if (hasBinDir) {
-      // Standard structure: move all entries as-is (preserves bin/ subdirectory)
-      for (const entry of sourceEntries) {
-        const sourcePath = join(sourceDir, entry.name)
-        const destPath = join(binPath, entry.name)
-        await moveEntry(sourcePath, destPath)
-      }
-    } else {
-      // Flat structure: create bin/ and move binaries there
-      const destBinDir = join(binPath, 'bin')
-      await mkdir(destBinDir, { recursive: true })
-
-      for (const entry of sourceEntries) {
-        const sourcePath = join(sourceDir, entry.name)
-        // Check if it's an executable (no extension on Unix, starts with 'clickhouse')
-        const isExecutable =
-          entry.isFile() &&
-          !entry.name.includes('.') &&
-          entry.name.startsWith('clickhouse')
-        const destPath = isExecutable
-          ? join(destBinDir, entry.name)
-          : join(binPath, entry.name)
-        await moveEntry(sourcePath, destPath)
-      }
-    }
-  }
-
-  /**
-   * Override verification for ClickHouse's YY.MM version format.
-   * ClickHouse uses `clickhouse client --version` and has 4-part versions (YY.MM.patch.build).
-   */
-  override async verify(
-    version: string,
-    platform: Platform,
-    arch: Arch,
-  ): Promise<boolean> {
-    const fullVersion = this.getFullVersion(version)
-    const binPath = paths.getBinaryPath({
-      engine: this.config.engineName,
-      version: fullVersion,
-      platform,
-      arch,
-    })
-
-    const clickhousePath = join(binPath, 'bin', 'clickhouse')
-
-    if (!existsSync(clickhousePath)) {
-      throw new Error(`ClickHouse binary not found at ${binPath}/bin/`)
-    }
-
-    try {
-      const { stdout, stderr } = await execAsync(
-        `"${clickhousePath}" client --version`,
-      )
-      // Log stderr if present (may contain benign warnings about config, etc.)
-      if (stderr && stderr.trim()) {
-        logDebug(
-          `clickhouse client stderr during version check: ${stderr.trim()}`,
-        )
-      }
-      const reportedVersion = this.parseVersionFromOutput(stdout)
-      if (!reportedVersion) {
-        throw new Error(`Could not parse version from: ${stdout.trim()}`)
-      }
-
-      // Check if major versions match (YY.MM format)
-      const expectedMajor = version.split('.').slice(0, 2).join('.')
-      const reportedMajor = reportedVersion.split('.').slice(0, 2).join('.')
-      if (expectedMajor === reportedMajor) {
-        return true
-      }
-
-      // Check if full versions match
-      if (reportedVersion === fullVersion) {
-        return true
-      }
-
-      throw new Error(
-        `Version mismatch: expected ${version}, got ${reportedVersion}`,
-      )
-    } catch (error) {
-      const err = error as Error & { stderr?: string; code?: number }
-      // Include stderr and exit code in error message for better debugging
-      const details = [err.message]
-      if (err.stderr) details.push(`stderr: ${err.stderr.trim()}`)
-      if (err.code !== undefined) details.push(`exit code: ${err.code}`)
-      throw new Error(
-        `Failed to verify ClickHouse binaries: ${details.join(', ')}`,
-      )
-    }
-  }
-}
-
-export const clickhouseBinaryManager = new ClickHouseBinaryManager()