spindb 0.37.2 → 0.38.0

package/engines/clickhouse/restore.ts

@@ -1,466 +0,0 @@
-/**
- * ClickHouse restore module
- * Supports SQL-based restores using clickhouse client
- */
-
-import { spawn } from 'child_process'
-import { open } from 'fs/promises'
-import { existsSync, statSync, createReadStream } from 'fs'
-import { logDebug, logWarning } from '../../core/error-handler'
-import {
-  requireClickHousePath,
-  validateClickHouseIdentifier,
-  escapeClickHouseIdentifier,
-} from './cli-utils'
-import type { BackupFormat, RestoreResult } from '../../types'
-
-/**
- * SQL keywords that indicate a ClickHouse SQL backup
- */
-const CLICKHOUSE_SQL_KEYWORDS = [
-  'CREATE TABLE',
-  'CREATE DATABASE',
-  'INSERT INTO',
-  'ALTER TABLE',
-  'DROP TABLE',
-  'SELECT',
-  'ATTACH',
-  'DETACH',
-]
-
-/**
- * Check if file content looks like ClickHouse SQL
- * Only reads first 8KB to avoid loading large files into memory
- */
-async function looksLikeClickHouseSql(filePath: string): Promise<boolean> {
-  try {
-    // Read only first 8KB - enough for several lines of SQL statements
-    // Using 8KB (vs 4KB for Redis/Valkey) since SQL statements can be longer
-    const HEADER_SIZE = 8192
-    const buffer = Buffer.alloc(HEADER_SIZE)
-
-    const fd = await open(filePath, 'r')
-    let bytesRead: number
-    try {
-      const result = await fd.read(buffer, 0, HEADER_SIZE, 0)
-      bytesRead = result.bytesRead
-    } finally {
-      await fd.close()
-    }
-
-    const content = buffer.toString('utf-8', 0, bytesRead)
-    const lines = content.split(/\r?\n/)
-
-    let sqlStatementsFound = 0
-    const linesToCheck = 20
-    let checkedLines = 0
-
-    for (const line of lines) {
-      // Stop after checking linesToCheck lines
-      if (checkedLines >= linesToCheck) break
-
-      const trimmed = line.trim().toUpperCase()
-
-      // Skip empty lines and comments (don't count toward linesToCheck)
-      if (!trimmed || trimmed.startsWith('--')) continue
-
-      checkedLines++
-
-      // Check for SQL keywords
-      for (const keyword of CLICKHOUSE_SQL_KEYWORDS) {
-        if (trimmed.startsWith(keyword)) {
-          sqlStatementsFound++
-          break
-        }
-      }
-
-      // Early success if we found 2 or more SQL statements
-      if (sqlStatementsFound >= 2) {
-        return true
-      }
-    }
-
-    return sqlStatementsFound > 0
-  } catch {
-    return false
-  }
-}
-
-/**
- * Detect backup format from file
- * Supports:
- * - SQL: DDL + INSERT statements
- */
-export async function detectBackupFormat(
-  filePath: string,
-): Promise<BackupFormat> {
-  if (!existsSync(filePath)) {
-    throw new Error(`Backup file not found: ${filePath}`)
-  }
-
-  const stats = statSync(filePath)
-
-  if (stats.isDirectory()) {
-    return {
-      format: 'unknown',
-      description: 'Directory found - ClickHouse restore expects a single file',
-      restoreCommand: 'ClickHouse requires a single .sql file for restore',
-    }
-  }
-
-  // Check file extension first for .sql files
-  if (filePath.endsWith('.sql')) {
-    return {
-      format: 'sql',
-      description: 'ClickHouse SQL backup',
-      restoreCommand:
-        'Execute SQL statements via clickhouse client (spindb restore handles this)',
-    }
-  }
-
-  // Content-based detection
-  if (await looksLikeClickHouseSql(filePath)) {
-    return {
-      format: 'sql',
-      description: 'ClickHouse SQL backup (detected by content)',
-      restoreCommand:
-        'Execute SQL statements via clickhouse client (spindb restore handles this)',
-    }
-  }
-
-  return {
-    format: 'unknown',
-    description: 'Unknown backup format',
-    restoreCommand: 'Use .sql file with ClickHouse SQL statements',
-  }
-}
-
-// Restore options for ClickHouse
-export type RestoreOptions = {
-  containerName: string
-  port: number
-  database?: string
-  version?: string
-  // Drop existing tables before restore
-  clean?: boolean
-}
-
-/**
- * Execute a ClickHouse query and return the result
- */
-async function executeQuery(
-  clickhousePath: string,
-  port: number,
-  database: string,
-  query: string,
-): Promise<string> {
-  return new Promise<string>((resolve, reject) => {
-    const args = [
-      'client',
-      '--host',
-      '127.0.0.1',
-      '--port',
-      String(port),
-      '--database',
-      database,
-      '--query',
-      query,
-    ]
-
-    const proc = spawn(clickhousePath, args, {
-      stdio: ['ignore', 'pipe', 'pipe'],
-    })
-
-    let stdout = ''
-    let stderr = ''
-
-    proc.stdout.on('data', (data: Buffer) => {
-      stdout += data.toString()
-    })
-
-    proc.stderr.on('data', (data: Buffer) => {
-      stderr += data.toString()
-    })
-
-    proc.on('close', (code) => {
-      if (code === 0) {
-        resolve(stdout.trim())
-      } else {
-        reject(new Error(`Query failed: ${stderr || `exit code ${code}`}`))
-      }
-    })
-
-    proc.on('error', reject)
-  })
-}
-
-/**
- * Get list of tables in a database
- */
-async function getTablesInDatabase(
-  clickhousePath: string,
-  port: number,
-  database: string,
-): Promise<string[]> {
-  try {
-    // Validate database name
-    validateClickHouseIdentifier(database, 'database')
-    const escapedDb = database.replace(/'/g, "''")
-
-    const result = await executeQuery(
-      clickhousePath,
-      port,
-      database,
-      `SELECT name FROM system.tables WHERE database = '${escapedDb}'`,
-    )
-
-    if (!result) {
-      return []
-    }
-
-    return result
-      .split('\n')
-      .map((line) => line.trim())
-      .filter(Boolean)
-  } catch (error) {
-    logDebug(`Failed to get tables: ${error}`)
-    return []
-  }
-}
-
-/**
- * Drop all tables in a database (for clean restore)
- */
-async function dropAllTables(
-  clickhousePath: string,
-  port: number,
-  database: string,
-): Promise<void> {
-  const tables = await getTablesInDatabase(clickhousePath, port, database)
-
-  if (tables.length === 0) {
-    logDebug('No existing tables to drop')
-    return
-  }
-
-  logDebug(
-    `Dropping ${tables.length} existing table(s) in database "${database}"`,
-  )
-
-  for (const table of tables) {
-    try {
-      // Validate table name
-      validateClickHouseIdentifier(table, 'table')
-      const escapedTable = escapeClickHouseIdentifier(table)
-
-      await executeQuery(
-        clickhousePath,
-        port,
-        database,
-        `DROP TABLE IF EXISTS ${escapedTable}`,
-      )
-      logDebug(`Dropped table: ${table}`)
-    } catch (error) {
-      // Log warning but continue with other tables
-      logWarning(`Failed to drop table "${table}": ${error}`)
-    }
-  }
-}
-
-/**
- * Restore from SQL backup
- * Streams SQL statements to clickhouse client
- */
-async function restoreSqlBackup(
-  backupPath: string,
-  port: number,
-  database: string,
-  version?: string,
-  clean: boolean = false,
-): Promise<RestoreResult> {
-  const clickhousePath = await requireClickHousePath(version)
-
-  // If clean mode, drop existing tables first
-  if (clean) {
-    logDebug('Clean mode: dropping existing tables before restore')
-    await dropAllTables(clickhousePath, port, database)
-  }
-
-  return new Promise<RestoreResult>((resolve, reject) => {
-    const args = [
-      'client',
-      '--host',
-      '127.0.0.1',
-      '--port',
-      String(port),
-      '--database',
-      database,
-      '--multiquery',
-    ]
-
-    const proc = spawn(clickhousePath, args, {
-      stdio: ['pipe', 'pipe', 'pipe'],
-    })
-
-    let stdout = ''
-    let stderr = ''
-    let streamError: Error | null = null
-
-    proc.stdout.on('data', (data: Buffer) => {
-      stdout += data.toString()
-    })
-
-    proc.stderr.on('data', (data: Buffer) => {
-      stderr += data.toString()
-    })
-
-    proc.on('close', (code) => {
-      // If there was a stream error, report it (include stderr for context)
-      if (streamError) {
-        const errorParts = [streamError.message]
-        if (stderr && stderr.trim()) {
-          errorParts.push(`ClickHouse stderr: ${stderr.trim()}`)
-        }
-        reject(new Error(errorParts.join('. ')))
-        return
-      }
-
-      if (code === 0) {
-        resolve({
-          format: 'sql',
-          stdout: stdout || 'SQL statements executed successfully',
-          stderr: stderr || undefined,
-          code: 0,
-        })
-      } else {
-        reject(
-          new Error(
-            `clickhouse client exited with code ${code}${stderr ? `: ${stderr}` : ''}`,
-          ),
-        )
-      }
-    })
-
-    proc.on('error', (error) => {
-      reject(new Error(`Failed to spawn clickhouse client: ${error.message}`))
-    })
-
-    // Stream backup file to clickhouse client stdin
-    const fileStream = createReadStream(backupPath, { encoding: 'utf-8' })
-
-    // Handle stdin errors (e.g., EPIPE when client exits early due to SQL error)
-    proc.stdin.on('error', (error: NodeJS.ErrnoException) => {
-      // EPIPE means the client closed its stdin (likely exited due to error)
-      // The actual error message will come from stderr in the 'close' handler
-      if (error.code === 'EPIPE') {
-        streamError = new Error(
-          'ClickHouse client closed connection early (likely SQL syntax error)',
-        )
-      } else {
-        streamError = new Error(
-          `Failed to write to clickhouse client: ${error.message}`,
-        )
-      }
-      fileStream.destroy()
-    })
-
-    fileStream.on('error', (error) => {
-      streamError = new Error(`Failed to read backup file: ${error.message}`)
-      fileStream.destroy()
-      proc.stdin.end()
-    })
-
-    fileStream.pipe(proc.stdin)
-  })
-}
-
-/**
- * Restore from backup
- * Supports:
- * - SQL: Execute statements via clickhouse client (ClickHouse must be running)
- */
-export async function restoreBackup(
-  backupPath: string,
-  options: RestoreOptions,
-): Promise<RestoreResult> {
-  const { port, database = 'default', version, clean = false } = options
-
-  if (!existsSync(backupPath)) {
-    throw new Error(`Backup file not found: ${backupPath}`)
-  }
-
-  // Detect backup format
-  const format = await detectBackupFormat(backupPath)
-  logDebug(`Detected backup format: ${format.format}`)
-
-  if (format.format === 'sql') {
-    return restoreSqlBackup(backupPath, port, database, version, clean)
-  }
-
-  throw new Error(
-    `Invalid backup format: ${format.format}. Use .sql file with ClickHouse SQL statements.`,
-  )
-}
-
-/**
- * Parse ClickHouse connection string
- * Format: clickhouse://[user:password@]host[:port][/database]
- */
-export function parseConnectionString(connectionString: string): {
-  host: string
-  port: number
-  database: string
-  user?: string
-  password?: string
-} {
-  if (!connectionString || typeof connectionString !== 'string') {
-    throw new Error(
-      'Invalid ClickHouse connection string: expected a non-empty string',
-    )
-  }
-
-  let url: URL
-  try {
-    url = new URL(connectionString)
-  } catch (error) {
-    // Mask credentials in error message if present
-    const sanitized = connectionString.replace(
-      /\/\/([^:]+):([^@]+)@/,
-      '//***:***@',
-    )
-    throw new Error(
-      `Invalid ClickHouse connection string: "${sanitized}". ` +
-        `Expected format: clickhouse://[user:password@]host[:port][/database]`,
-      { cause: error },
-    )
-  }
-
-  // Validate protocol
-  if (
-    url.protocol !== 'clickhouse:' &&
-    url.protocol !== 'http:' &&
-    url.protocol !== 'https:'
-  ) {
-    throw new Error(
-      `Invalid ClickHouse connection string: unsupported protocol "${url.protocol}". ` +
-        `Expected "clickhouse://", "http://", or "https://"`,
-    )
-  }
-
-  const host = url.hostname || '127.0.0.1'
-  // Default port depends on protocol - 9000 for native, 8123 for HTTP
-  const defaultPort =
-    url.protocol === 'http:' || url.protocol === 'https:' ? 8123 : 9000
-  const port = parseInt(url.port, 10) || defaultPort
-
-  // Database is in pathname
-  const database = url.pathname.replace(/^\//, '') || 'default'
-
-  return {
-    host,
-    port,
-    database,
-    user: url.username || undefined,
-    password: url.password || undefined,
-  }
-}

package/engines/clickhouse/version-maps.ts

@@ -1,95 +0,0 @@
-/**
- * ClickHouse Version Maps
- *
- * TEMPORARY: This version map will be replaced by the hostdb npm package once published.
- * Until then, manually keep this in sync with robertjbass/hostdb releases.json:
- * https://github.com/robertjbass/hostdb/blob/main/releases.json
- *
- * When updating versions:
- * 1. Check hostdb releases.json for available versions
- * 2. Update CLICKHOUSE_VERSION_MAP to match
- *
- * ClickHouse uses YY.MM.X.build versioning (e.g., 25.12.3.21)
- * We use the YY.MM format as the major version identifier.
- */
-
-import { logDebug } from '../../core/error-handler'
-
-/**
- * Map of ClickHouse versions to their latest stable patch versions.
- * Must match versions available in hostdb releases.json.
- */
-export const CLICKHOUSE_VERSION_MAP: Record<string, string> = {
-  // 1-part: year -> latest release for that year
-  '25': '25.12.3.21',
-  // 2-part: year.month -> latest
-  '25.12': '25.12.3.21',
-  // 3-part: year.month.patch -> latest build
-  '25.12.3': '25.12.3.21',
-  // 4-part: exact version (identity mapping)
-  '25.12.3.21': '25.12.3.21',
-}
-
-/**
- * Supported major ClickHouse versions (YY.MM format).
- * Used for grouping and display purposes.
- */
-export const SUPPORTED_MAJOR_VERSIONS = ['25.12']
-
-/**
- * Get the full version string for a version.
- *
- * @param version - Version (e.g., '25.12', '25.12.3', '25.12.3.21')
- * @returns Full version string (e.g., '25.12.3.21') or null if not supported
- */
-export function getFullVersion(version: string): string | null {
-  return CLICKHOUSE_VERSION_MAP[version] || null
-}
-
-/**
- * Normalize a version string to full version format.
- *
- * @param version - Version string (e.g., '25.12', '25.12.3', '25.12.3.21')
- * @returns Normalized version (e.g., '25.12.3.21')
- */
-export function normalizeVersion(version: string): string {
-  // If it's in the version map, return the mapped value
-  const fullVersion = CLICKHOUSE_VERSION_MAP[version]
-  if (fullVersion) {
-    return fullVersion
-  }
-
-  // Unknown version - warn and return as-is
-  const parts = version.split('.')
-
-  // ClickHouse versions can be 2-4 parts (YY.MM, YY.MM.X, YY.MM.X.build)
-  const isValidFormat =
-    parts.length >= 2 &&
-    parts.length <= 4 &&
-    parts.every((p) => /^\d+$/.test(p))
-
-  if (!isValidFormat) {
-    logDebug(
-      `ClickHouse version '${version}' has invalid format, may not be available in hostdb`,
-    )
-  } else {
-    logDebug(
-      `ClickHouse version '${version}' not in version map, may not be available in hostdb`,
-    )
-  }
-  return version
-}
-
-/**
- * Get the major version (YY.MM) from a full version string.
- *
- * @param version - Full version (e.g., '25.12.3.21')
- * @returns Major version (e.g., '25.12')
- */
-export function getMajorVersion(version: string): string {
-  const parts = version.split('.')
-  if (parts.length >= 2) {
-    return `${parts[0]}.${parts[1]}`
-  }
-  return version
-}

package/engines/clickhouse/version-validator.ts

@@ -1,154 +0,0 @@
-/**
- * ClickHouse version validation utilities
- * Handles version parsing, comparison, and compatibility checking
- *
- * ClickHouse uses YY.MM.X.build versioning (e.g., 25.12.3.21)
- * - YY: Year (2-digit)
- * - MM: Month
- * - X: Patch number
- * - build: Build number
- */
-
-/**
- * Parse a ClickHouse version string into components
- * Handles formats like "25.12.3.21", "25.12.3", "25.12", "v25.12.3.21"
- */
-export function parseVersion(versionString: string): {
-  year: number
-  month: number
-  patch: number
-  build: number
-  raw: string
-} | null {
-  const cleaned = versionString.replace(/^v/, '').trim()
-  const parts = cleaned.split('.')
-
-  if (parts.length < 2) return null
-
-  const year = parseInt(parts[0], 10)
-  const month = parseInt(parts[1], 10)
-  const patch = parts[2] ? parseInt(parts[2], 10) : 0
-  const build = parts[3] ? parseInt(parts[3], 10) : 0
-
-  if (isNaN(year) || isNaN(month)) return null
-  if (parts[2] && isNaN(patch)) return null
-  if (parts[3] && isNaN(build)) return null
-
-  return { year, month, patch, build, raw: cleaned }
-}
-
-/**
- * Check if a ClickHouse version is supported by SpinDB
- * Minimum supported version: 24.1.0.0
- */
-export function isVersionSupported(version: string): boolean {
-  const parsed = parseVersion(version)
-  if (!parsed) return false
-
-  // Support ClickHouse 24.x and newer
-  return parsed.year >= 24
-}
-
-/**
- * Get major version from full version string (YY.MM format)
- * e.g., "25.12.3.21" -> "25.12"
- */
-export function getMajorVersion(version: string): string {
-  const parsed = parseVersion(version)
-  if (!parsed) return version
-  return `${parsed.year}.${parsed.month}`
-}
-
-/**
- * Get major.minor.patch version from full version string
- * e.g., "25.12.3.21" -> "25.12.3"
- */
-export function getMajorMinorPatchVersion(version: string): string {
-  const parsed = parseVersion(version)
-  if (!parsed) return version
-  return `${parsed.year}.${parsed.month}.${parsed.patch}`
-}
-
-/**
- * Compare two ClickHouse versions
- * Returns: -1 if a < b, 0 if a == b, 1 if a > b, null if either version cannot be parsed
- */
-export function compareVersions(a: string, b: string): number | null {
-  const parsedA = parseVersion(a)
-  const parsedB = parseVersion(b)
-
-  if (!parsedA || !parsedB) {
-    return null
-  }
-
-  if (parsedA.year !== parsedB.year) {
-    return parsedA.year < parsedB.year ? -1 : 1
-  }
-  if (parsedA.month !== parsedB.month) {
-    return parsedA.month < parsedB.month ? -1 : 1
-  }
-  if (parsedA.patch !== parsedB.patch) {
-    return parsedA.patch < parsedB.patch ? -1 : 1
-  }
-  if (parsedA.build !== parsedB.build) {
-    return parsedA.build < parsedB.build ? -1 : 1
-  }
-  return 0
-}
-
-/**
- * Check if a backup version is compatible with the restore version
- * ClickHouse is generally forward-compatible for backups
- */
-export function isVersionCompatible(
-  backupVersion: string,
-  restoreVersion: string,
-): { compatible: boolean; warning?: string } {
-  const backup = parseVersion(backupVersion)
-  const restore = parseVersion(restoreVersion)
-
-  if (!backup || !restore) {
-    return {
-      compatible: true,
-      warning: 'Could not parse versions, proceeding with restore',
-    }
-  }
-
-  // Cannot restore backup from much newer version
-  // Calculate total months to handle year boundaries correctly
-  const backupMonths = backup.year * 12 + backup.month
-  const restoreMonths = restore.year * 12 + restore.month
-
-  if (backupMonths > restoreMonths + 6) {
-    // More than 6 months newer
-    return {
-      compatible: false,
-      warning: `Cannot restore ClickHouse ${backupVersion} backup to ${restoreVersion} server. The backup is from a much newer version.`,
-    }
-  }
-
-  // Allow same or close versions
-  if (backupMonths === restoreMonths) {
-    return { compatible: true }
-  }
-
-  // Allow upgrading from older version
-  if (restoreMonths > backupMonths) {
-    return {
-      compatible: true,
-      warning: `Restoring ClickHouse ${backupVersion} backup to ${restoreVersion} server. Schema may need updates.`,
-    }
-  }
-
-  // Restoring to slightly older version (within 6 months)
-  return {
-    compatible: true,
-    warning: `Restoring ClickHouse ${backupVersion} backup to older ${restoreVersion} server. Some features may not be available.`,
-  }
-}
-
-// Validate that a version string matches supported format
-export function isValidVersionFormat(version: string): boolean {
-  const parsed = parseVersion(version)
-  return parsed !== null
-}