spindb 0.37.2 → 0.38.0

package/engines/ferretdb/restore.ts

@@ -1,357 +0,0 @@
-/**
- * FerretDB restore module
- *
- * Restores backups using pg_restore or psql on the embedded PostgreSQL backend.
- */
-
-import { spawn, type SpawnOptions } from 'child_process'
-import { existsSync, statSync } from 'fs'
-import { open } from 'fs/promises'
-import { join } from 'path'
-import { logDebug, logWarning } from '../../core/error-handler'
-import { platformService } from '../../core/platform-service'
-import { ferretdbBinaryManager } from './binary-manager'
-import {
-  DEFAULT_DOCUMENTDB_VERSION,
-  DEFAULT_V1_POSTGRESQL_VERSION,
-  isV1,
-} from './version-maps'
-import type { ContainerConfig, BackupFormat, RestoreResult } from '../../types'
-
-/**
- * Resolve the path to a backend binary (pg_restore, psql, etc.)
- * Uses version-aware backend resolution (v1 = plain PostgreSQL, v2 = postgresql-documentdb)
- */
-function getBackendBinaryFullPath(
-  container: ContainerConfig,
-  binaryName: string,
-): string {
-  const { version, backendVersion } = container
-  const { platform, arch } = platformService.getPlatformInfo()
-  const v1 = isV1(version)
-
-  const effectiveBackendVersion = v1
-    ? backendVersion || DEFAULT_V1_POSTGRESQL_VERSION
-    : backendVersion || DEFAULT_DOCUMENTDB_VERSION
-
-  const backendPath = ferretdbBinaryManager.getBackendBinaryPath(
-    version,
-    effectiveBackendVersion,
-    platform,
-    arch,
-  )
-
-  const ext = platformService.getExecutableExtension()
-  return join(backendPath, 'bin', `${binaryName}${ext}`)
-}
-
-/**
- * Get the path to pg_restore from the backend installation
- */
-function getPgRestorePath(container: ContainerConfig): string {
-  return getBackendBinaryFullPath(container, 'pg_restore')
-}
-
-/**
- * Get the path to psql from the backend installation
- */
-function getPsqlPath(container: ContainerConfig): string {
-  return getBackendBinaryFullPath(container, 'psql')
-}
-
-/**
- * Detect the format of a backup file
- */
-export async function detectBackupFormat(
-  filePath: string,
-): Promise<BackupFormat> {
-  if (!existsSync(filePath)) {
-    throw new Error(`Backup file not found: ${filePath}`)
-  }
-
-  const stats = statSync(filePath)
-
-  if (stats.isDirectory()) {
-    return {
-      format: 'directory',
-      description: 'PostgreSQL directory format backup',
-      restoreCommand: `pg_restore -h 127.0.0.1 -p PORT -U postgres -d DATABASE ${filePath}`,
-    }
-  }
-
-  // Check file header to determine format
-  try {
-    const buffer = Buffer.alloc(256)
-    const fd = await open(filePath, 'r')
-    let bytesRead = 0
-    try {
-      const result = await fd.read(buffer, 0, 256, 0)
-      bytesRead = result.bytesRead
-    } finally {
-      await fd.close().catch(() => {})
-    }
-
-    // PostgreSQL custom format starts with "PGDMP"
-    const header = buffer.toString('ascii', 0, 5)
-    if (header === 'PGDMP') {
-      return {
-        format: 'custom',
-        description: 'PostgreSQL custom format backup',
-        restoreCommand: `pg_restore -h 127.0.0.1 -p PORT -U postgres -d DATABASE ${filePath}`,
-      }
-    }
-
-    // Check for SQL format using a larger buffer with word-boundary checks
-    // Strip BOM (Byte Order Mark) if present to avoid false negatives
-    const textHeader = buffer
-      .toString('utf8', 0, bytesRead)
-      .replace(/^\uFEFF/, '')
-      .toLowerCase()
-    const isSqlFormat =
-      textHeader.startsWith('--') ||
-      textHeader.startsWith('/*') ||
-      textHeader.includes('\n--') ||
-      textHeader.includes('\n/*') ||
-      /(?:^|\s)create\s/.test(textHeader) ||
-      /(?:^|\s)insert\s/.test(textHeader) ||
-      /(?:^|\s)drop\s/.test(textHeader) ||
-      textHeader.includes('pg_dump')
-    if (isSqlFormat) {
-      return {
-        format: 'sql',
-        description: 'Plain SQL backup',
-        restoreCommand: `psql -h 127.0.0.1 -p PORT -U postgres -d DATABASE -f ${filePath}`,
-      }
-    }
-
-    // Check file extension as fallback
-    if (filePath.endsWith('.dump')) {
-      return {
-        format: 'custom',
-        description: 'PostgreSQL custom format backup (by extension)',
-        restoreCommand: `pg_restore -h 127.0.0.1 -p PORT -U postgres -d DATABASE ${filePath}`,
-      }
-    }
-
-    if (filePath.endsWith('.sql')) {
-      return {
-        format: 'sql',
-        description: 'Plain SQL backup (by extension)',
-        restoreCommand: `psql -h 127.0.0.1 -p PORT -U postgres -d DATABASE -f ${filePath}`,
-      }
-    }
-
-    return {
-      format: 'unknown',
-      description: 'Unknown format - attempting as custom format',
-      restoreCommand: `pg_restore -h 127.0.0.1 -p PORT -U postgres -d DATABASE ${filePath}`,
-    }
-  } catch {
-    return {
-      format: 'unknown',
-      description: 'Could not detect format',
-      restoreCommand: `pg_restore -h 127.0.0.1 -p PORT -U postgres -d DATABASE ${filePath}`,
-    }
-  }
-}
-
-// Restore options
-export type RestoreOptions = {
-  database: string
-  drop?: boolean
-  timeoutMs?: number // Timeout in milliseconds (default: 5 minutes)
-}
-
-// Default timeout for restore operations (5 minutes)
-const DEFAULT_RESTORE_TIMEOUT_MS = 5 * 60 * 1000
-
-/**
- * Restore a backup to a FerretDB container
- *
- * Uses pg_restore for custom format backups, psql for SQL format
- */
-export async function restoreBackup(
-  container: ContainerConfig,
-  backupPath: string,
-  options: RestoreOptions,
-): Promise<RestoreResult> {
-  const { backendPort } = container
-  const {
-    database,
-    drop = true,
-    timeoutMs = DEFAULT_RESTORE_TIMEOUT_MS,
-  } = options
-
-  if (!backendPort) {
-    throw new Error(
-      'Backend port not set. Make sure the container is running before restoring.',
-    )
-  }
-
-  // Detect backup format
-  const format = await detectBackupFormat(backupPath)
-  logDebug(`Detected backup format: ${format.format}`)
-
-  // Choose restore tool based on format
-  const isSqlFormat = format.format === 'sql'
-  const toolPath = isSqlFormat
-    ? getPsqlPath(container)
-    : getPgRestorePath(container)
-
-  if (!existsSync(toolPath)) {
-    const toolName = isSqlFormat ? 'psql' : 'pg_restore'
-    const backendName = isV1(container.version)
-      ? 'PostgreSQL'
-      : 'postgresql-documentdb'
-    throw new Error(
-      `${toolName} not found at ${toolPath}. Make sure ${backendName} is installed.`,
-    )
-  }
-
-  const args: string[] = [
-    '-h',
-    '127.0.0.1',
-    '-p',
-    String(backendPort),
-    '-U',
-    'postgres',
-    '-d',
-    database,
-  ]
-
-  if (isSqlFormat) {
-    // psql: use -f flag for file input
-    args.push('-f', backupPath)
-    if (drop) {
-      logWarning(
-        'SQL format restore: --clean is not supported. ' +
-          'If you need to drop existing objects, ensure the SQL dump was created with pg_dump --clean.',
-      )
-    }
-  } else {
-    // pg_restore: add options for custom/directory format
-    if (drop) {
-      args.push('--clean', '--if-exists')
-    }
-    args.push(backupPath)
-  }
-
-  logDebug(
-    `Running ${isSqlFormat ? 'psql' : 'pg_restore'} with args: ${args.join(' ')}`,
-  )
-
-  const spawnOptions: SpawnOptions = {
-    stdio: ['pipe', 'pipe', 'pipe'],
-  }
-
-  return new Promise((resolve, reject) => {
-    const proc = spawn(toolPath, args, spawnOptions)
-
-    let stdout = ''
-    let stderr = ''
-    let finished = false
-    let timeoutId: ReturnType<typeof setTimeout> | null = null
-
-    // Helper to clean up and mark finished
-    const cleanup = () => {
-      if (timeoutId) {
-        clearTimeout(timeoutId)
-        timeoutId = null
-      }
-      finished = true
-    }
-
-    // Start timeout timer
-    timeoutId = setTimeout(() => {
-      if (finished) return
-      cleanup()
-      // Defensively kill the process - it may have already exited between
-      // the finished check and this call, so wrap in try-catch
-      try {
-        if (proc.exitCode === null && !proc.killed) {
-          proc.kill('SIGTERM')
-        }
-      } catch {
-        // Process already exited or otherwise not killable - ignore
-      }
-      reject(
-        new Error(
-          `Restore timed out after ${timeoutMs}ms. The ${isSqlFormat ? 'psql' : 'pg_restore'} process was killed.`,
-        ),
-      )
-    }, timeoutMs)
-
-    proc.stdout?.on('data', (data: Buffer) => {
-      stdout += data.toString()
-    })
-    proc.stderr?.on('data', (data: Buffer) => {
-      stderr += data.toString()
-    })
-
-    proc.on('error', (err) => {
-      // Immediately reject and mark finished to prevent race with close handler
-      if (finished) return
-      cleanup()
-      reject(err)
-    })
-
-    proc.on('close', (code) => {
-      if (finished) return
-      cleanup()
-
-      if (code === 0) {
-        resolve({
-          format: format.format,
-          stdout,
-          stderr,
-          code,
-        })
-      } else {
-        // pg_restore may exit with non-zero but still restore some data
-        // Only treat as warning-only if ALL non-empty lines match warning patterns
-        // This prevents real errors mixed with warnings from being suppressed
-        const stderrLines = stderr
-          .split('\n')
-          .map((line) => line.trim())
-          .filter((line) => line.length > 0)
-
-        const warningPatterns = [
-          /\balready exists\b/,
-          /^WARNING:/i,
-          /^pg_restore: warning:/i,
-        ]
-
-        const allLinesAreWarnings =
-          stderrLines.length > 0 &&
-          stderrLines.every((line) =>
-            warningPatterns.some((pattern) => pattern.test(line)),
-          )
-
-        // pg_restore outputs "errors ignored on restore: N" when it completes
-        // despite encountering errors - this means the restore finished
-        const pgRestoreCompletedWithIgnoredErrors =
-          /pg_restore: warning: errors ignored on restore: \d+/i.test(stderr)
-
-        const isWarningOnly =
-          allLinesAreWarnings || pgRestoreCompletedWithIgnoredErrors
-
-        if (isWarningOnly) {
-          logWarning(`Restore completed with warnings: ${stderr}`)
-          resolve({
-            format: format.format,
-            stdout,
-            stderr,
-            code: code ?? undefined,
-          })
-        } else {
-          reject(
-            new Error(
-              stderr ||
-                `${isSqlFormat ? 'psql' : 'pg_restore'} exited with code ${code}`,
-            ),
-          )
-        }
-      }
-    })
-  })
-}

package/engines/file-based-utils.ts

@@ -1,262 +0,0 @@
-/**
- * Centralized utilities for file-based engines (SQLite, DuckDB)
- *
- * This module is the single source of truth for:
- * - Extension → engine mapping
- * - Engine → valid extensions
- * - Engine → registry
- * - Container name derivation from filenames
- * - Scanning for unregistered files
- *
- * All file-based engine behavior should go through this module
- * so that adding a new file-based engine only requires changes here.
- */
-
-import { readdir } from 'fs/promises'
-import { existsSync } from 'fs'
-import { resolve, extname } from 'path'
-import { Engine, isFileBasedEngine } from '../types'
-import { sqliteRegistry } from './sqlite/registry'
-import { duckdbRegistry } from './duckdb/registry'
-
-// ============================================================
-// Extension Mapping
-// ============================================================
-
-/**
- * Map of file extensions to their corresponding file-based engine.
- * This is the single source of truth for extension → engine detection.
- */
-const EXTENSION_TO_ENGINE: Record<string, Engine> = {
-  '.sqlite': Engine.SQLite,
-  '.sqlite3': Engine.SQLite,
-  '.db': Engine.SQLite,
-  '.duckdb': Engine.DuckDB,
-  '.ddb': Engine.DuckDB,
-}
-
-/**
- * Valid extensions per engine, derived from the extension map.
- * Used for validation (e.g., ensuring a SQLite container only relocates to a SQLite extension).
- */
-const ENGINE_EXTENSIONS: Record<Engine.SQLite | Engine.DuckDB, string[]> = {
-  [Engine.SQLite]: ['.sqlite', '.sqlite3', '.db'],
-  [Engine.DuckDB]: ['.duckdb', '.ddb'],
-}
-
-/**
- * Extension regex per engine (for stripping extensions from filenames).
- */
-const ENGINE_EXTENSION_REGEX: Record<Engine.SQLite | Engine.DuckDB, RegExp> = {
-  [Engine.SQLite]: /\.(sqlite3?|db)$/i,
-  [Engine.DuckDB]: /\.(duckdb|ddb)$/i,
-}
-
-/**
- * Combined regex matching any file-based engine extension.
- */
-export const FILE_BASED_EXTENSION_REGEX = /\.(sqlite3?|db|duckdb|ddb)$/i
-
-/**
- * Detect which file-based engine a file belongs to based on its extension.
- * Returns null if the extension is not recognized as a file-based database.
- */
-export function detectEngineFromPath(filePath: string): Engine | null {
-  const ext = extname(filePath).toLowerCase()
-  return EXTENSION_TO_ENGINE[ext] ?? null
-}
-
-/**
- * Get the valid file extensions for a file-based engine.
- */
-export function getExtensionsForEngine(
-  engine: Engine.SQLite | Engine.DuckDB,
-): string[] {
-  return ENGINE_EXTENSIONS[engine]
-}
-
-/**
- * Get all valid file-based database extensions.
- */
-export function getAllFileBasedExtensions(): string[] {
-  return Object.keys(EXTENSION_TO_ENGINE)
-}
-
-/**
- * Check if a file path has a valid extension for the given engine.
- */
-export function isValidExtensionForEngine(
-  filePath: string,
-  engine: Engine.SQLite | Engine.DuckDB,
-): boolean {
-  const ext = extname(filePath).toLowerCase()
-  return ENGINE_EXTENSIONS[engine].includes(ext)
-}
-
-/**
- * Human-readable string of valid extensions for a file-based engine.
- */
-export function formatExtensionsForEngine(
-  engine: Engine.SQLite | Engine.DuckDB,
-): string {
-  return ENGINE_EXTENSIONS[engine].join(', ')
-}
-
-/**
- * Human-readable string of all valid file-based extensions.
- */
-export function formatAllExtensions(): string {
-  return Object.keys(EXTENSION_TO_ENGINE).join(', ')
-}
-
-// ============================================================
-// Registry Access
-// ============================================================
-
-/**
- * Common interface for file-based engine registries.
- * Both SQLite and DuckDB registries implement this shape.
- */
-export type FileBasedRegistry = {
-  add(entry: { name: string; filePath: string; created: string }): Promise<void>
-  get(name: string): Promise<{ name: string; filePath: string } | null>
-  remove(name: string): Promise<boolean>
-  update(
-    name: string,
-    updates: { filePath?: string; lastVerified?: string },
-  ): Promise<boolean>
-  exists(name: string): Promise<boolean>
-  isPathRegistered(filePath: string): Promise<boolean>
-  getByPath(
-    filePath: string,
-  ): Promise<{ name: string; filePath: string } | null>
-  isFolderIgnored(folderPath: string): Promise<boolean>
-  addIgnoreFolder(folderPath: string): Promise<void>
-  removeIgnoreFolder(folderPath: string): Promise<boolean>
-  listIgnoredFolders(): Promise<string[]>
-}
-
-/**
- * Get the registry for a file-based engine.
- * Throws if the engine is not file-based.
- */
-export function getRegistryForEngine(engine: Engine): FileBasedRegistry {
-  switch (engine) {
-    case Engine.SQLite:
-      return sqliteRegistry
-    case Engine.DuckDB:
-      return duckdbRegistry
-    default:
-      if (isFileBasedEngine(engine)) {
-        throw new Error(
-          `File-based engine "${engine}" has no registry configured in getRegistryForEngine()`,
-        )
-      }
-      throw new Error(`"${engine}" is not a file-based engine`)
-  }
-}
-
-// ============================================================
-// Container Name Derivation
-// ============================================================
-
-/**
- * Derive a valid container name from a database filename.
- * Removes the engine-specific extension and sanitizes for use as a container name.
- *
- * - Must start with a letter
- * - Can contain letters, numbers, hyphens, underscores
- * - Falls back to engine-specific default if result is empty
- */
-export function deriveContainerName(
-  fileName: string,
-  engine: Engine.SQLite | Engine.DuckDB,
-): string {
-  const extensionRegex = ENGINE_EXTENSION_REGEX[engine]
-  const fallback = engine === Engine.SQLite ? 'sqlite-db' : 'duckdb-db'
-
-  // Remove extension
-  const base = fileName.replace(extensionRegex, '')
-
-  // If nothing remains after extension removal, return engine-specific fallback
-  const sanitizedBase = base
-    .replace(/[^a-zA-Z0-9_-]/g, '-')
-    .replace(/^-+|-+$/g, '')
-  if (!sanitizedBase) {
-    return fallback
-  }
-
-  // Convert to valid container name (alphanumeric, hyphens, underscores)
-  let name = base.replace(/[^a-zA-Z0-9_-]/g, '-')
-
-  // Ensure starts with letter
-  if (!/^[a-zA-Z]/.test(name)) {
-    name = 'db-' + name
-  }
-
-  // Remove consecutive hyphens
-  name = name.replace(/-+/g, '-')
-
-  // Trim leading/trailing hyphens
-  name = name.replace(/^-+|-+$/g, '')
-
-  return name || fallback
-}
-
-// ============================================================
-// File Scanning
-// ============================================================
-
-export type UnregisteredFile = {
-  fileName: string
-  absolutePath: string
-}
-
-/**
- * Scan a directory for unregistered file-based database files.
- *
- * @param engine The file-based engine to scan for
- * @param directory Directory to scan (defaults to CWD)
- * @returns Array of unregistered files
- */
-export async function scanForUnregisteredFiles(
-  engine: Engine.SQLite | Engine.DuckDB,
-  directory: string = process.cwd(),
-): Promise<UnregisteredFile[]> {
-  const absoluteDir = resolve(directory)
-  const registry = getRegistryForEngine(engine)
-  const extensionRegex = ENGINE_EXTENSION_REGEX[engine]
-
-  // Check if folder is ignored
-  if (await registry.isFolderIgnored(absoluteDir)) {
-    return []
-  }
-
-  // Check if directory exists
-  if (!existsSync(absoluteDir)) {
-    return []
-  }
-
-  try {
-    const entries = await readdir(absoluteDir, { withFileTypes: true })
-
-    const matchingFiles = entries
-      .filter((e) => e.isFile())
-      .filter((e) => extensionRegex.test(e.name))
-      .map((e) => ({
-        fileName: e.name,
-        absolutePath: resolve(absoluteDir, e.name),
-      }))
-
-    const unregistered: UnregisteredFile[] = []
-    for (const file of matchingFiles) {
-      if (!(await registry.isPathRegistered(file.absolutePath))) {
-        unregistered.push(file)
-      }
-    }
-
-    return unregistered
-  } catch {
-    return []
-  }
-}