spindb 0.37.2 → 0.38.0

package/core/hostdb-metadata.ts

@@ -1,350 +0,0 @@
-/**
- * Fetches metadata from hostdb (databases.json and downloads.json)
- * to understand what tools each engine needs and how to install them.
- *
- * Primary registry: registry.layerbase.host
- * Fallback registry: GitHub raw (robertjbass/hostdb)
- *
- * Architecture:
- * - databases.json: Lists server, client, utilities, and enhanced CLI tools for each engine
- * - downloads.json: Provides package manager commands for installing tools
- */
-
-import { logDebug } from './error-handler'
-import { LAYERBASE_REGISTRY_BASE } from './hostdb-client'
-import type { Engine } from '../types'
-
-const GITHUB_RAW_BASE =
-  'https://raw.githubusercontent.com/robertjbass/hostdb/main'
-
-const CACHE_TTL_MS = 5 * 60 * 1000 // 5 minutes
-
-type CliTools = {
-  server: string | null
-  client: string | null
-  utilities: string[]
-  enhanced: string[]
-}
-
-export type VersionEntryObject = {
-  enabled?: boolean
-  platforms?: string[]
-  dependencies?: Array<{
-    database: string
-    cascadeDelete: boolean
-    note?: string
-  }>
-  cliTools?: CliTools
-}
-
-type DatabaseEntry = {
-  displayName: string
-  cliTools: CliTools
-  versions: Record<string, boolean | VersionEntryObject>
-  platforms?: string[]
-  dependencies?: Array<{
-    database: string
-    cascadeDelete: boolean
-    note?: string
-  }>
-  spindbStatus?: string
-  hostedServiceAllowed?: boolean
-}
-
-type PackageManagerDef = {
-  name: string
-  platforms: string[]
-  installCmd: string
-  checkCmd: string
-}
-
-type ToolPackageInfo = {
-  package: string
-  tap?: string // Homebrew tap
-  repo?: string // apt repository
-}
-
-type ToolDownloadInfo = {
-  packages?: {
-    brew?: ToolPackageInfo
-    apt?: ToolPackageInfo
-    yum?: ToolPackageInfo
-    dnf?: ToolPackageInfo
-    choco?: ToolPackageInfo
-  }
-}
-
-// databases.json is keyed directly by engine name
-type DatabasesJson = Record<string, DatabaseEntry>
-
-type DownloadsJson = {
-  packageManagers: Record<string, PackageManagerDef>
-  tools: Record<string, ToolDownloadInfo>
-}
-
-// Simple in-memory cache
-let databasesCache: { data: DatabasesJson; timestamp: number } | null = null
-let downloadsCache: { data: DownloadsJson; timestamp: number } | null = null
-
-// In-flight request deduplication to prevent parallel fetches for the same URL
-const inFlightRequests = new Map<string, Promise<unknown>>()
-
-async function fetchWithCache<T>(
-  urls: string[],
-  getCache: () => { data: T; timestamp: number } | null,
-  setCache: (cache: { data: T; timestamp: number }) => void,
-): Promise<T> {
-  // Use getter to always check the freshest cache state
-  const cache = getCache()
-  const now = Date.now()
-  if (cache && now - cache.timestamp < CACHE_TTL_MS) {
-    return cache.data
-  }
-
-  // Check for in-flight request using the primary URL as key
-  const cacheKey = urls[0]
-  const inFlight = inFlightRequests.get(cacheKey)
-  if (inFlight) {
-    return inFlight as Promise<T>
-  }
-
-  // Create the fetch promise — try each URL in order
-  const fetchPromise = (async () => {
-    try {
-      let lastError: Error | null = null
-      for (const url of urls) {
-        try {
-          const response = await fetch(url)
-          if (!response.ok) {
-            throw new Error(`Failed to fetch ${url}: ${response.status}`)
-          }
-
-          const data = (await response.json()) as T
-          setCache({ data, timestamp: Date.now() })
-          return data
-        } catch (error) {
-          lastError = error as Error
-          logDebug(`Metadata fetch from ${url} failed: ${lastError.message}`)
-        }
-      }
-      throw lastError ?? new Error('All metadata URLs failed')
-    } finally {
-      inFlightRequests.delete(cacheKey)
-    }
-  })()
-
-  inFlightRequests.set(cacheKey, fetchPromise)
-  return fetchPromise
-}
-
-export async function fetchDatabasesJson(): Promise<DatabasesJson> {
-  return fetchWithCache(
-    [
-      `${LAYERBASE_REGISTRY_BASE}/databases.json`,
-      `${GITHUB_RAW_BASE}/databases.json`,
-    ],
-    () => databasesCache,
-    (c) => {
-      databasesCache = c
-    },
-  )
-}
-
-export async function fetchDownloadsJson(): Promise<DownloadsJson> {
-  return fetchWithCache(
-    [
-      `${LAYERBASE_REGISTRY_BASE}/downloads.json`,
-      `${GITHUB_RAW_BASE}/downloads.json`,
-    ],
-    () => downloadsCache,
-    (c) => {
-      downloadsCache = c
-    },
-  )
-}
-
-/**
- * Get the CLI tools definition for a database engine
- * @param engine Engine (e.g., Engine.PostgreSQL or 'postgresql')
- */
-export async function getDatabaseTools(
-  engine: Engine | string,
-): Promise<CliTools | null> {
-  try {
-    const data = await fetchDatabasesJson()
-    // hostdb uses lowercase engine names
-    const key = engine.toLowerCase()
-    const entry = data[key]
-    return entry?.cliTools || null
-  } catch (error) {
-    logDebug('Failed to fetch database tools from hostdb', {
-      engine,
-      error: error instanceof Error ? error.message : String(error),
-    })
-    return null
-  }
-}
-
-/**
- * Get the client tools required for a database engine
- * Returns an array of tool names (client + utilities)
- */
-export async function getRequiredClientTools(
-  engine: Engine | string,
-): Promise<string[]> {
-  const cliTools = await getDatabaseTools(engine)
-  if (!cliTools) return []
-
-  const required: string[] = []
-  if (cliTools.client) {
-    required.push(cliTools.client)
-  }
-  if (cliTools.utilities) {
-    required.push(...cliTools.utilities)
-  }
-  return required
-}
-
-/**
- * Get package manager info for a specific tool
- * @param tool Tool name (e.g., 'psql', 'mysqldump')
- */
-export async function getToolPackageInfo(
-  tool: string,
-): Promise<ToolDownloadInfo | null> {
-  try {
-    const data = await fetchDownloadsJson()
-    return data.tools[tool] || null
-  } catch (error) {
-    logDebug('Failed to fetch tool package info from hostdb', {
-      tool,
-      error: error instanceof Error ? error.message : String(error),
-    })
-    return null
-  }
-}
-
-/**
- * Get the install command for a tool using a specific package manager
- * @param tool Tool name (e.g., 'psql', 'pg_dump')
- * @param packageManager Package manager key (e.g., 'brew', 'apt')
- * @returns The full install command, or null if not available
- */
-export async function getInstallCommand(
-  tool: string,
-  packageManager: 'brew' | 'apt' | 'yum' | 'dnf' | 'choco',
-): Promise<string | null> {
-  try {
-    const data = await fetchDownloadsJson()
-
-    // Get the package manager definition
-    const pmDef = data.packageManagers[packageManager]
-    if (!pmDef) return null
-
-    // Get the tool's package info for this package manager
-    const toolInfo = data.tools[tool]
-    if (!toolInfo?.packages?.[packageManager]) return null
-
-    const pkgInfo = toolInfo.packages[packageManager]
-
-    // Build the command
-    let cmd = pmDef.installCmd.replace('{package}', pkgInfo.package)
-
-    // Handle Homebrew taps
-    if (packageManager === 'brew' && pkgInfo.tap) {
-      cmd = `brew tap ${pkgInfo.tap} && ${cmd}`
-    }
-
-    return cmd
-  } catch (error) {
-    logDebug('Failed to get install command from hostdb', {
-      tool,
-      packageManager,
-      error: error instanceof Error ? error.message : String(error),
-    })
-    return null
-  }
-}
-
-/**
- * Get the package to install that provides a set of tools
- * For many tools, a single package provides multiple binaries (e.g., postgresql provides psql, pg_dump, etc.)
- * This returns the unique packages needed to install all the requested tools
- */
-export async function getPackagesForTools(
-  tools: string[],
-  packageManager: 'brew' | 'apt' | 'yum' | 'dnf' | 'choco',
-): Promise<Array<{ package: string; tap?: string; tools: string[] }>> {
-  try {
-    const data = await fetchDownloadsJson()
-    const packageMap = new Map<string, { tap?: string; tools: string[] }>()
-
-    for (const tool of tools) {
-      const toolInfo = data.tools[tool]
-      if (!toolInfo?.packages?.[packageManager]) continue
-
-      const pkgInfo = toolInfo.packages[packageManager]
-      const key = pkgInfo.package
-
-      const existing = packageMap.get(key)
-      if (existing) {
-        existing.tools.push(tool)
-      } else {
-        packageMap.set(key, {
-          tap: pkgInfo.tap,
-          tools: [tool],
-        })
-      }
-    }
-
-    return Array.from(packageMap.entries()).map(([pkg, info]) => ({
-      package: pkg,
-      tap: info.tap,
-      tools: info.tools,
-    }))
-  } catch (error) {
-    logDebug('Failed to get packages for tools from hostdb', {
-      tools,
-      packageManager,
-      error: error instanceof Error ? error.message : String(error),
-    })
-    return []
-  }
-}
-
-/**
- * Check if a version entry is enabled.
- * Handles both old schema (boolean) and new schema (object with optional enabled field).
- * Objects are enabled by default unless explicitly `{ enabled: false }`.
- */
-export function isVersionEnabled(value: boolean | VersionEntryObject): boolean {
-  if (typeof value === 'boolean') return value
-  return value.enabled !== false
-}
-
-/**
- * Get available versions for a database engine from databases.json
- * This is the authoritative source for what versions are actually available in hostdb.
- * @param engine Engine (e.g., Engine.PostgreSQL or 'postgresql')
- * @returns Array of available version strings, or null if fetch fails
- */
-export async function getAvailableVersions(
-  engine: Engine | string,
-): Promise<string[] | null> {
-  try {
-    const data = await fetchDatabasesJson()
-    const key = engine.toLowerCase()
-    const entry = data[key]
-    if (!entry?.versions) return null
-
-    return Object.entries(entry.versions)
-      .filter(([, value]) => isVersionEnabled(value))
-      .map(([version]) => version)
-  } catch (error) {
-    logDebug('Failed to fetch available versions from hostdb', {
-      engine,
-      error: error instanceof Error ? error.message : String(error),
-    })
-    return null
-  }
-}

package/core/hostdb-releases-factory.ts

@@ -1,237 +0,0 @@
-/**
- * hostdb Releases Factory
- *
- * Creates standardized hostdb-releases modules for database engines.
- * This reduces code duplication across engines that follow the same pattern.
- *
- * The factory creates two functions:
- * - fetchAvailableVersions(): Fetches available versions grouped by major version
- * - getLatestVersion(major): Gets the latest version for a major version
- */
-
-import { compareVersions } from './version-utils'
-import { getAvailableVersions as getHostdbVersions } from './hostdb-metadata'
-import { logDebug } from './error-handler'
-import type { Engine, InstalledBinary } from '../types'
-
-/**
- * Grouping strategy for version numbers:
- * - 'single-digit': Group by first segment (e.g., 17.7.0 → 17)
- * - 'xy-format': Group by first two segments (e.g., 8.0.40 → 8.0)
- */
-export type GroupingStrategy = 'single-digit' | 'xy-format'
-
-/**
- * Configuration for creating hostdb-releases functions
- */
-export type HostdbReleasesConfig = {
-  /** Engine enum value */
-  engine: Engine
-  /** Display name for log messages */
-  displayName: string
-  /** Version map from major to full version */
-  versionMap: Record<string, string>
-  /** Supported major versions */
-  supportedMajorVersions: readonly string[]
-  /** Strategy for grouping versions by major version */
-  groupingStrategy: GroupingStrategy
-  /** Function to list installed binaries for offline fallback */
-  listInstalled: () => Promise<InstalledBinary[]>
-  /**
-   * Optional custom function to extract major version.
-   * Use this for engines with non-standard version grouping (e.g., ClickHouse's YY.MM,
-   * MySQL's conditional X.Y vs X grouping).
-   */
-  getMajorVersion?: (version: string) => string
-}
-
-/**
- * Return type for the factory
- */
-export type HostdbReleasesModule = {
-  fetchAvailableVersions: () => Promise<Record<string, string[]>>
-  getLatestVersion: (major: string) => Promise<string>
-}
-
-/**
- * Default function to extract major version based on grouping strategy
- */
-function defaultGetMajorVersion(
-  version: string,
-  strategy: GroupingStrategy,
-): string {
-  const trimmed = version?.trim()
-  if (!trimmed) {
-    return ''
-  }
-  const parts = trimmed.split('.')
-  if (strategy === 'xy-format') {
-    return parts.length >= 2 ? `${parts[0]}.${parts[1]}` : (parts[0] ?? '')
-  }
-  return parts[0] ?? ''
-}
-
-/**
- * Create hostdb-releases functions for an engine
- */
-export function createHostdbReleases(
-  config: HostdbReleasesConfig,
-): HostdbReleasesModule {
-  const {
-    engine,
-    displayName,
-    versionMap,
-    supportedMajorVersions,
-    groupingStrategy,
-    listInstalled,
-  } = config
-
-  // Use custom getMajorVersion if provided, otherwise use default
-  const getMajorVersion =
-    config.getMajorVersion ??
-    ((version: string) => defaultGetMajorVersion(version, groupingStrategy))
-
-  // Cache for fetchAvailableVersions to avoid repeated network requests
-  let cachedVersions: Record<string, string[]> | null = null
-  let cachedAt = 0
-  const cacheTTLMs = 30_000 // 30 seconds
-  let inflightFetchPromise: Promise<Record<string, string[]>> | null = null
-
-  /**
-   * Get hardcoded versions as last resort fallback
-   */
-  function getHardcodedVersions(): Record<string, string[]> {
-    const grouped: Record<string, string[]> = {}
-    for (const major of supportedMajorVersions) {
-      const version = versionMap[major]
-      if (version) {
-        grouped[major] = [version]
-      }
-    }
-    return grouped
-  }
-
-  /**
-   * Get available versions from hostdb, grouped by major version
-   */
-  async function fetchAvailableVersions(): Promise<Record<string, string[]>> {
-    // Try to fetch from hostdb databases.json (authoritative source)
-    try {
-      const versions = await getHostdbVersions(engine)
-
-      if (versions && versions.length > 0) {
-        // Group versions by major version
-        const grouped: Record<string, string[]> = {}
-
-        for (const version of versions) {
-          const major = getMajorVersion(version)
-          if (!grouped[major]) {
-            grouped[major] = []
-          }
-          grouped[major].push(version)
-        }
-
-        // Sort each group descending (latest first)
-        for (const major of Object.keys(grouped)) {
-          grouped[major].sort((a, b) => compareVersions(b, a))
-        }
-
-        return grouped
-      }
-    } catch (error) {
-      logDebug(
-        `Failed to fetch ${displayName} versions from hostdb, checking local`,
-        {
-          error: error instanceof Error ? error.message : String(error),
-        },
-      )
-    }
-
-    // Offline fallback: return only locally installed versions
-    const installed = await listInstalled()
-    if (installed.length > 0) {
-      const result: Record<string, string[]> = {}
-      for (const binary of installed) {
-        const major = getMajorVersion(binary.version)
-        if (!result[major]) {
-          result[major] = []
-        }
-        if (!result[major].includes(binary.version)) {
-          result[major].push(binary.version)
-        }
-      }
-      // Sort each major version group descending
-      for (const major of Object.keys(result)) {
-        result[major].sort((a, b) => compareVersions(b, a))
-      }
-      return result
-    }
-
-    // Last resort: return hardcoded version map
-    return getHardcodedVersions()
-  }
-
-  /**
-   * Get cached versions if fresh, otherwise fetch and cache.
-   * Uses inflightFetchPromise to dedupe concurrent requests.
-   */
-  async function getCachedVersions(): Promise<Record<string, string[]>> {
-    const now = Date.now()
-
-    // Return cached result if fresh
-    if (cachedVersions && now - cachedAt < cacheTTLMs) {
-      return cachedVersions
-    }
-
-    // If there's an inflight request, await it instead of starting a new one
-    if (inflightFetchPromise) {
-      return inflightFetchPromise
-    }
-
-    // Start new fetch, store promise to dedupe concurrent calls
-    inflightFetchPromise = fetchAvailableVersions()
-      .then((versions) => {
-        cachedVersions = versions
-        cachedAt = Date.now()
-        inflightFetchPromise = null
-        return versions
-      })
-      .catch((error) => {
-        // Clear inflight promise on error so subsequent calls can retry
-        inflightFetchPromise = null
-        throw error
-      })
-
-    return inflightFetchPromise
-  }
-
-  /**
-   * Get the latest version for a major version from hostdb.
-   * Uses cached fetchAvailableVersions result to avoid repeated network requests.
-   */
-  async function getLatestVersion(major: string): Promise<string> {
-    const versions = await getCachedVersions()
-    const majorVersions = versions[major]
-    if (majorVersions && majorVersions.length > 0) {
-      return majorVersions[0] // First is latest due to descending sort
-    }
-
-    // Fallback to version map
-    if (versionMap[major]) {
-      return versionMap[major]
-    }
-
-    // Generate default version based on grouping strategy
-    if (groupingStrategy === 'xy-format') {
-      // For X.Y format, add .0 to get X.Y.0
-      return `${major}.0`
-    }
-    // For single-digit, add .0.0 to get X.0.0
-    return `${major}.0.0`
-  }
-
-  return {
-    fetchAvailableVersions: getCachedVersions,
-    getLatestVersion,
-  }
-}

package/core/library-env.ts

@@ -1,118 +0,0 @@
-/**
- * Library environment utilities for dynamically-linked engine binaries.
- *
- * MariaDB, Redis, and Valkey hostdb binaries are linked against Homebrew's
- * OpenSSL at absolute paths (e.g. /opt/homebrew/opt/openssl@3/lib/libssl.3.dylib).
- * On systems without that library, they fail with cryptic dyld errors.
- *
- * This module provides:
- * - getLibraryEnv(): sets DYLD_FALLBACK_LIBRARY_PATH / LD_LIBRARY_PATH so
- *   the dynamic linker checks {binPath}/lib first (preparing for when hostdb
- *   bundles dylibs alongside binaries).
- * - detectLibraryError(): scans process output for library-loading patterns
- *   and returns an actionable error message.
- */
-
-import { platform as osPlatform } from 'os'
-import { join } from 'path'
-
-/**
- * Returns env vars that point the dynamic linker at {binPath}/lib.
- * On macOS: DYLD_FALLBACK_LIBRARY_PATH
- * On Linux: LD_LIBRARY_PATH
- * On Windows: returns undefined (not applicable).
- *
- * Usage: spread into spawn env: `{ ...process.env, ...getLibraryEnv(binPath) }`
- */
-export function getLibraryEnv(
-  binPath: string,
-): Record<string, string> | undefined {
-  const plat = osPlatform()
-  const libDir = join(binPath, 'lib')
-
-  if (plat === 'darwin') {
-    return { DYLD_FALLBACK_LIBRARY_PATH: libDir }
-  }
-  if (plat === 'linux') {
-    return { LD_LIBRARY_PATH: libDir }
-  }
-  return undefined
-}
-
-/**
- * Scans stderr/log output for dynamic library loading errors and returns
- * an actionable message, or null if no library error was detected.
- */
-export function detectLibraryError(
-  output: string,
-  engineName: string,
-): string | null {
-  if (!output) return null
-
-  const plat = osPlatform()
-  const lower = output.toLowerCase()
-
-  // macOS dyld errors
-  if (
-    lower.includes('library not loaded') ||
-    lower.includes('dyld:') ||
-    lower.includes('dyld[')
-  ) {
-    const needsOpenssl = lower.includes('libssl') || lower.includes('libcrypto')
-
-    if (needsOpenssl && plat === 'darwin') {
-      return (
-        `${engineName} failed to start: missing OpenSSL libraries.\n` +
-        `The downloaded binary requires OpenSSL 3 which is not installed.\n` +
-        `Fix: brew install openssl@3\n` +
-        `Alternatively, re-download binaries after hostdb ships relocatable builds.`
-      )
-    }
-
-    return (
-      `${engineName} failed to start: a required dynamic library could not be loaded.\n` +
-      `This typically means the hostdb binary was built against libraries not present on this system.\n` +
-      (plat === 'darwin'
-        ? `Try: brew install openssl@3\n`
-        : `Try: sudo apt-get install libssl-dev  (or the equivalent for your distro)\n`) +
-      `See: https://github.com/robertjbass/hostdb/issues`
-    )
-  }
-
-  // Linux GLIBC version errors
-  if (lower.includes('glibc') || lower.includes('libc.so')) {
-    return (
-      `${engineName} failed to start: incompatible system C library (GLIBC).\n` +
-      `The downloaded binary requires a newer GLIBC version than is installed.\n` +
-      `Options:\n` +
-      `  - Upgrade your OS to a newer version\n` +
-      `  - Use Docker: spindb can run inside containers with newer GLIBC\n` +
-      `See: https://github.com/robertjbass/hostdb/issues`
-    )
-  }
-
-  // Generic shared library errors on Linux
-  if (
-    lower.includes('error while loading shared libraries') ||
-    lower.includes('cannot open shared object file')
-  ) {
-    const needsOpenssl = lower.includes('libssl') || lower.includes('libcrypto')
-
-    if (needsOpenssl) {
-      return (
-        `${engineName} failed to start: missing OpenSSL libraries.\n` +
-        `Fix: sudo apt-get install libssl-dev  (Debian/Ubuntu)\n` +
-        `     sudo dnf install openssl-devel   (Fedora/RHEL)\n` +
-        `See: https://github.com/robertjbass/hostdb/issues`
-      )
-    }
-
-    return (
-      `${engineName} failed to start: a required shared library is missing.\n` +
-      `Check the error output above for the specific library name and install it.\n` +
-      `See: https://github.com/robertjbass/hostdb/issues`
-    )
-  }
-
-  return null
-}