spindb 0.34.5 → 0.35.3

package/core/hostdb-client.ts

@@ -1,15 +1,28 @@
 /**
  * Shared hostdb Client Module
  *
- * Provides centralized access to the hostdb repository at
- * https://github.com/robertjbass/hostdb
+ * Provides centralized access to pre-built database binaries.
+ * Primary registry: registry.layerbase.host
+ * Fallback registry: GitHub releases (robertjbass/hostdb)
  *
- * hostdb provides pre-built database binaries for multiple platforms.
  * This module handles fetching releases.json with caching to avoid
  * repeated network requests.
  */
 
 import { Platform, type Arch, type Engine } from '../types'
+import { logDebug } from './error-handler'
+
+// Registry base URLs
+export const LAYERBASE_REGISTRY_BASE = 'https://registry.layerbase.host'
+export const GITHUB_REGISTRY_BASE =
+  'https://github.com/robertjbass/hostdb/releases/download'
+
+/**
+ * Toggle GitHub fallback for binary downloads and releases.json fetches.
+ * Set to `false` to exclusively test the Layerbase registry.
+ * Must be re-enabled before release (see PRE_RELEASE_TASKS.md).
+ */
+export const ENABLE_GITHUB_FALLBACK = false
 
 // Platform definition in hostdb releases.json
 export type HostdbPlatform = {
@@ -59,9 +72,21 @@ let cachedReleases: HostdbReleasesData | null = null
 let cacheTimestamp = 0
 const CACHE_TTL_MS = 5 * 60 * 1000 // 5 minutes
 
-const HOSTDB_RELEASES_URL =
+export const LAYERBASE_RELEASES_URL =
+  'https://registry.layerbase.host/releases.json'
+export const GITHUB_RELEASES_URL =
   'https://raw.githubusercontent.com/robertjbass/hostdb/main/releases.json'
 
+/**
+ * Get the list of releases.json URLs to try, respecting ENABLE_GITHUB_FALLBACK.
+ * When fallback is disabled, only the Layerbase URL is returned.
+ */
+export function getReleasesUrls(): string[] {
+  return ENABLE_GITHUB_FALLBACK
+    ? [LAYERBASE_RELEASES_URL, GITHUB_RELEASES_URL]
+    : [LAYERBASE_RELEASES_URL]
+}
+
 /**
  * Fetch releases.json from hostdb repository with caching.
  *
@@ -74,27 +99,39 @@ export async function fetchHostdbReleases(): Promise<HostdbReleasesData> {
     return cachedReleases
   }
 
-  try {
-    const response = await fetch(HOSTDB_RELEASES_URL, {
-      signal: AbortSignal.timeout(5000),
-    })
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}: ${response.statusText}`)
-    }
+  // Try layerbase registry first, fall back to GitHub (if enabled)
+  const urls = getReleasesUrls()
+  for (let i = 0; i < urls.length; i++) {
+    const url = urls[i]
+    const isLast = i === urls.length - 1
+    try {
+      const response = await fetch(url, {
+        signal: AbortSignal.timeout(5000),
+      })
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`)
+      }
 
-    const data = (await response.json()) as HostdbReleasesData
+      const data = (await response.json()) as HostdbReleasesData
 
-    // Cache the results
-    cachedReleases = data
-    cacheTimestamp = Date.now()
+      // Cache the results
+      cachedReleases = data
+      cacheTimestamp = Date.now()
 
-    return data
-  } catch (error) {
-    const err = error as Error
-    // Log the failure and rethrow - caller decides whether to use fallback
-    console.warn(`Warning: Failed to fetch hostdb releases: ${err.message}`)
-    throw error
+      return data
+    } catch (error) {
+      const err = error as Error
+      logDebug(`Failed to fetch releases from ${url}: ${err.message}`)
+      // If this was the last URL, rethrow
+      if (isLast) {
+        throw error
+      }
+      // Otherwise try the next URL
+    }
   }
+
+  // Should be unreachable (loop always throws on last iteration)
+  throw new Error('Failed to fetch hostdb releases from all registries')
 }
 
 /**
@@ -175,7 +212,105 @@ export function buildHostdbUrl(
   const tag = `${engine}-${version}`
   const filename = `${engine}-${version}-${hostdbPlatform}.${extension}`
 
-  return `https://github.com/robertjbass/hostdb/releases/download/${tag}/${filename}`
+  return `${LAYERBASE_REGISTRY_BASE}/${tag}/${filename}`
+}
+
+/**
+ * Build a GitHub fallback URL for a hostdb release (same path scheme as layerbase).
+ */
+export function buildGithubFallbackUrl(
+  engine: Engine | string,
+  options: BuildHostdbUrlOptions,
+): string {
+  const { version, hostdbPlatform, extension = 'tar.gz' } = options
+  const tag = `${engine}-${version}`
+  const filename = `${engine}-${version}-${hostdbPlatform}.${extension}`
+
+  return `${GITHUB_REGISTRY_BASE}/${tag}/${filename}`
+}
+
+/**
+ * Convert a layerbase registry URL to its GitHub fallback equivalent.
+ * Returns null if the URL is not a layerbase URL or if GitHub fallback is disabled.
+ */
+export function getRegistryFallbackUrl(url: string): string | null {
+  if (!ENABLE_GITHUB_FALLBACK) return null
+  if (url.startsWith(LAYERBASE_REGISTRY_BASE)) {
+    return url.replace(LAYERBASE_REGISTRY_BASE, GITHUB_REGISTRY_BASE)
+  }
+  return null
+}
+
+/**
+ * Fetch wrapper that tries the primary URL first, then falls back to the
+ * GitHub registry if the primary is a layerbase URL and the request fails
+ * with a 404, 5xx, or network error.
+ *
+ * AbortError (timeout) is never retried — it propagates immediately.
+ */
+export async function fetchWithRegistryFallback(
+  url: string,
+  options?: RequestInit,
+): Promise<Response> {
+  try {
+    const response = await fetch(url, options)
+    if (response.status === 404 || response.status >= 500) {
+      const fallbackUrl = getRegistryFallbackUrl(url)
+      if (fallbackUrl) {
+        logDebug(
+          `Primary registry returned ${response.status}, trying GitHub fallback`,
+        )
+        return await fetch(fallbackUrl, options)
+      }
+    }
+    return response
+  } catch (error) {
+    const err = error as Error
+    // Never retry on timeout (AbortError)
+    if (err.name === 'AbortError') {
+      throw error
+    }
+    const fallbackUrl = getRegistryFallbackUrl(url)
+    if (fallbackUrl) {
+      logDebug(
+        `Primary registry fetch failed (${err.message}), trying GitHub fallback`,
+      )
+      return await fetch(fallbackUrl, options)
+    }
+    throw error
+  }
+}
+
+/**
+ * Try fetching from multiple registry URLs in order, returning the first
+ * successful (response.ok) Response. Logs per-URL failures via the
+ * supplied logger callback.
+ *
+ * @param urls - URLs to try in order (e.g., layerbase then GitHub)
+ * @param logger - Callback for logging per-URL failures
+ * @param timeoutMs - Per-request timeout in milliseconds (default: 5000)
+ * @returns The first successful Response
+ * @throws Error if all URLs fail
+ */
+export async function fetchFromRegistryUrls(
+  urls: string[],
+  logger: (message: string) => void,
+  timeoutMs: number = 5000,
+): Promise<Response> {
+  let lastError: Error | null = null
+  for (const url of urls) {
+    try {
+      const response = await fetch(url, {
+        signal: AbortSignal.timeout(timeoutMs),
+      })
+      if (response.ok) return response
+      logger(`Registry fetch from ${url}: HTTP ${response.status}`)
+    } catch (error) {
+      logger(`Registry fetch from ${url} failed: ${error}`)
+      lastError = error as Error
+    }
+  }
+  throw lastError ?? new Error('All release registries failed')
 }
 
 export type BuildDownloadUrlOptions = {

package/core/hostdb-metadata.ts

@@ -2,15 +2,19 @@
  * Fetches metadata from hostdb (databases.json and downloads.json)
  * to understand what tools each engine needs and how to install them.
  *
+ * Primary registry: registry.layerbase.host
+ * Fallback registry: GitHub raw (robertjbass/hostdb)
+ *
  * Architecture:
  * - databases.json: Lists server, client, utilities, and enhanced CLI tools for each engine
  * - downloads.json: Provides package manager commands for installing tools
  */
 
 import { logDebug } from './error-handler'
+import { LAYERBASE_REGISTRY_BASE } from './hostdb-client'
 import type { Engine } from '../types'
 
-const HOSTDB_RAW_BASE =
+const GITHUB_RAW_BASE =
   'https://raw.githubusercontent.com/robertjbass/hostdb/main'
 
 const CACHE_TTL_MS = 5 * 60 * 1000 // 5 minutes
@@ -22,12 +26,29 @@ type CliTools = {
   enhanced: string[]
 }
 
+export type VersionEntryObject = {
+  enabled?: boolean
+  platforms?: string[]
+  dependencies?: Array<{
+    database: string
+    cascadeDelete: boolean
+    note?: string
+  }>
+  cliTools?: CliTools
+}
+
 type DatabaseEntry = {
   displayName: string
   cliTools: CliTools
-  versions: Record<string, boolean> // version string -> available (true/false)
-  latestLts: string
-  // Other fields exist but we don't need them
+  versions: Record<string, boolean | VersionEntryObject>
+  platforms?: string[]
+  dependencies?: Array<{
+    database: string
+    cascadeDelete: boolean
+    note?: string
+  }>
+  spindbStatus?: string
+  hostedServiceAllowed?: boolean
 }
 
 type PackageManagerDef = {
@@ -69,7 +90,7 @@ let downloadsCache: { data: DownloadsJson; timestamp: number } | null = null
 const inFlightRequests = new Map<string, Promise<unknown>>()
 
 async function fetchWithCache<T>(
-  url: string,
+  urls: string[],
   getCache: () => { data: T; timestamp: number } | null,
   setCache: (cache: { data: T; timestamp: number }) => void,
 ): Promise<T> {
@@ -80,35 +101,48 @@ async function fetchWithCache<T>(
     return cache.data
   }
 
-  // Check for in-flight request to prevent duplicate fetches
-  const inFlight = inFlightRequests.get(url)
+  // Check for in-flight request using the primary URL as key
+  const cacheKey = urls[0]
+  const inFlight = inFlightRequests.get(cacheKey)
   if (inFlight) {
     return inFlight as Promise<T>
   }
 
-  // Create the fetch promise and store it
+  // Create the fetch promise — try each URL in order
   const fetchPromise = (async () => {
     try {
-      const response = await fetch(url)
-      if (!response.ok) {
-        throw new Error(`Failed to fetch ${url}: ${response.status}`)
+      let lastError: Error | null = null
+      for (const url of urls) {
+        try {
+          const response = await fetch(url)
+          if (!response.ok) {
+            throw new Error(`Failed to fetch ${url}: ${response.status}`)
+          }
+
+          const data = (await response.json()) as T
+          setCache({ data, timestamp: Date.now() })
+          return data
+        } catch (error) {
+          lastError = error as Error
+          logDebug(`Metadata fetch from ${url} failed: ${lastError.message}`)
+        }
       }
-
-      const data = (await response.json()) as T
-      setCache({ data, timestamp: Date.now() })
-      return data
+      throw lastError ?? new Error('All metadata URLs failed')
     } finally {
-      inFlightRequests.delete(url)
+      inFlightRequests.delete(cacheKey)
     }
   })()
 
-  inFlightRequests.set(url, fetchPromise)
+  inFlightRequests.set(cacheKey, fetchPromise)
   return fetchPromise
 }
 
 export async function fetchDatabasesJson(): Promise<DatabasesJson> {
   return fetchWithCache(
-    `${HOSTDB_RAW_BASE}/databases.json`,
+    [
+      `${LAYERBASE_REGISTRY_BASE}/databases.json`,
+      `${GITHUB_RAW_BASE}/databases.json`,
+    ],
     () => databasesCache,
     (c) => {
       databasesCache = c
@@ -118,7 +152,10 @@ export async function fetchDatabasesJson(): Promise<DatabasesJson> {
 
 export async function fetchDownloadsJson(): Promise<DownloadsJson> {
   return fetchWithCache(
-    `${HOSTDB_RAW_BASE}/downloads.json`,
+    [
+      `${LAYERBASE_REGISTRY_BASE}/downloads.json`,
+      `${GITHUB_RAW_BASE}/downloads.json`,
+    ],
     () => downloadsCache,
     (c) => {
       downloadsCache = c
@@ -275,6 +312,16 @@ export async function getPackagesForTools(
   }
 }
 
+/**
+ * Check if a version entry is enabled.
+ * Handles both old schema (boolean) and new schema (object with optional enabled field).
+ * Objects are enabled by default unless explicitly `{ enabled: false }`.
+ */
+export function isVersionEnabled(value: boolean | VersionEntryObject): boolean {
+  if (typeof value === 'boolean') return value
+  return value.enabled !== false
+}
+
 /**
  * Get available versions for a database engine from databases.json
  * This is the authoritative source for what versions are actually available in hostdb.
@@ -290,9 +337,8 @@ export async function getAvailableVersions(
     const entry = data[key]
     if (!entry?.versions) return null
 
-    // Return only versions marked as available (true)
     return Object.entries(entry.versions)
-      .filter(([, available]) => available)
+      .filter(([, value]) => isVersionEnabled(value))
       .map(([version]) => version)
   } catch (error) {
     logDebug('Failed to fetch available versions from hostdb', {
@@ -302,25 +348,3 @@ export async function getAvailableVersions(
     return null
   }
 }
-
-/**
- * Get the latest LTS version for a database engine
- * @param engine Engine (e.g., Engine.PostgreSQL or 'postgresql')
- * @returns Latest LTS version string, or null if not found
- */
-export async function getLatestLtsVersion(
-  engine: Engine | string,
-): Promise<string | null> {
-  try {
-    const data = await fetchDatabasesJson()
-    const key = engine.toLowerCase()
-    const entry = data[key]
-    return entry?.latestLts || null
-  } catch (error) {
-    logDebug('Failed to fetch latest LTS version from hostdb', {
-      engine,
-      error: error instanceof Error ? error.message : String(error),
-    })
-    return null
-  }
-}

package/engines/clickhouse/binary-urls.ts

@@ -35,7 +35,7 @@ export function getHostdbPlatform(
 /**
  * Build the download URL for ClickHouse binaries from hostdb
  *
- * Format: https://github.com/robertjbass/hostdb/releases/download/clickhouse-{version}/clickhouse-{version}-{platform}-{arch}.tar.gz
+ * Format: https://registry.layerbase.host/clickhouse-{version}/clickhouse-{version}-{platform}-{arch}.tar.gz
  *
  * @param version - ClickHouse version (e.g., '25.12', '25.12.3.21')
  * @param platform - Platform identifier (e.g., 'darwin', 'linux')

package/engines/cockroachdb/binary-urls.ts

@@ -1,19 +1,17 @@
 /**
  * CockroachDB binary URL generation
  *
- * Generates download URLs for CockroachDB binaries from hostdb.
+ * Generates download URLs for CockroachDB binaries from the layerbase registry.
  */
 
-import type { Platform, Arch } from '../../types'
+import { type Platform, type Arch, Engine } from '../../types'
 import { normalizeVersion } from './version-maps'
-
-const HOSTDB_BASE_URL =
-  'https://github.com/robertjbass/hostdb/releases/download'
+import { buildHostdbUrl } from '../../core/hostdb-client'
 
 /**
  * Get the binary download URL for a specific version and platform
  *
- * URL format: https://github.com/robertjbass/hostdb/releases/download/cockroachdb-{version}/cockroachdb-{version}-{platform}-{arch}.{ext}
+ * URL format: https://registry.layerbase.host/cockroachdb-{version}/cockroachdb-{version}-{platform}-{arch}.{ext}
  *
  * @param version - CockroachDB version (e.g., '25.4.2' or '25')
  * @param platform - Target platform (darwin, linux, win32)
@@ -27,7 +25,11 @@ export function getBinaryUrl(
   const fullVersion = normalizeVersion(version)
   const ext = platform === 'win32' ? 'zip' : 'tar.gz'
 
-  return `${HOSTDB_BASE_URL}/cockroachdb-${fullVersion}/cockroachdb-${fullVersion}-${platform}-${arch}.${ext}`
+  return buildHostdbUrl(Engine.CockroachDB, {
+    version: fullVersion,
+    hostdbPlatform: `${platform}-${arch}`,
+    extension: ext,
+  })
 }
 
 /**

package/engines/cockroachdb/hostdb-releases.ts

@@ -1,114 +1,26 @@
 /**
- * CockroachDB hostdb releases integration
+ * hostdb Releases Module for CockroachDB
  *
- * Fetches available versions from hostdb releases.json and provides
- * fallback to local version maps.
+ * Fetches CockroachDB binary information from the hostdb repository at
+ * https://github.com/robertjbass/hostdb
  */
 
-import { logDebug } from '../../core/error-handler'
+import { createHostdbReleases } from '../../core/hostdb-releases-factory'
 import {
   COCKROACHDB_VERSION_MAP,
   SUPPORTED_MAJOR_VERSIONS,
 } from './version-maps'
-
-const HOSTDB_RELEASES_URL =
-  'https://raw.githubusercontent.com/robertjbass/hostdb/main/releases.json'
-
-// Cache for fetched versions (expires after 5 minutes)
-let cachedVersions: Record<string, string[]> | null = null
-let cacheExpiry = 0
-const CACHE_TTL_MS = 5 * 60 * 1000
-
-type HostdbReleases = {
-  [engine: string]: {
-    versions: {
-      version: string
-      platforms: string[]
-    }[]
-  }
-}
-
-/**
- * Fetch available CockroachDB versions from hostdb
- * Returns a map of major version to available patch versions
- *
- * Falls back to local version maps if fetch fails
- */
-export async function fetchAvailableVersions(): Promise<
-  Record<string, string[]>
-> {
-  // Return cached versions if still valid
-  if (cachedVersions && Date.now() < cacheExpiry) {
-    return cachedVersions
-  }
-
-  try {
-    const response = await fetch(HOSTDB_RELEASES_URL)
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}`)
-    }
-
-    const releases = (await response.json()) as HostdbReleases
-    const cockroachdbReleases = releases.cockroachdb
-
-    if (!cockroachdbReleases?.versions) {
-      throw new Error('No CockroachDB versions found in releases.json')
-    }
-
-    // Group versions by major version (YY.MM format)
-    const versionMap: Record<string, string[]> = {}
-
-    for (const { version } of cockroachdbReleases.versions) {
-      // Extract major version (e.g., "25" from "25.4.2")
-      const majorMatch = version.match(/^(\d+)/)
-      if (!majorMatch) continue
-
-      const majorVersion = majorMatch[1]
-      if (!versionMap[majorVersion]) {
-        versionMap[majorVersion] = []
-      }
-      versionMap[majorVersion].push(version)
-    }
-
-    // Sort versions within each major version (newest first)
-    for (const major of Object.keys(versionMap)) {
-      versionMap[major].sort((a, b) => {
-        const partsA = a.split('.').map(Number)
-        const partsB = b.split('.').map(Number)
-        for (let i = 0; i < Math.max(partsA.length, partsB.length); i++) {
-          const diff = (partsB[i] || 0) - (partsA[i] || 0)
-          if (diff !== 0) return diff
-        }
-        return 0
-      })
-    }
-
-    // Cache the results
-    cachedVersions = versionMap
-    cacheExpiry = Date.now() + CACHE_TTL_MS
-
-    logDebug('Fetched CockroachDB versions from hostdb', { versionMap })
-    return versionMap
-  } catch (error) {
-    logDebug(`Failed to fetch hostdb releases: ${error}`)
-
-    // Fall back to local version maps
-    const fallbackMap: Record<string, string[]> = {}
-    for (const major of SUPPORTED_MAJOR_VERSIONS) {
-      const fullVersion = COCKROACHDB_VERSION_MAP[major]
-      if (fullVersion) {
-        fallbackMap[major] = [fullVersion]
-      }
-    }
-
-    return fallbackMap
-  }
-}
-
-/**
- * Clear the version cache (useful for testing)
- */
-export function clearVersionCache(): void {
-  cachedVersions = null
-  cacheExpiry = 0
-}
+import { cockroachdbBinaryManager } from './binary-manager'
+import { Engine } from '../../types'
+
+const hostdbReleases = createHostdbReleases({
+  engine: Engine.CockroachDB,
+  displayName: 'CockroachDB',
+  versionMap: COCKROACHDB_VERSION_MAP,
+  supportedMajorVersions: SUPPORTED_MAJOR_VERSIONS,
+  groupingStrategy: 'single-digit',
+  listInstalled: () => cockroachdbBinaryManager.listInstalled(),
+})
+
+export const fetchAvailableVersions = hostdbReleases.fetchAvailableVersions
+export const getLatestVersion = hostdbReleases.getLatestVersion

package/engines/cockroachdb/version-maps.ts

@@ -2,7 +2,7 @@
  * CockroachDB version mapping
  *
  * Maps short version aliases to full versions from hostdb releases.
- * MUST stay in sync with hostdb releases.json
+ * MUST stay in sync with hostdb databases.json
  */
 
 // Full version map for CockroachDB

package/engines/couchdb/binary-urls.ts

@@ -37,7 +37,7 @@ export function getHostdbPlatform(
 /**
  * Build the download URL for CouchDB binaries from hostdb
  *
- * Format: https://github.com/robertjbass/hostdb/releases/download/couchdb-{version}/couchdb-{version}-{platform}-{arch}.{ext}
+ * Format: https://registry.layerbase.host/couchdb-{version}/couchdb-{version}-{platform}-{arch}.{ext}
  *
  * @param version - CouchDB version (e.g., '3', '3.5.1')
  * @param platform - Platform identifier (e.g., 'darwin', 'linux', 'win32')

package/engines/duckdb/binary-urls.ts

@@ -12,7 +12,7 @@ import { Engine, type Platform, type Arch } from '../../types'
 /**
  * Build the download URL for DuckDB binaries from hostdb
  *
- * Format: https://github.com/robertjbass/hostdb/releases/download/duckdb-{version}/duckdb-{version}-{platform}-{arch}.{ext}
+ * Format: https://registry.layerbase.host/duckdb-{version}/duckdb-{version}-{platform}-{arch}.{ext}
  *
  * @param version - DuckDB version (e.g., '1', '1.4.3')
  * @param platform - Platform identifier (e.g., 'darwin', 'linux', 'win32')