spindb 0.31.4 → 0.33.1

package/engines/qdrant/index.ts

@@ -1,6 +1,6 @@
 import { spawn, type SpawnOptions } from 'child_process'
 import { createWriteStream, existsSync } from 'fs'
-import { mkdir, writeFile, readFile, unlink } from 'fs/promises'
+import { chmod, mkdir, writeFile, readFile, unlink } from 'fs/promises'
 import { join } from 'path'
 import { Readable } from 'stream'
 import { pipeline } from 'stream/promises'
@@ -9,7 +9,11 @@ import { paths } from '../../config/paths'
 import { getEngineDefaults } from '../../config/defaults'
 import { platformService, isWindows } from '../../core/platform-service'
 import { configManager } from '../../core/config-manager'
-import { logDebug, logWarning } from '../../core/error-handler'
+import {
+  logDebug,
+  logWarning,
+  assertValidUsername,
+} from '../../core/error-handler'
 import { processManager } from '../../core/process-manager'
 import { portManager } from '../../core/port-manager'
 import { qdrantBinaryManager } from './binary-manager'
@@ -39,6 +43,8 @@ import {
   type StatusResult,
   type QueryResult,
   type QueryOptions,
+  type CreateUserOptions,
+  type UserCredentials,
 } from '../../types'
 import { parseRESTAPIResult } from '../../core/query-parser'
 
@@ -1202,6 +1208,105 @@ export class QdrantEngine extends BaseEngine {
     // Return the container's configured database
     return [container.database]
   }
+
+  /**
+   * Create/update the global API key for Qdrant.
+   *
+   * Qdrant supports only a single global API key (set in config.yaml).
+   * Calling createUser multiple times will overwrite the previous key.
+   * The caller-provided username is stored in the credential file for
+   * bookkeeping but has no effect on Qdrant itself — authentication
+   * is solely via the api-key header.
+   */
+  async createUser(
+    container: ContainerConfig,
+    options: CreateUserOptions,
+  ): Promise<UserCredentials> {
+    const { username, password } = options
+    assertValidUsername(username)
+    const { port, name } = container
+
+    // Qdrant uses a single global API key in config.yaml.
+    // Read current config, set/replace api_key, write back, and restart.
+    const containerDir = paths.getContainerPath(name, { engine: ENGINE })
+    const configPath = join(containerDir, 'config.yaml')
+
+    const currentConfig = await readFile(configPath, 'utf-8')
+
+    // Parse YAML config line-by-line to find service section and api_key.
+    // Assumes 2-space indentation, no inline comments on api_key lines, and simple scalar values.
+    // This is a lightweight string-edit approach; use a YAML parser if those cases must be supported.
+    const lines = currentConfig.split('\n')
+    const serviceIdx = lines.findIndex((l) => /^service:/.test(l))
+
+    if (serviceIdx < 0) {
+      throw new Error('Could not find service section in Qdrant config')
+    }
+
+    // Scan the service section for existing api_key and last property line
+    let apiKeyIdx = -1
+    let lastServicePropIdx = serviceIdx
+    for (let i = serviceIdx + 1; i < lines.length; i++) {
+      if (/^\s+\S/.test(lines[i])) {
+        lastServicePropIdx = i
+        if (/^\s+api_key:/.test(lines[i])) {
+          apiKeyIdx = i
+        }
+      } else if (/^\S/.test(lines[i]) && lines[i].trim() !== '') {
+        break // Next top-level section
+      }
+    }
+
+    const yamlSafePassword = JSON.stringify(password)
+    if (apiKeyIdx >= 0) {
+      lines[apiKeyIdx] = `  api_key: ${yamlSafePassword}`
+    } else {
+      lines.splice(lastServicePropIdx + 1, 0, `  api_key: ${yamlSafePassword}`)
+    }
+
+    const updatedConfig = lines.join('\n')
+
+    // Validate the modified config is structurally sound before writing
+    // Check that service section and api_key line are present
+    const updatedLines = updatedConfig.split('\n')
+    const hasService = updatedLines.some((l) => /^service:/.test(l))
+    const hasApiKey = updatedLines.some((l) => /^\s+api_key:/.test(l))
+    if (!hasService || !hasApiKey) {
+      throw new Error(
+        'Failed to update Qdrant config: modified YAML is structurally invalid. ' +
+          'The service section or api_key entry is missing after modification.',
+      )
+    }
+
+    // Only restart if the container is currently running
+    const statusResult = await this.status(container)
+    if (statusResult.running) {
+      logWarning(
+        `Restarting Qdrant container "${name}" to apply API key change. ` +
+          'Active client connections will be disconnected.',
+      )
+      await this.stop(container)
+      await writeFile(configPath, updatedConfig)
+      await chmod(configPath, 0o600)
+      await this.start(container)
+    } else {
+      await writeFile(configPath, updatedConfig)
+      await chmod(configPath, 0o600)
+    }
+
+    logDebug(`Configured Qdrant global API key (credential label: ${username})`)
+
+    const connectionString = `http://127.0.0.1:${port}`
+
+    return {
+      username,
+      password: '',
+      connectionString,
+      engine: container.engine,
+      container: container.name,
+      apiKey: password,
+    }
+  }
 }
 
 export const qdrantEngine = new QdrantEngine()

package/engines/redis/index.ts

@@ -8,7 +8,11 @@ import { paths } from '../../config/paths'
 import { getEngineDefaults } from '../../config/defaults'
 import { platformService, isWindows } from '../../core/platform-service'
 import { configManager } from '../../core/config-manager'
-import { logDebug, logWarning } from '../../core/error-handler'
+import {
+  logDebug,
+  logWarning,
+  assertValidUsername,
+} from '../../core/error-handler'
 import { processManager } from '../../core/process-manager'
 import { redisBinaryManager } from './binary-manager'
 import { getBinaryUrl } from './binary-urls'
@@ -37,6 +41,8 @@ import {
   type StatusResult,
   type QueryResult,
   type QueryOptions,
+  type CreateUserOptions,
+  type UserCredentials,
 } from '../../types'
 import { parseRedisResult } from '../../core/query-parser'
 
@@ -232,6 +238,11 @@ dbfilename dump.rdb
 
 # Append Only File (disabled for local dev)
 appendonly no
+
+# Suppress ARM64 copy-on-write warning with Transparent Huge Pages.
+# Redis refuses to start on ARM64 with THP enabled unless this is set.
+# Safe for local development (SpinDB's use case).
+ignore-warnings ARM64-COW-BUG
 `
 }
 
@@ -658,18 +669,45 @@ export class RedisEngine extends BaseEngine {
               reject(new Error(portError))
               return
             }
-            reject(
-              new Error(
-                `Redis failed to start within timeout. Check logs at: ${logFile}`,
-              ),
-            )
+
+            // Include log content in error for CI debugging
+            let logContent = ''
+            try {
+              logContent = await readFile(logFile, 'utf-8')
+            } catch {
+              logContent = '(log file not found or empty)'
+            }
+
+            const errorDetails = [
+              'Redis failed to start within timeout.',
+              `Binary: ${redisServer}`,
+              `Log file: ${logFile}`,
+              `Log content:\n${logContent || '(empty)'}`,
+              stderr ? `Stderr:\n${stderr}` : '',
+              stdout ? `Stdout:\n${stdout}` : '',
+            ]
+              .filter(Boolean)
+              .join('\n')
+
+            reject(new Error(errorDetails))
           }
         } else {
-          reject(
-            new Error(
-              stderr || stdout || `redis-server exited with code ${code}`,
-            ),
-          )
+          // Include log content for non-zero exit codes too
+          let logContent = ''
+          try {
+            logContent = await readFile(logFile, 'utf-8')
+          } catch {
+            logContent = ''
+          }
+
+          const errorDetails = [
+            stderr || stdout || `redis-server exited with code ${code}`,
+            logContent ? `Log content:\n${logContent}` : '',
+          ]
+            .filter(Boolean)
+            .join('\n')
+
+          reject(new Error(errorDetails))
         }
       })
     })
@@ -679,7 +717,7 @@ export class RedisEngine extends BaseEngine {
   private async waitForReady(
     port: number,
     version: string,
-    timeoutMs = 30000,
+    timeoutMs = 60000,
   ): Promise<boolean> {
     const startTime = Date.now()
     const checkInterval = 500
@@ -1434,6 +1472,62 @@ export class RedisEngine extends BaseEngine {
     // Return the container's configured database
     return [container.database]
   }
+
+  async createUser(
+    container: ContainerConfig,
+    options: CreateUserOptions,
+  ): Promise<UserCredentials> {
+    const { username, password } = options
+    assertValidUsername(username)
+    const { port } = container
+    const db = options.database ?? container.database ?? '0'
+    const redisCli = await this.getRedisCliPath(container.version)
+
+    // Reject passwords with characters that break ACL SETUSER syntax:
+    // '>' sets password, '#' sets hash, '<' removes password — all are ACL delimiters.
+    // Whitespace and newlines would split the command unexpectedly.
+    if (/[>#<\s\n\r]/.test(password)) {
+      throw new Error(
+        'Password contains invalid characters for Redis ACL. Passwords must not contain ">", "#", "<", whitespace, or newlines.',
+      )
+    }
+
+    // ACL SETUSER is idempotent - sets user with full access
+    // Pass the full ACL command via stdin to avoid exposing the password in argv
+    const connArgs = ['-h', '127.0.0.1', '-p', String(port), '-n', db]
+
+    await new Promise<void>((resolve, reject) => {
+      const proc = spawn(redisCli, connArgs, {
+        stdio: ['pipe', 'pipe', 'pipe'],
+      })
+
+      let stderr = ''
+      proc.stderr?.on('data', (data: Buffer) => {
+        stderr += data.toString()
+      })
+
+      proc.stdin?.write(`ACL SETUSER ${username} on >${password} ~* &* +@all\n`)
+      proc.stdin?.end()
+
+      proc.on('close', (code) => {
+        if (code === 0) resolve()
+        else reject(new Error(`Failed to create user: ${stderr}`))
+      })
+      proc.on('error', reject)
+    })
+    logDebug(`Created Redis user: ${username}`)
+
+    const connectionString = `redis://${encodeURIComponent(username)}:${encodeURIComponent(password)}@127.0.0.1:${port}/${db}`
+
+    return {
+      username,
+      password,
+      connectionString,
+      engine: container.engine,
+      container: container.name,
+      database: db,
+    }
+  }
 }
 
 export const redisEngine = new RedisEngine()

package/engines/surrealdb/index.ts

@@ -22,7 +22,11 @@ import { paths } from '../../config/paths'
 import { getEngineDefaults } from '../../config/defaults'
 import { platformService } from '../../core/platform-service'
 import { configManager } from '../../core/config-manager'
-import { logDebug, logWarning } from '../../core/error-handler'
+import {
+  logDebug,
+  logWarning,
+  assertValidUsername,
+} from '../../core/error-handler'
 import { processManager } from '../../core/process-manager'
 import { surrealdbBinaryManager } from './binary-manager'
 import { getBinaryUrl } from './binary-urls'
@@ -51,6 +55,8 @@ import {
   type StatusResult,
   type QueryResult,
   type QueryOptions,
+  type CreateUserOptions,
+  type UserCredentials,
 } from '../../types'
 import { parseSurrealDBResult } from '../../core/query-parser'
 
@@ -394,7 +400,7 @@ export class SurrealDBEngine extends BaseEngine {
   private async waitForReady(
     port: number,
     version: string,
-    timeoutMs = 30000,
+    timeoutMs = 60000,
   ): Promise<boolean> {
     logDebug(`waitForReady called for port ${port}, version ${version}`)
     const startTime = Date.now()
@@ -1141,6 +1147,100 @@ export class SurrealDBEngine extends BaseEngine {
       })
     })
   }
+
+  async createUser(
+    container: ContainerConfig,
+    options: CreateUserOptions,
+  ): Promise<UserCredentials> {
+    const { username, password, database } = options
+    assertValidUsername(username)
+    const { port, version, name } = container
+    const namespace = name.replace(/-/g, '_')
+    const db = database || container.database || 'default'
+
+    const surreal = await this.getSurrealPath(version)
+    const containerDir = paths.getContainerPath(name, { engine: ENGINE })
+
+    // DEFINE USER OVERWRITE with EDITOR role (idempotent)
+    // Scope to database when options.database is provided, otherwise namespace-level
+    // Escape backslashes first, then single quotes for SurrealQL string literals
+    const escapedPass = password.replace(/\\/g, '\\\\').replace(/'/g, "\\'")
+    const scopeClause = database
+      ? `ON DATABASE ${escapeSurrealIdentifier(database)}`
+      : 'ON NAMESPACE'
+    const sql = `DEFINE USER OVERWRITE ${escapeSurrealIdentifier(username)} ${scopeClause} PASSWORD '${escapedPass}' ROLES EDITOR;`
+
+    const args = [
+      'sql',
+      '--endpoint',
+      `ws://127.0.0.1:${port}`,
+      '--user',
+      'root',
+      '--pass',
+      'root',
+      '--ns',
+      namespace,
+      '--db',
+      db,
+      '--hide-welcome',
+    ]
+
+    const timeoutMs = 15000
+    await new Promise<void>((resolve, reject) => {
+      const proc = spawn(surreal, args, {
+        stdio: ['pipe', 'pipe', 'pipe'],
+        cwd: containerDir,
+      })
+
+      let stderr = ''
+      let settled = false
+      const timeoutId = setTimeout(() => {
+        if (settled) return
+        settled = true
+        proc.kill()
+        reject(
+          new Error(
+            `Timed out creating SurrealDB user "${username}" after ${timeoutMs}ms`,
+          ),
+        )
+      }, timeoutMs)
+      proc.stderr?.on('data', (data: Buffer) => {
+        stderr += data.toString()
+      })
+
+      proc.stdin?.write(sql + '\n')
+      proc.stdin?.end()
+
+      proc.on('close', (code) => {
+        if (settled) return
+        settled = true
+        clearTimeout(timeoutId)
+        if (code === 0) {
+          logDebug(`Created SurrealDB user: ${username}`)
+          resolve()
+        } else {
+          reject(new Error(`Failed to create user: ${stderr}`))
+        }
+      })
+      proc.on('error', (error) => {
+        if (settled) return
+        settled = true
+        clearTimeout(timeoutId)
+        reject(error)
+      })
+    })
+
+    const connectionString = `ws://127.0.0.1:${port}/rpc`
+
+    return {
+      username,
+      password,
+      connectionString,
+      engine: container.engine,
+      container: container.name,
+      database: db,
+    }
+  }
 }
 
 export const surrealdbEngine = new SurrealDBEngine()

package/engines/typedb/backup.ts

@@ -0,0 +1,167 @@
+/**
+ * TypeDB backup module
+ *
+ * TypeDB exports databases as two files: schema (.typeql) and data (.typeql).
+ * We use the console's `database export` command which creates both files.
+ */
+
+import { spawn } from 'child_process'
+import { mkdir } from 'fs/promises'
+import { stat } from 'fs/promises'
+import { dirname } from 'path'
+import { logDebug } from '../../core/error-handler'
+import { requireTypeDBConsolePath, getConsoleBaseArgs } from './cli-utils'
+import type { ContainerConfig, BackupOptions, BackupResult } from '../../types'
+
+/**
+ * Create a TypeQL backup using typedb console export
+ *
+ * TypeDB export creates two files derived from outputPath:
+ * - {base}-schema.typeql (schema definitions)
+ * - {base}-data.typeql (data inserts)
+ *
+ * The returned BackupResult.path is the original outputPath (base path),
+ * NOT a single backup file. Callers (e.g., restore) must derive the actual
+ * file paths using the same `-schema.typeql` / `-data.typeql` convention.
+ * See also: restore.ts restoreTypeQLBackup() which mirrors this derivation.
+ */
+async function createTypeQLBackup(
+  container: ContainerConfig,
+  outputPath: string,
+  database: string,
+): Promise<BackupResult> {
+  const consolePath = await requireTypeDBConsolePath(container.version)
+  const { port } = container
+
+  // Ensure output directory exists
+  await mkdir(dirname(outputPath), { recursive: true })
+
+  // Derive schema and data paths from output path
+  const schemaPath = outputPath.endsWith('.typeql')
+    ? outputPath.replace(/\.typeql$/, '-schema.typeql')
+    : outputPath + '-schema.typeql'
+  const dataPath = outputPath.endsWith('.typeql')
+    ? outputPath.replace(/\.typeql$/, '-data.typeql')
+    : outputPath + '-data.typeql'
+
+  const args = [
+    ...getConsoleBaseArgs(port),
+    '--command',
+    `database export ${database} ${schemaPath} ${dataPath}`,
+  ]
+
+  const sanitizedArgs = args.map((a, i) =>
+    args[i - 1] === '--password' ? '***' : a,
+  )
+  logDebug(`Running: typedb_console_bin ${sanitizedArgs.join(' ')}`)
+
+  return new Promise<BackupResult>((resolve, reject) => {
+    const proc = spawn(consolePath, args, {
+      stdio: ['ignore', 'pipe', 'pipe'],
+    })
+
+    let stdout = ''
+    let stderr = ''
+
+    proc.stdout.on('data', (data: Buffer) => {
+      stdout += data.toString()
+    })
+    proc.stderr.on('data', (data: Buffer) => {
+      stderr += data.toString()
+    })
+
+    proc.on('error', reject)
+
+    proc.on('close', async (code) => {
+      if (code === 0) {
+        try {
+          // Calculate total size of both files (schema + data)
+          let schemaSize: number | null = null
+          let dataSize: number | null = null
+          const errors: string[] = []
+          try {
+            const schemaStats = await stat(schemaPath)
+            schemaSize = schemaStats.size
+          } catch (err) {
+            errors.push(`schema(${schemaPath}): ${err}`)
+          }
+          try {
+            const dataStats = await stat(dataPath)
+            dataSize = dataStats.size
+          } catch (err) {
+            errors.push(`data(${dataPath}): ${err}`)
+          }
+
+          if (
+            errors.length > 0 ||
+            schemaSize === null ||
+            dataSize === null ||
+            schemaSize === 0 ||
+            dataSize === 0
+          ) {
+            reject(
+              new Error(
+                `Backup produced empty or missing files: schema=${schemaPath}, data=${dataPath}` +
+                  (errors.length > 0
+                    ? `. Stat errors: ${errors.join('; ')}`
+                    : ''),
+              ),
+            )
+            return
+          }
+
+          // path is the base outputPath; actual files are schemaPath and dataPath
+          resolve({
+            path: outputPath,
+            format: 'typeql',
+            size: schemaSize + dataSize,
+          })
+        } catch (error) {
+          reject(new Error(`Backup files not created: ${error}`))
+        }
+      } else if (code === null) {
+        const detail = stderr || stdout
+        reject(
+          new Error(
+            `typedb console export was terminated by signal${detail ? `: ${detail}` : ''}`,
+          ),
+        )
+      } else {
+        const detail = stderr || stdout
+        reject(
+          new Error(
+            `typedb console export exited with code ${code}${detail ? `: ${detail}` : ''}`,
+          ),
+        )
+      }
+    })
+  })
+}
+
+/**
+ * Create a backup
+ *
+ * @param container - Container configuration
+ * @param outputPath - Path to write backup file
+ * @param options - Backup options
+ */
+export async function createBackup(
+  container: ContainerConfig,
+  outputPath: string,
+  options: BackupOptions,
+): Promise<BackupResult> {
+  const database = options.database || container.database
+
+  return createTypeQLBackup(container, outputPath, database)
+}
+
+/**
+ * Create a backup for cloning purposes
+ * Uses TypeQL format for reliability
+ */
+export async function createCloneBackup(
+  container: ContainerConfig,
+  outputPath: string,
+): Promise<BackupResult> {
+  return createTypeQLBackup(container, outputPath, container.database)
+}