vibecarbon 0.1.0

package/src/lib/deploy/compose/index.js

@@ -0,0 +1,1294 @@
+/**
+ * Docker Compose Production Deployment Module
+ *
+ * Deploys Vibecarbon projects to a single VPS using Docker Compose.
+ * Available to all tiers (Fast + Pro).
+ *
+ * Two provisioning modes:
+ * 1. Hetzner (automated): Uses HetznerProvider API to create VPS
+ * 2. Any VPS via SSH (--ssh user@ip): Skip provisioning, deploy to existing server
+ */
+
+import { spawn } from 'node:child_process';
+import { randomBytes } from 'node:crypto';
+import { existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as p from '@clack/prompts';
+import { uploadS3Backup } from '../../backup-s3.js';
+import { runCommand, runCommandAsync } from '../../command.js';
+import { perfAsync, perfTimer } from '../../perf.js';
+import { shEscape } from '../../shell.js';
+import { scpDownload } from '../../ssh.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// Path to the cloud-init YAML shipped with the CLI package. Referenced from
+// carbon/cloud-init/ (one level up from src/ in the installed tree).
+const CLOUD_INIT_PATH = join(__dirname, '../../../../carbon/cloud-init/docker-ce-setup.yaml');
+
+/**
+ * Read the cloud-init user-data YAML that runs at VPS boot to front-load
+ * ufw + unattended-upgrades installation. Callers pass the returned string
+ * as `userData` to HetznerProvider.createServer().
+ *
+ * Kept as a thin wrapper so tests can stub the read and so `vibecarbon
+ * upgrade` has a single place to notice layout changes.
+ */
+export function loadCloudInitScript() {
+  return readFileSync(CLOUD_INIT_PATH, 'utf-8');
+}
+
+// ServerAliveInterval+CountMax: force ssh to give up after 60s of no
+// SSH-protocol traffic, so a freshly-provisioned VPS that accepts TCP on 22
+// but never sends the SSH banner ("Connection timed out during banner
+// exchange") fails fast instead of hanging the caller indefinitely. Same
+// fix applied in lib/ssh.js — keep them in lockstep.
+const SSH_OPTS =
+  '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4';
+
+/**
+ * Run a command on a remote server via SSH
+ */
+function sshRun(ip, sshKeyPath, command, options = {}) {
+  const { timeout = 120_000 } = options;
+  return runCommand(['ssh', ...SSH_OPTS.split(' '), '-i', sshKeyPath, `root@${ip}`, command], {
+    stdio: 'pipe',
+    timeout,
+    encoding: 'utf-8',
+    ...options,
+  });
+}
+
+/**
+ * Async SSH runner — uses spawn (non-blocking) so spinners can animate.
+ * Retries on transient SSH connection errors (common on freshly provisioned servers
+ * where cloud-init may restart sshd).
+ */
+export async function sshRunAsync(ip, sshKeyPath, command, options = {}) {
+  const { timeout = 120_000, retries = 3, ignoreError, ...rest } = options;
+  const sshArgs = ['ssh', ...SSH_OPTS.split(' '), '-i', sshKeyPath, `root@${ip}`, command];
+  // Strip ignoreError from inner call so connection errors always throw and
+  // the retry loop can catch them. We apply ignoreError ourselves after retries.
+  const runOpts = { silent: true, timeout, ...rest };
+
+  for (let attempt = 0; attempt < retries; attempt++) {
+    try {
+      return await runCommandAsync(sshArgs, runOpts);
+    } catch (err) {
+      const msg = err.message || '';
+      const isConnectionError =
+        msg.includes('Connection reset') ||
+        msg.includes('Connection refused') ||
+        msg.includes('Connection closed') ||
+        msg.includes('kex_exchange_identification') ||
+        msg.includes('ssh_exchange_identification') ||
+        msg.includes('No route to host');
+      // Timeouts are transient too: a freshly-provisioned Hetzner VPS can
+      // have unattended-upgrades chewing CPU/disk in the background after
+      // cloud-init's marker file lands — SSH responds, but each command
+      // takes longer. Observed in iter-validate4 compose-ha restore where
+      // dockerLogin (a tiny stdin-script SSH call with a 30s timeout)
+      // hit the timeout twice in a row on the primary VPS without
+      // retry. The retry budget here (3 attempts × 5s backoff) is small
+      // enough that real hangs still surface fast, but absorbs the
+      // background-work blip on a healthy server.
+      const isTimeout =
+        msg.includes('timed out') ||
+        msg.includes('Command was killed with SIGTERM') ||
+        msg.includes('ETIMEDOUT');
+      const isTransient = isConnectionError || isTimeout;
+
+      if (isTransient && attempt < retries - 1) {
+        console.error(
+          `[retry] ssh ${ip}: attempt ${attempt + 1}/${retries} failed (${msg.slice(0, 80).split('\n')[0]}); retrying in 5s`,
+        );
+        await new Promise((r) => setTimeout(r, 5000));
+        continue;
+      }
+
+      // All retries exhausted — respect ignoreError if set
+      if (ignoreError) return null;
+      throw err;
+    }
+  }
+}
+
+/**
+ * Wait for SSH to become available on a newly provisioned server.
+ *
+ * Hetzner VPS typically accepts SSH ~20–35s after the create API returns.
+ * Early attempts use a 2s interval so we catch the edge quickly; after
+ * 10 tries (~20s) we fall back to 5s so we're not hammering the sshd
+ * during slow boots.
+ */
+export async function waitForSSH(host, sshKeyPath, maxAttempts = 30) {
+  for (let i = 0; i < maxAttempts; i++) {
+    try {
+      runCommand(
+        [
+          'ssh',
+          ...SSH_OPTS.split(' '),
+          '-i',
+          sshKeyPath,
+          '-o',
+          'ConnectTimeout=5',
+          `root@${host}`,
+          'echo',
+          'ok',
+        ],
+        { stdio: 'pipe', timeout: 10_000 },
+      );
+      return true;
+    } catch {
+      if (i === maxAttempts - 1) return false;
+      // Tighter than the previous [2s ×10, 5s ×20]: SSH is usually live by
+      // attempt 3-5, so the fast leg pays off most often. Tail keeps 5s
+      // pacing for the genuinely slow boots.
+      const interval = i < 5 ? 1000 : i < 10 ? 2000 : 5000;
+      await new Promise((r) => setTimeout(r, interval));
+    }
+  }
+  return false;
+}
+
+/**
+ * Wait for the boot-time cloud-init script to finish.
+ *
+ * The Hetzner VPS is created with `user_data` that installs ufw + unattended-
+ * upgrades and touches `/var/lib/vibecarbon/ready`. `waitForSSH` only tells us
+ * sshd is accepting connections — cloud-init may still be mid-way through its
+ * runcmd list. Polling for the marker keeps deploy's critical path tight:
+ * by the time SSH is up, cloud-init is usually 10-30s away from finishing, and
+ * this returns without any apt-get work on our side.
+ *
+ * If cloud-init fails (marker never appears within the deadline), dump the
+ * tail of /var/log/cloud-init-output.log into the error so diagnosis is a
+ * one-line log grep for the operator.
+ */
+export async function setupServer(ip, sshKeyPath) {
+  const deadline = Date.now() + 180_000; // 3 min — generous ceiling vs typical ~20-40s
+  let attempt = 0;
+  while (Date.now() < deadline) {
+    const result = await sshRunAsync(
+      ip,
+      sshKeyPath,
+      'test -f /var/lib/vibecarbon/ready && echo READY || echo NOT_READY',
+      { timeout: 10_000, ignoreError: true },
+    );
+    if (result?.trim()?.endsWith('READY') && !result.includes('NOT_READY')) return;
+    // Early attempts poll fast — cloud-init typically finishes in 20-40s.
+    // First 15 tries at 750ms catch the typical case ~10s sooner; back off
+    // to 5s afterward to avoid hammering an actually-slow boot.
+    const interval = attempt < 15 ? 750 : 5000;
+    attempt++;
+    await new Promise((r) => setTimeout(r, interval));
+  }
+
+  // Marker never appeared — surface cloud-init output so the operator can see why.
+  let tail = '';
+  try {
+    tail =
+      (await sshRunAsync(
+        ip,
+        sshKeyPath,
+        'tail -n 50 /var/log/cloud-init-output.log 2>/dev/null || echo "(cloud-init log unavailable)"',
+        { timeout: 15_000, ignoreError: true },
+      )) || '';
+  } catch {
+    // Best effort — don't mask the outer failure.
+  }
+  throw new Error(
+    `cloud-init never reached the ready marker on ${ip} within 180s.\n` +
+      `--- last 50 lines of /var/log/cloud-init-output.log ---\n${tail.trim()}\n` +
+      '--- end cloud-init tail ---',
+  );
+}
+
+/**
+ * Authenticate with a container registry on a remote server.
+ *
+ * Writes credentials directly into `/root/.docker/config.json` rather than
+ * invoking `docker login --password-stdin`. `docker login` has two moving
+ * parts that have caused silent failures in e2e:
+ *
+ *   1. SSH stdin-input race — on a freshly-booted VM, the shell can start
+ *      reading `docker login`'s prompt before our piped token arrives,
+ *      causing docker to hang 30s then exit non-zero.
+ *   2. Any `credsStore` / `credsHelpers` entry in `/root/.docker/config.json`
+ *      makes `docker login` try to delegate to a helper binary that doesn't
+ *      exist on a bare Hetzner VM — login exits 0 but config.json never
+ *      gets the new auth entry.
+ *
+ * The `auths` entry we write here is exactly what `docker login` produces
+ * internally (base64 of `username:token`, no encryption). Docker reads it
+ * on every pull/manifest-HEAD.
+ *
+ * Fails loud: if the write fails (unreachable server, permission denied,
+ * malformed JSON) we throw. A silent fallback here has burned us on
+ * standby GHCR 401s — a loud failure during login is much easier to debug
+ * than an `unauthorized` 20 minutes into reconcile.
+ */
+export async function dockerLoginOnServer(ip, sshKeyPath, creds) {
+  if (!creds?.username || !creds?.token) return;
+  const { username, token, registry } = creds;
+  const registryHost = registry || 'https://index.docker.io/v1/';
+  const authB64 = Buffer.from(`${username}:${token}`).toString('base64');
+
+  // Merge into any existing config.json via jq (preserves other registries'
+  // auth when we log in to multiple). jq is guaranteed on Hetzner docker-ce
+  // images; if not present we fall back to writing a minimal file.
+  // Bash here-doc writes a minimal config.json directly. No jq dependency:
+  // bare Hetzner images don't ship with it, and installing via apt-get races
+  // cloud-init's unattended-upgrades apt lock (observed 04:08 UTC —
+  // "Could not get lock /var/lib/apt/lists/lock").
+  //
+  // Python3 is preinstalled on Ubuntu's docker-ce cloud image, so we use
+  // it to merge cleanly into any existing config.json (preserves other
+  // registries' auth when logging into Docker Hub + GHCR sequentially).
+  // Falls back to overwriting config.json if python3 is absent.
+  const script = `
+set -e
+mkdir -p /root/.docker
+cfg=/root/.docker/config.json
+if [ ! -s "$cfg" ]; then echo '{}' > "$cfg"; fi
+
+if command -v python3 >/dev/null 2>&1; then
+  HOST=${shEscape(registryHost)} AUTH=${shEscape(authB64)} python3 - <<'PY'
+import json, os
+cfg_path = '/root/.docker/config.json'
+try:
+    with open(cfg_path) as f:
+        cfg = json.load(f)
+except Exception:
+    cfg = {}
+if not isinstance(cfg, dict):
+    cfg = {}
+cfg.setdefault('auths', {})[os.environ['HOST']] = {'auth': os.environ['AUTH']}
+cfg.pop('credsStore', None)
+cfg.pop('credHelpers', None)
+with open(cfg_path, 'w') as f:
+    json.dump(cfg, f)
+PY
+else
+  # Minimal fallback: overwrite with single-registry config. Acceptable
+  # for bare images where we're the only thing writing this file.
+  printf '{"auths":{"%s":{"auth":"%s"}}}' ${shEscape(registryHost)} ${shEscape(authB64)} > "$cfg"
+fi
+
+chmod 600 "$cfg"
+# Verify the auth entry landed. Deploys that silently lose auth here
+# surface as 401 during reconcile 15+ min later — this grep catches it
+# at login time so the failure is actionable.
+if ! grep -q ${shEscape(authB64)} "$cfg"; then
+  echo "docker auth write verification failed — config.json:" >&2
+  cat "$cfg" >&2
+  exit 1
+fi
+`;
+  await sshRunAsync(ip, sshKeyPath, `/bin/bash -s`, {
+    silent: true,
+    timeout: 30_000,
+    input: script,
+  });
+}
+
+/**
+ * Collect deploy files and sync them to the server.
+ * Supports pre-rendered bundlePath (Phase 1 Optimization).
+ */
+export async function setupServerFiles(ip, sshKeyPath, projectName, options = {}) {
+  const remoteDir = `/opt/${projectName}`;
+  const stageDir = options.bundlePath || mkdtempSync(join(tmpdir(), 'vc-deploy-'));
+
+  try {
+    // If not using a pre-rendered bundle, perform ad-hoc rendering (legacy path)
+    if (!options.bundlePath) {
+      // ... (rest of the manual rendering logic if needed, but we now use bundle.js)
+      // Actually, to keep it clean, we just import renderBundle if bundlePath is missing
+      const { renderBundle } = await import('../bundle.js');
+      const adhocBundle = renderBundle(projectName, options);
+      // Copy from adhoc to stageDir or just use adhoc
+      // For simplicity, let's just use the renderBundle result
+      options.bundlePath = adhocBundle;
+    }
+
+    // -- Single tar + SSH pipe to extract on server --
+    // Per-arm filename. HA-scale and HA-deploy fan two arms in parallel
+    // through this function, both with the same projectName. A shared
+    // /tmp path was racing — Arm A's stream of the bundle could mid-flight
+    // read a file Arm B had already begun overwriting, surfacing on the
+    // remote side as "gzip: stdin: unexpected end of file". The ip + pid +
+    // random suffix gives each arm its own file so the cleanup rmSync below
+    // can't yank Arm A's stream out from under it.
+    const safeIp = ip.replace(/[^0-9a-zA-Z]/g, '_');
+    const tarPath = join(
+      tmpdir(),
+      `vc-deploy-${projectName}-${safeIp}-${process.pid}-${randomBytes(6).toString('hex')}.tar.gz`,
+    );
+    // Inner pack vs upload+extract split — pack is local CPU-bound, upload
+    // is bandwidth-bound. Splitting them isolates whether a slow deploy is
+    // dev-laptop or network/server side. Clack spinners replace the older
+    // `console.log('[sync] ...')` triplet so the operator sees animation
+    // during the perfAsync wait instead of dead silence.
+    const packSpinner = p.spinner();
+    packSpinner.start(`Packing bundle from ${options.bundlePath}`);
+    try {
+      await perfAsync('deploy.bundle.tarPack', () =>
+        runCommandAsync(['tar', '-czf', tarPath, '-C', options.bundlePath, '.'], {
+          silent: true,
+          timeout: 30_000,
+        }),
+      );
+      packSpinner.stop('Bundle packed');
+    } catch (err) {
+      packSpinner.stop('Bundle pack failed', 1);
+      throw err;
+    }
+
+    const uploadSpinner = p.spinner();
+    uploadSpinner.start(`Uploading + extracting bundle on ${ip}`);
+    try {
+      await perfAsync('deploy.bundle.extract', () =>
+        runCommandAsync(
+          [
+            'bash',
+            '-c',
+            'cat "$1" | ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -i "$2" "root@$3" "mkdir -p $4/backups && tar --no-xattrs --no-same-owner -xzf - -C $4 && cp $4/$5.service /etc/systemd/system/ && systemctl daemon-reload"',
+            '--',
+            tarPath,
+            sshKeyPath,
+            ip,
+            remoteDir,
+            projectName,
+          ],
+          { silent: true, timeout: 300_000 },
+        ),
+      );
+      uploadSpinner.stop('Bundle synced');
+    } catch (err) {
+      uploadSpinner.stop('Bundle upload failed', 1);
+      throw err;
+    }
+
+    // Clean up local tar
+    try {
+      rmSync(tarPath, { force: true });
+    } catch {
+      // Ignore cleanup errors
+    }
+  } finally {
+    // Clean up staging dir only if we created it (adhoc)
+    if (!options.bundlePath) {
+      try {
+        rmSync(stageDir, { recursive: true, force: true });
+      } catch {
+        // Ignore cleanup errors
+      }
+    }
+  }
+}
+
+/**
+ * Heuristic: does this image tag refer to a registry path or a local-only tag?
+ *
+ * A registry path always contains either a `/` (e.g. `ghcr.io/owner/repo:tag`,
+ * `library/postgres`) or a host:port prefix. Local tags built by direct mode
+ * are bare names like `<project>-app:local`. Distinguishing the two lets
+ * callers decide whether to ship the image to peer servers (local-only)
+ * or rely on `docker pull` (registry path).
+ */
+export function isLocalOnlyImageTag(imageRef) {
+  if (!imageRef || typeof imageRef !== 'string') return false;
+  const [namePart] = imageRef.split(':');
+  // A '/' anywhere in the name part means the tag carries a registry path
+  // (e.g. ghcr.io/owner/repo or owner/repo). Local docker tags built via
+  // `docker build -t <project>-app:local` have no slash.
+  return !namePart.includes('/');
+}
+
+/**
+ * Distribute a locally-built Docker image from one server's daemon to another
+ * via an SSH-piped `docker save | docker load`.
+ *
+ * Direct-mode deploys build the image into a single server's local Docker
+ * daemon (via `DOCKER_HOST=ssh://<source>`). Multi-server scenarios (HA
+ * standby, compose-scale's blue-green new VM) need that image on the other
+ * servers before reconcile.sh runs `docker compose up -d` — otherwise compose
+ * tries to `docker pull <local-tag>`, which is not a registry path, and fails
+ * with `pull access denied`.
+ *
+ * The pure SSH-pipe approach avoids any temp-file or scp intermediate: stdout
+ * of `docker save` on the source streams directly into `docker load` on the
+ * destination. For a 600MB image across hel1↔nbg1 this typically lands in
+ * 30-60s; we budget 10 minutes (large images + slow inter-DC links).
+ *
+ * No-op (returns without erroring) when imageRef looks like a registry path,
+ * since `docker compose pull` will already handle those.
+ *
+ * @param {string} sourceIp - Server that already has the image in its daemon
+ * @param {string} destIp   - Server that needs the image
+ * @param {string} sshKeyPath
+ * @param {string} imageRef - Tag to transfer (e.g. `<project>-app:local`)
+ * @returns {Promise<void>}
+ */
+export async function transferImageBetweenServers(sourceIp, destIp, sshKeyPath, imageRef) {
+  if (!isLocalOnlyImageTag(imageRef)) {
+    // Registry-path image — let `docker compose pull` handle it.
+    return;
+  }
+  if (sourceIp === destIp) return;
+  // shEscape the imageRef so a future tag with shell metacharacters (':',
+  // '@', '/' are all fine; we still escape for defense-in-depth) can't
+  // break out of `docker save`. The other tokens (ip, sshKeyPath) are
+  // controlled by the deploy code, not user input.
+  // gzip on the wire — same trick as sideloadCompose / sideloadK3s. Node
+  // images compress 3-5x; without gzip a ~600MB save streams uncompressed
+  // across two regions on top of the operator's residential upload, which
+  // historically tripped the 600s timeout under load.
+  const saveCmd = `ssh ${SSH_OPTS} -i ${shEscape(sshKeyPath)} root@${sourceIp} docker save ${shEscape(imageRef)}`;
+  const loadCmd = `ssh ${SSH_OPTS} -i ${shEscape(sshKeyPath)} root@${destIp} 'gunzip | docker load'`;
+  await runCommandAsync(['bash', '-c', `set -o pipefail && ${saveCmd} | gzip -1 | ${loadCmd}`], {
+    silent: true,
+    timeout: 600_000,
+  });
+}
+
+/**
+ * Wait for Docker daemon to be ready on a remote server.
+ *
+ * On Hetzner docker-ce VPSes Docker is already running once cloud-init
+ * releases the dpkg lock, so the first check usually succeeds immediately.
+ * Short initial interval keeps the latency tight on the rare miss.
+ */
+export async function waitForDockerReady(ip, sshKeyPath, maxAttempts = 12) {
+  for (let i = 0; i < maxAttempts; i++) {
+    const result = await sshRunAsync(ip, sshKeyPath, 'docker info >/dev/null 2>&1 && echo READY', {
+      timeout: 10_000,
+      ignoreError: true,
+    });
+    if (result?.trim() === 'READY') return true;
+    if (i < maxAttempts - 1) {
+      const interval = i < 3 ? 1500 : 5000;
+      await new Promise((r) => setTimeout(r, interval));
+    }
+  }
+  return false;
+}
+
+/**
+ * Build the compose files flags string for a given set of service options.
+ */
+function composeFileFlags(options = {}) {
+  const files = ['-f', 'docker-compose.yml', '-f', 'docker-compose.prod.yml'];
+  if (options.observability) {
+    files.push('-f', 'docker-compose.observability.yml');
+    files.push('-f', 'docker-compose.observability.prod.yml');
+  }
+  if (options.n8n) {
+    files.push('-f', 'docker-compose.n8n.yml', '-f', 'docker-compose.n8n.prod.yml');
+  }
+  if (options.metabase) {
+    files.push('-f', 'docker-compose.metabase.yml', '-f', 'docker-compose.metabase.prod.yml');
+  }
+  if (options.redis) files.push('-f', 'docker-compose.redis.yml');
+  return files.join(' ');
+}
+
+/**
+ * Start the Docker Compose stack on the remote server
+ */
+export async function startComposeStack(ip, sshKeyPath, projectName, _options = {}) {
+  const remoteDir = `/opt/${projectName}`;
+  // Spinner over the long silent compose.up — typical 60-130s warm, longer
+  // cold while Supabase services pull. Replaces the previous
+  // `console.log('[reconcile] Running...')` + silence + `Reconcile
+  // complete.` pair.
+  const reconcileSpinner = p.spinner();
+  reconcileSpinner.start(`Reconciling stack on ${ip} (docker compose up)`);
+  try {
+    // reconcile.sh combines `docker compose pull` + `docker compose up -d`
+    // on the server. The orchestrator-level `deploy.reconcile.run` wraps the
+    // outer call; this inner timer captures the SSH-side cost separately so
+    // we can attribute slow deploys to the remote-bash phase vs. the SSH
+    // round-trip overhead.
+    //
+    // 1800s (30 min) timeout: iter-banner 2026-05-02 had compose-ha standby
+    // reconcile run past the previous 900s budget while Docker Hub / Hetzner
+    // egress was slow — SSH was making progress (cluster came up healthy
+    // after the wrapper timed out) but the timer ate the run. 30 min covers
+    // a slow-pull cold start on any of the three S3 regions plus margin
+    // without masking a true regression (a real hang would still surface
+    // via the 60s ServerAlive keepalives in lib/ssh.js + compose SSH_OPTS).
+    await perfAsync('deploy.compose.up', () =>
+      sshRunAsync(ip, sshKeyPath, `/bin/bash ${remoteDir}/reconcile.sh`, { timeout: 1_800_000 }),
+    );
+    reconcileSpinner.stop('Stack reconciled');
+  } catch (err) {
+    reconcileSpinner.stop('Stack reconcile failed', 1);
+    const stdout = err.stdout?.trim?.();
+    if (stdout) {
+      const lines = stdout.split('\n');
+      const tail = lines.slice(-30).join('\n');
+      err.message = `${err.message}\n--- reconcile output (last 30 lines) ---\n${tail}`;
+    }
+    // Capture the db container's actual stderr — `docker compose up` only
+    // surfaces "container is unhealthy" / "Error", not the postgres exit
+    // reason. Without these logs, debugging a failed re-deploy means
+    // "guess and re-run". Mirrors what compose/ha.js already does on
+    // failover. Best-effort — we re-throw the original failure regardless.
+    try {
+      const dbLogs = await sshRunAsync(
+        ip,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose logs db --tail=60 2>&1 || true`,
+        { timeout: 30_000 },
+      );
+      if (typeof dbLogs === 'string' && dbLogs.trim()) {
+        err.message = `${err.message}\n--- db container logs (last 60 lines) ---\n${dbLogs.trim()}`;
+      }
+    } catch (logErr) {
+      err.message = `${err.message}\n(failed to capture db logs: ${logErr.message})`;
+    }
+    throw err;
+  }
+}
+
+/**
+ * Wait for all compose services with healthchecks to reach "healthy".
+ *
+ * `docker compose up -d` returns as soon as containers are *created*, not
+ * when their healthchecks pass. For Supabase, auth/storage/rest all need
+ * 10-60s after container-start to become healthy — verify-deploy's
+ * storage_upload / rest_api checks 502 when they run against a starting
+ * service. Polls every 5s up to 3min; returns early once every service
+ * with a declared healthcheck reports "healthy".
+ *
+ * Non-critical — on timeout or error we log but don't fail, since the
+ * caller's primary gate is the app-container probe which already passed.
+ */
+async function waitForServicesHealthy(ip, sshKeyPath, projectName, maxWaitMs = 180_000) {
+  const remoteDir = `/opt/${projectName}`;
+  const cmd = `cd ${remoteDir} && docker compose ps --format '{{.Service}} {{.Health}}' 2>&1`;
+  const deadline = Date.now() + maxWaitMs;
+  let lastState = '';
+  while (Date.now() < deadline) {
+    try {
+      const out = await sshRunAsync(ip, sshKeyPath, cmd, {
+        timeout: 15_000,
+        ignoreError: true,
+      });
+      const lines = (out || '')
+        .trim()
+        .split('\n')
+        .map((l) => l.trim())
+        .filter(Boolean);
+      // Partition by health state. Services without a declared healthcheck
+      // report empty/no-health — skip those, they don't block readiness.
+      const stillWarming = lines.filter((l) => l.endsWith(' starting') || l.endsWith(' unhealthy'));
+      lastState = stillWarming.join(', ') || 'all healthy';
+      if (stillWarming.length === 0) return true;
+    } catch {
+      /* retry */
+    }
+    await new Promise((r) => setTimeout(r, 5000));
+  }
+  // Best-effort; don't fail the deploy here — the app probe already passed.
+  console.error(
+    `[verifyAppHealth] services still warming after ${maxWaitMs / 1000}s: ${lastState}`,
+  );
+  return false;
+}
+
+/**
+ * Verify the deployed app is actually serving requests.
+ *
+ * Probes `http://localhost/api/health` on the server itself with a Host header
+ * (if a domain is configured), bypassing public DNS and TLS to isolate app
+ * health from DNS propagation / cert issuance races. Polls up to `attempts`
+ * times with 3s between attempts.
+ *
+ * On failure, captures `docker compose ps` + the last 40 lines of the `app`
+ * container log so operators see WHY the probe failed, not just THAT it did.
+ *
+ * Returns `{ healthy, status, details }`:
+ *   - healthy  — true iff the probe returned HTTP 2xx within the budget
+ *   - status   — HTTP status of the last probe ('none' if we never got one)
+ *   - details  — diagnostic string (empty when healthy)
+ */
+export async function verifyAppHealth(ip, sshKeyPath, projectName, options = {}) {
+  // Probe the app container directly on its internal port (3000), bypassing
+  // Traefik + TLS. Traefik's port 80 listener redirects to HTTPS (301) and
+  // HTTPS requires a valid cert which may still be issuing on a fresh
+  // deploy. Hitting the container directly tests app-layer health without
+  // entangling the ingress path — the e2e harness's own
+  // waitForHealthy check (which hits the public URL) verifies end-to-end.
+  //
+  // First-boot route registration takes longer than the initial `docker
+  // compose up -d` return: the app has to start + bind port 3000.
+  // Empirically 10-30s. 3 min tolerates worst-case cold starts.
+  const { path = '/api/health', attempts = 18, delayMs = 10_000 } = options;
+  const remoteDir = `/opt/${projectName}`;
+  // Use `node` for the probe — the app's runtime image is guaranteed to have
+  // it (it runs the server), whereas curl/wget may not be present in slim
+  // node base images. One-liner: HEAD the local port 3000 path; exit 0 on
+  // 2xx, 1 otherwise. The outer shell echoes "2xx" or "fail" so the caller
+  // sees a deterministic token on stdout regardless of node's exit code.
+  const nodeScript = `const r=require('http').request({host:'localhost',port:3000,path:${JSON.stringify(path)},method:'GET',timeout:4000},res=>{process.exit(res.statusCode<300?0:1)});r.on('error',()=>process.exit(2));r.on('timeout',()=>process.exit(3));r.end();`;
+  const nodeScriptB64 = Buffer.from(nodeScript).toString('base64');
+  const probeCmd = `cd ${remoteDir} && (echo ${nodeScriptB64} | base64 -d | docker compose exec -T app node - && echo 2xx || echo fail) 2>&1`;
+
+  let lastStatus = 'none';
+  for (let i = 0; i < attempts; i++) {
+    if (i > 0) await new Promise((r) => setTimeout(r, delayMs));
+    try {
+      const result = await sshRunAsync(ip, sshKeyPath, probeCmd, {
+        timeout: 15_000,
+        ignoreError: true,
+      });
+      const out = (result || '').trim();
+      lastStatus = out.slice(-8) || 'none';
+      if (out.endsWith('2xx')) {
+        // App is serving, but Supabase services may still be warming up
+        // their healthchecks. The e2e harness's verify-deploy probes
+        // auth/storage/rest endpoints immediately after deploy returns —
+        // a 502 from storage that takes 60s to become healthy would fail
+        // otherwise-valid deploys. Wait for all compose-managed services
+        // with healthchecks to reach "healthy". Bounded: 3 min budget
+        // (wasn't spent during the app probe since that succeeded quickly).
+        await waitForServicesHealthy(ip, sshKeyPath, projectName);
+        return { healthy: true, status: '200', details: '' };
+      }
+    } catch (err) {
+      lastStatus = err.message?.slice(0, 80) || 'ssh-error';
+    }
+  }
+
+  // Unhealthy — collect diagnostics.
+  let ps = '';
+  let logs = '';
+  try {
+    ps =
+      (await sshRunAsync(
+        ip,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose ps --format '{{.Name}}\\t{{.State}}\\t{{.Status}}' 2>&1 | head -20`,
+        { timeout: 15_000, ignoreError: true },
+      )) || '';
+  } catch {
+    /* best effort */
+  }
+  try {
+    logs =
+      (await sshRunAsync(
+        ip,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose logs --tail=40 app 2>&1 | tail -40`,
+        { timeout: 15_000, ignoreError: true },
+      )) || '';
+  } catch {
+    /* best effort */
+  }
+  const details =
+    `Health probe failed after ${attempts} attempts (last status: ${lastStatus})\n` +
+    `--- docker compose ps ---\n${ps.trim()}\n` +
+    `--- app container log tail ---\n${logs.trim()}`;
+  return { healthy: false, status: lastStatus, details };
+}
+
+/**
+ * Stop the Docker Compose stack on the remote server
+ */
+export async function stopComposeStack(ip, sshKeyPath, projectName, options = {}) {
+  const remoteDir = `/opt/${projectName}`;
+  const flags = composeFileFlags(options);
+  const composeCmd = `cd ${remoteDir} && docker compose ${flags} down --timeout 30`;
+  await sshRunAsync(ip, sshKeyPath, composeCmd, { timeout: 120_000 });
+}
+
+/**
+ * Pre-pull container images on the remote server.
+ * Run this in parallel with the local Docker build to overlap I/O.
+ */
+export async function pullComposeImages(ip, sshKeyPath, projectName, options = {}) {
+  const remoteDir = `/opt/${projectName}`;
+  const flags = composeFileFlags(options);
+  // `docker compose pull` over SSH — first deploys pull ~2GB of Supabase
+  // images and Hetzner ingress can spike to several minutes. Warm pulls
+  // (everything cached) finish in <5s.
+  await perfAsync('deploy.compose.imagesPull', () =>
+    sshRunAsync(
+      ip,
+      sshKeyPath,
+      `cd ${remoteDir} && (docker compose ${flags} pull --policy missing --ignore-buildable || docker compose ${flags} pull --ignore-buildable || docker compose ${flags} pull 2>&1 || true)`,
+      { timeout: 600_000, ignoreError: true },
+    ),
+  );
+}
+
+/**
+ * Run database migrations on the remote server
+ */
+export async function runMigrations(ip, sshKeyPath, projectName) {
+  const remoteDir = `/opt/${projectName}`;
+
+  // Wait for PostgreSQL to be ready, then run every migration — all in one
+  // SSH round-trip. A `for f in *.sql; do ...; done` on the remote shell
+  // executes the migrations sequentially without N+1 SSH handshakes (one
+  // ls, one psql per file = N+1 trips before). `|| true` per-file keeps a
+  // bad migration from halting the rest (matches prior behaviour).
+  await sshRunAsync(
+    ip,
+    sshKeyPath,
+    `cd ${remoteDir} && docker compose exec -T db pg_isready -U postgres && ` +
+      `for f in $(ls supabase/migrations/ 2>/dev/null | sort); do ` +
+      `cat "supabase/migrations/$f" | docker compose exec -T db psql -U supabase_admin -d postgres 2>/dev/null || true; ` +
+      `done`,
+    { timeout: 300_000 },
+  );
+}
+
+/**
+ * Main Compose deployment orchestrator
+ *
+ * @param {object} options
+ * @param {string} options.projectName - Project name
+ * @param {string} options.environment - Environment name (prod, staging, etc.)
+ * @param {string} options.ip - Server IP address
+ * @param {string} options.sshKeyPath - Path to SSH private key
+ * @param {string} [options.domain] - Custom domain
+ * @param {boolean} [options.isNewServer] - Whether server was just provisioned
+ * @param {object} [options.services] - Enabled services (observability, n8n, etc.)
+ * @param {Function} [options.onProgress] - Progress callback
+ * @returns {object} Deployment result
+ */
+export async function deployCompose(options) {
+  const {
+    projectName,
+    ip,
+    sshKeyPath,
+    domain,
+    isNewServer = false,
+    services = {},
+    imageRef,
+    onProgress = () => {},
+  } = options;
+
+  if (!imageRef) {
+    throw new Error('deployCompose requires options.imageRef (ghcr.io/<owner>/<repo>:<tag>)');
+  }
+
+  // Step 1: Install Docker on new servers
+  if (isNewServer) {
+    onProgress('Configuring server...');
+    await setupServer(ip, sshKeyPath);
+  }
+
+  const serviceOpts = {
+    observability: services.observability,
+    n8n: services.n8n,
+    metabase: services.metabase,
+    redis: services.redis,
+  };
+
+  // Step 2: Copy project files + point APP_IMAGE at the CI-built ghcr.io tag
+  onProgress('Copying project files to server...');
+  await setupServerFiles(ip, sshKeyPath, projectName, {
+    domain,
+    image: imageRef,
+    ...serviceOpts,
+  });
+
+  // Step 3: Pull all images (app + Supabase) — docker compose pull handles it.
+  onProgress('Pulling images...');
+  await pullComposeImages(ip, sshKeyPath, projectName, serviceOpts);
+
+  // Step 4: Start Docker Compose stack
+  onProgress('Starting services...');
+  await startComposeStack(ip, sshKeyPath, projectName, serviceOpts);
+
+  // Step 5: Run migrations
+  onProgress('Running database migrations...');
+  const remoteDir = `/opt/${projectName}`;
+  try {
+    await runMigrations(ip, sshKeyPath, projectName);
+    // Reload PostgREST schema cache so it sees newly created tables
+    await sshRunAsync(
+      ip,
+      sshKeyPath,
+      `cd ${remoteDir} && docker compose exec -T rest kill -s SIGUSR1 1 2>/dev/null || true`,
+    );
+  } catch {
+    // Migrations may partially fail on first run, non-fatal
+  }
+
+  return {
+    success: true,
+    ip,
+    domain,
+    deployMode: 'compose',
+  };
+}
+
+/**
+ * Fast re-deploy: sync new config and reconcile.
+ */
+export async function redeployCompose(options) {
+  const { projectName, ip, sshKeyPath, services = {}, imageRef, onProgress = () => {} } = options;
+
+  if (!imageRef) {
+    throw new Error('redeployCompose requires options.imageRef');
+  }
+
+  // Step 1: Sync new files (including .env with new image tag)
+  onProgress('Syncing configuration...');
+  await setupServerFiles(ip, sshKeyPath, projectName, {
+    image: imageRef,
+    observability: services.observability,
+    n8n: services.n8n,
+    metabase: services.metabase,
+    redis: services.redis,
+    domain: options.domain,
+  });
+
+  // Step 2: Reconcile (pulls images if needed and restarts)
+  onProgress('Reconciling services...');
+  await startComposeStack(ip, sshKeyPath, projectName, {
+    observability: services.observability,
+    n8n: services.n8n,
+    metabase: services.metabase,
+    redis: services.redis,
+  });
+
+  return { success: true };
+}
+
+/**
+ * Get status of Docker Compose services on the remote server
+ */
+export function getComposeStatus(ip, sshKeyPath, projectName) {
+  const remoteDir = `/opt/${projectName}`;
+
+  try {
+    const output = sshRun(ip, sshKeyPath, `cd ${remoteDir} && docker compose ps --format json`, {
+      timeout: 15_000,
+    });
+
+    if (!output?.trim()) {
+      return { running: false, services: [] };
+    }
+
+    // docker compose ps --format json outputs one JSON object per line
+    const services = output
+      .trim()
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => {
+        try {
+          return JSON.parse(line);
+        } catch {
+          return null;
+        }
+      })
+      .filter(Boolean);
+
+    return {
+      running: services.some((s) => s.State === 'running'),
+      services,
+    };
+  } catch {
+    return { running: false, services: [] };
+  }
+}
+
+/**
+ * Destroy a Docker Compose deployment
+ */
+export function destroyCompose(ip, sshKeyPath, projectName) {
+  const remoteDir = `/opt/${projectName}`;
+
+  // Stop and remove containers + volumes
+  sshRun(
+    ip,
+    sshKeyPath,
+    `cd ${remoteDir} && docker compose down -v --remove-orphans 2>/dev/null; rm -rf ${remoteDir}`,
+    {
+      timeout: 120_000,
+    },
+  );
+}
+
+/**
+ * Create a full backup via Docker Compose.
+ * Produces a .tar.gz archive containing:
+ *   - postgres.sql.gz (main Supabase database, always)
+ *   - n8n.sql.gz (if n8n database exists)
+ *   - metabase.sql.gz (if metabase database exists)
+ *   - storage.tar.gz (if file-backend storage files exist)
+ */
+export function backupCompose(ip, sshKeyPath, projectName, options = {}) {
+  const { clean = false } = options;
+  const remoteDir = `/opt/${projectName}`;
+  const timestamp = new Date().toISOString().replace(/T/, '_').replace(/[:.]/g, '').slice(0, 15);
+  const archiveName = `${projectName}_${timestamp}_full.tar.gz`;
+  const cleanFlags = clean ? '--clean --if-exists' : '';
+
+  // Step 1: Dump postgres database
+  const t1 = perfTimer('backup.pgDump.main');
+  sshRun(
+    ip,
+    sshKeyPath,
+    `cd ${remoteDir} && docker compose exec -T db pg_dump -U postgres ${cleanFlags} postgres | gzip > backups/_postgres.sql.gz`,
+    { timeout: 300_000 },
+  );
+  t1.end();
+
+  // Step 2: Dump optional service databases (n8n, metabase) if they exist
+  const t2 = perfTimer('backup.pgDump.features');
+  sshRun(
+    ip,
+    sshKeyPath,
+    `cd ${remoteDir} && for db in n8n metabase; do docker compose exec -T db psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='\${db}'" 2>/dev/null | grep -q 1 && docker compose exec -T db pg_dump -U postgres "\${db}" | gzip > "backups/_\${db}.sql.gz" || true; done`,
+    { timeout: 300_000 },
+  );
+  t2.end();
+
+  // Step 3: Backup storage files if using file backend
+  const t3 = perfTimer('backup.storageFiles');
+  sshRun(
+    ip,
+    sshKeyPath,
+    `cd ${remoteDir} && FCOUNT=$(docker compose exec -T storage sh -c 'find /var/lib/storage -type f 2>/dev/null | wc -l' 2>/dev/null || echo 0) && [ "$FCOUNT" -gt 0 ] 2>/dev/null && docker compose exec -T storage tar czf - -C / var/lib/storage > backups/_storage.tar.gz || true`,
+    { timeout: 300_000 },
+  );
+  t3.end();
+
+  // Step 4: Create combined archive from work files (prefixed with _)
+  const t4 = perfTimer('backup.combineArchive');
+  sshRun(
+    ip,
+    sshKeyPath,
+    `cd ${remoteDir}/backups && tar czf ${archiveName} $(ls _postgres.sql.gz _n8n.sql.gz _metabase.sql.gz _storage.tar.gz 2>/dev/null) && rm -f _postgres.sql.gz _n8n.sql.gz _metabase.sql.gz _storage.tar.gz`,
+    { timeout: 120_000 },
+  );
+  t4.end();
+
+  return archiveName;
+}
+
+/**
+ * Upload a compose backup to S3.
+ *
+ * Downloads the archive from the VPS to a local temp file via SCP, then uploads
+ * to S3 using the Node.js SDK. This replaces the previous approach of installing
+ * awscli on the VPS (which failed silently on many server images).
+ */
+export async function uploadComposeBackupToS3(
+  ip,
+  sshKeyPath,
+  projectName,
+  archiveName,
+  backupS3Config,
+) {
+  const localTmpPath = join(tmpdir(), `vc-backup-${archiveName}`);
+  try {
+    await perfAsync('backup.scpFromServer', async () =>
+      scpDownload(ip, sshKeyPath, `/opt/${projectName}/backups/${archiveName}`, localTmpPath),
+    );
+    await perfAsync('backup.uploadS3', async () =>
+      uploadS3Backup(backupS3Config, localTmpPath, `backups/${archiveName}`),
+    );
+  } finally {
+    try {
+      unlinkSync(localTmpPath);
+    } catch {
+      // Ignore cleanup errors
+    }
+  }
+}
+
+/**
+ * Set up automated backup cron on the VPS.
+ */
+export function setupComposeBackupCron(ip, sshKeyPath, projectName, backupS3Config, backupConfig) {
+  const remoteDir = `/opt/${projectName}`;
+  const schedule = backupConfig?.schedule || '0 */6 * * *';
+  const retentionDays = backupConfig?.retentionDays || 30;
+
+  // Build environment file for cron
+  const envLines = [`PROJECT_NAME=${projectName}`, `BACKUP_RETENTION_DAYS=${retentionDays}`];
+  if (backupS3Config) {
+    envLines.push(
+      `S3_BACKUP_BUCKET=${backupS3Config.bucket}`,
+      `S3_ACCESS_KEY=${backupS3Config.accessKey}`,
+      `S3_SECRET_KEY=${backupS3Config.secretKey}`,
+      `S3_ENDPOINT=${backupS3Config.endpoint}`,
+      `S3_REGION=${backupS3Config.region}`,
+    );
+  }
+
+  // Install awscli for automated S3 uploads (fail loudly if it can't be installed)
+  if (backupS3Config) {
+    sshRun(
+      ip,
+      sshKeyPath,
+      'command -v aws >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y awscli || pip3 install awscli)',
+      { timeout: 120_000 },
+    );
+  }
+
+  // Write env file and install cron
+  const envContent = envLines.join('\\n');
+  sshRun(
+    ip,
+    sshKeyPath,
+    [
+      `printf '${envContent}\\n' > ${remoteDir}/backups/.backup-env`,
+      `chmod 600 ${remoteDir}/backups/.backup-env`,
+      `chmod +x ${remoteDir}/backup/compose-backup.sh`,
+      `(crontab -l 2>/dev/null | grep -v 'compose-backup.sh' ; echo '${schedule} set -a && . ${remoteDir}/backups/.backup-env && set +a && ${remoteDir}/backup/compose-backup.sh >> ${remoteDir}/backups/backup.log 2>&1') | crontab -`,
+    ].join(' && '),
+    { timeout: 120_000 },
+  );
+}
+
+/**
+ * Restore from a backup archive via Docker Compose.
+ * Supports legacy .tar.gz (pg_dump) and new WAL-G point-in-time recovery.
+ */
+export async function restoreCompose(ip, sshKeyPath, projectName, backupFile, options = {}) {
+  const remoteDir = `/opt/${projectName}`;
+  const isWalg = backupFile === 'latest' || backupFile === 'walg';
+
+  // Stop the app container to prevent connections
+  await sshRunAsync(
+    ip,
+    sshKeyPath,
+    `cd ${remoteDir} && docker compose stop app 2>/dev/null || true`,
+  );
+
+  if (isWalg) {
+    // --- BLAZING FAST WAL-G RESTORE ---
+    // 1. Stop DB
+    await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose stop db`);
+    // 2. Clear data (WAL-G restore-fetch requires empty dir)
+    await sshRunAsync(
+      ip,
+      sshKeyPath,
+      `cd ${remoteDir} && docker compose run --rm -v db_data:/data alpine sh -c "rm -rf /data/*"`,
+    );
+    // 3. Run WAL-G restore
+    const restoreCmd = `cd ${remoteDir} && docker compose run --rm \
+      -e WALG_S3_PREFIX="${options.walgS3Prefix}" \
+      -e AWS_ACCESS_KEY_ID="${options.s3AccessKey}" \
+      -e AWS_SECRET_ACCESS_KEY="${options.s3SecretKey}" \
+      -e AWS_ENDPOINT="${options.s3Endpoint}" \
+      -e AWS_REGION="${options.s3Region}" \
+      db wal-g backup-fetch /var/lib/postgresql/data LATEST`;
+    await sshRunAsync(ip, sshKeyPath, restoreCmd, { timeout: 900_000 });
+    // 4. Start DB
+    await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose start db`);
+  } else {
+    // --- LEGACY PG_DUMP RESTORE ---
+    // Extract archive
+    await sshRunAsync(
+      ip,
+      sshKeyPath,
+      `mkdir -p /tmp/vb-restore && tar --no-xattrs --no-same-owner -xzf ${remoteDir}/backups/${backupFile} -C /tmp/vb-restore`,
+      { timeout: 120_000 },
+    );
+
+    // Restore postgres database
+    await sshRunAsync(
+      ip,
+      sshKeyPath,
+      `cd ${remoteDir} && gunzip -c /tmp/vb-restore/postgres.sql.gz | docker compose exec -T db psql -U postgres -d postgres`,
+      { timeout: 300_000 },
+    );
+
+    // Restore optional service databases if present
+    for (const db of ['n8n', 'metabase']) {
+      await sshRunAsync(
+        ip,
+        sshKeyPath,
+        `test -f /tmp/vb-restore/${db}.sql.gz && cd ${remoteDir} && gunzip -c /tmp/vb-restore/${db}.sql.gz | docker compose exec -T db psql -U postgres -d ${db} || true`,
+        { timeout: 300_000 },
+      );
+    }
+
+    // Restore storage files if present
+    await sshRunAsync(
+      ip,
+      sshKeyPath,
+      `test -f /tmp/vb-restore/storage.tar.gz && cd ${remoteDir} && docker compose exec -T storage sh -c 'tar --no-xattrs --no-same-owner -xzf - -C /' < /tmp/vb-restore/storage.tar.gz || true`,
+      { timeout: 300_000 },
+    );
+
+    // Cleanup
+    await sshRunAsync(ip, sshKeyPath, 'rm -rf /tmp/vb-restore');
+  }
+
+  // Restart app
+  await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose start app`);
+}
+
+/**
+ * Wait for an HTTPS endpoint to respond with a valid certificate.
+ * Polls until a successful response (2xx/3xx) or timeout.
+ *
+ * @param {string} url - Full HTTPS URL to check
+ * @param {object} [options]
+ * @param {number} [options.timeout=90000] - Max wait time in ms
+ * @param {number} [options.interval=5000] - Poll interval in ms
+ * @returns {Promise<boolean>} true if SSL is ready, false if timed out
+ */
+export async function waitForSSL(url, { timeout = 90_000, interval = 5_000 } = {}) {
+  const deadline = Date.now() + timeout;
+  while (Date.now() < deadline) {
+    try {
+      const result = runCommand(
+        ['curl', '-sf', '--max-time', '5', '-o', '/dev/null', '-w', '%{http_code}', url],
+        {
+          stdio: 'pipe',
+          encoding: 'utf-8',
+          timeout: 10_000,
+        },
+      );
+      if (result?.trim().match(/^[23]\d\d$/)) return true;
+    } catch {
+      // Not ready yet
+    }
+    await new Promise((r) => setTimeout(r, interval));
+  }
+  return false;
+}
+
+/**
+ * Create the admin user in production Supabase via the GoTrue admin API.
+ *
+ * The user created during `vibecarbon create` only exists in the local Docker
+ * Compose Supabase. Production needs its own admin user. The API returns 422
+ * if the user already exists, making this idempotent.
+ *
+ * @param {string} serverIp - IP of the server running Supabase
+ * @param {string} sshKeyPath - Path to SSH private key
+ * @param {string} projectName - Project name (for remote dir)
+ * @returns {Promise<{success: boolean, message: string}>}
+ */
+export async function createAdminUser(serverIp, sshKeyPath, projectName) {
+  const envPath = join(process.cwd(), '.env');
+  if (!existsSync(envPath)) {
+    return { success: false, message: 'No .env file found' };
+  }
+
+  const envContent = readFileSync(envPath, 'utf-8');
+  const adminEmail = envContent.match(/^ADMIN_EMAIL="?([^"\n]+)"?/m)?.[1];
+  const adminPassword = envContent.match(/^ADMIN_PASSWORD="?([^"\n]+)"?/m)?.[1];
+  const serviceRoleKey = envContent.match(/^SUPABASE_SERVICE_ROLE_KEY="?([^"\n]+)"?/m)?.[1];
+
+  if (!adminEmail || !adminPassword || !serviceRoleKey) {
+    return { success: false, message: 'Admin credentials not found in .env' };
+  }
+
+  const remoteDir = `/opt/${projectName}`;
+
+  // Wait for auth service to become healthy. Once the stack starts, GoTrue
+  // typically answers /health within 15-25s. Short initial interval catches
+  // the fast path without penalising slower boots.
+  let authReady = false;
+  for (let i = 0; i < 30; i++) {
+    const health = await sshRunAsync(
+      serverIp,
+      sshKeyPath,
+      `cd ${remoteDir} && docker compose exec -T auth wget -qO- http://localhost:9999/health 2>/dev/null || true`,
+      { timeout: 10_000, ignoreError: true },
+    );
+    if (health?.includes('GoTrue')) {
+      authReady = true;
+      break;
+    }
+    const interval = i < 5 ? 500 : 2000;
+    await new Promise((r) => setTimeout(r, interval));
+  }
+
+  if (!authReady) {
+    return { success: false, message: 'Auth service not ready' };
+  }
+
+  // Create admin user via GoTrue admin API through an SSH port-forward.
+  // Uses native fetch to avoid shell escaping issues with special chars in passwords.
+  const localPort = 19876;
+  const pf = spawn(
+    'ssh',
+    [
+      '-o',
+      'StrictHostKeyChecking=no',
+      '-o',
+      'UserKnownHostsFile=/dev/null',
+      '-o',
+      'BatchMode=yes',
+      '-i',
+      sshKeyPath,
+      '-N',
+      '-L',
+      `${localPort}:localhost:8000`,
+      `root@${serverIp}`,
+    ],
+    { stdio: 'ignore' },
+  );
+
+  try {
+    // Wait for port-forward to be ready. ssh -L is usually listening on
+    // localhost within 200-500ms; first few retries are at 200ms before
+    // falling back to 1s.
+    let connected = false;
+    for (let i = 0; i < 15; i++) {
+      try {
+        await fetch(`http://localhost:${localPort}/auth/v1/health`, {
+          signal: AbortSignal.timeout(2000),
+        });
+        connected = true;
+        break;
+      } catch {
+        const interval = i < 5 ? 200 : 1000;
+        await new Promise((r) => setTimeout(r, interval));
+      }
+    }
+
+    if (!connected) {
+      return { success: false, message: 'Could not reach auth service via SSH tunnel' };
+    }
+
+    const response = await fetch(`http://localhost:${localPort}/auth/v1/admin/users`, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${serviceRoleKey}`,
+        apikey: serviceRoleKey,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        email: adminEmail,
+        password: adminPassword,
+        email_confirm: true,
+        app_metadata: { role: 'super_admin' },
+      }),
+      signal: AbortSignal.timeout(15000),
+    });
+
+    if (response.ok) {
+      return { success: true, message: `Admin user created: ${adminEmail}` };
+    }
+    if (response.status === 422) {
+      return { success: true, message: `Admin user already exists: ${adminEmail}` };
+    }
+    const body = await response.text().catch(() => '');
+    return {
+      success: false,
+      message: `Auth returned ${response.status}${body ? `: ${body}` : ''}`,
+    };
+  } finally {
+    pf.kill();
+  }
+}