vibecarbon 0.3.1 → 0.5.0

package/src/lib/deploy/k8s/k3s.js

@@ -40,10 +40,17 @@ import * as p from '@clack/prompts';
 import { runCommandAsync } from '../../command.js';
 import { knownHostsPath } from '../../host-keys.js';
 import { loadCloudInit, renderScript } from '../../iac/cloud-init.js';
+import { dbImageRef } from '../../images.js';
 import { perfAsync } from '../../perf.js';
+import { postAdminUser, waitForGotrueHealth } from '../admin-user.js';
 
 export const K3S_VERSION = 'v1.31.5+k3s1';
 
+/** The db image is pre-published + multi-arch — pulled, never built/sideloaded. */
+export function resolveDbImageTag() {
+  return dbImageRef();
+}
+
 /**
  * Pattern of kube-apiserver transient signatures we've actually observed
  * mid-RPC during k3s control-plane warm-up. See runKubectlWithRetry —
@@ -52,7 +59,12 @@ export const K3S_VERSION = 'v1.31.5+k3s1';
  * obvious place to do it).
  */
 export const KUBECTL_TRANSIENT_PATTERN =
-  /http2: client connection lost|connection reset by peer|context deadline exceeded|EOF|i\/o timeout|TLS handshake|unexpected error when reading response body/i;
+  // `connection timed out` covers cross-cluster API blips like
+  // `read tcp <ip>:<port>->...:6443: read: connection timed out` — seen when a
+  // kubectl exec (e.g. enable-WAL-archiving) hits a momentarily unreachable
+  // standby control-plane in k8s-ha. Without it the call failed without
+  // retrying, aborting the deploy. (RCA 2026-06-01: k8s-ha standby deploy.)
+  /http2: client connection lost|connection reset by peer|context deadline exceeded|connection timed out|EOF|i\/o timeout|TLS handshake|unexpected error when reading response body/i;
 
 /**
  * Run a short-lived kubectl command with retry-on-transient-error.
@@ -208,6 +220,35 @@ export function pickIssuerName({ dnsProvider, acmeServer }) {
   return `letsencrypt-${stage}-${providerSuffix}`;
 }
 
+/**
+ * Build the `dnsNames` list for the `Certificate/vibecarbon-tls` patch.
+ *
+ * DNS-01 issuers (cloudflare, hetzner) support wildcard SANs — cert-manager
+ * can obtain `*.${domain}` in a single ACME order, covering every subdomain
+ * (api, studio, dashboard, grafana, n8n …) without separate per-subdomain
+ * certificates. One wildcard cert = no per-router cert-resolver needed =
+ * no per-domain ACME rate-limit risk.
+ *
+ * Manual issuers use HTTP-01 which cannot issue wildcards, so fall back to
+ * the apex domain only (the IngressRoutes already work; only non-apex
+ * subdomains are uncovered, but those are admin-only tools the operator
+ * configures manually for manual-DNS deploys).
+ *
+ * @param {string} domain - The apex deploy domain, e.g. `e1.carbonstack.dev`
+ * @param {string} issuerName - The ClusterIssuer name, e.g.
+ *   `letsencrypt-prod-hetzner`, produced by `pickIssuerName`
+ * @returns {string[]}
+ */
+export function certificateDnsNames(domain, issuerName) {
+  // Only the known DNS-01 providers (cloudflare, hetzner) support wildcard SANs.
+  // `manual` and any future/unknown issuer name falls back to apex-only so an
+  // unrecognised issuer never attempts a wildcard via HTTP-01 (which LE rejects).
+  const isDns01 = /-(cloudflare|hetzner)$/.test(issuerName);
+  if (!isDns01) return [domain];
+  // DNS-01 issuers (cloudflare, hetzner — prod or staging): issue one wildcard.
+  return [domain, `*.${domain}`];
+}
+
 /**
  * Render the per-DNS-provider Secret YAML that cert-manager's DNS-01
  * solvers read for API auth, or null when the provider doesn't need a
@@ -1046,9 +1087,18 @@ export async function pushImageToLocalRegistry({
   const openTunnel = openTunnelOrWalk;
 
   /** Push the localhost-aliased tag through the local-forwarded port. */
-  const push = () => {
-    execFileSync('docker', ['push', localTag], { stdio: 'inherit' });
-  };
+  // Async spawn (not execFileSync): a docker push of a multi-hundred-MB image
+  // takes minutes, and a synchronous call blocks Node's event loop the whole
+  // time — freezing the spinner/log output and leaking unreaped child
+  // processes (SIGCHLD can't fire while blocked). spawn keeps the loop live.
+  const push = () =>
+    new Promise((resolve, reject) => {
+      const child = spawn('docker', ['push', localTag], { stdio: 'inherit' });
+      child.on('error', (err) => reject(new Error(`docker push spawn failed: ${err.message}`)));
+      child.on('exit', (code) =>
+        code === 0 ? resolve(undefined) : reject(new Error(`docker push exited with code ${code}`)),
+      );
+    });
 
   // Three attempts with progressive backoff. Failure modes we're
   // covering, in priority order:
@@ -1094,7 +1144,7 @@ export async function pushImageToLocalRegistry({
         taggedAliases.add(localTag);
       }
       await delay(settleDelays[attempt]);
-      push();
+      await push();
       pushed = true;
       break;
     } catch (err) {
@@ -1201,6 +1251,14 @@ export async function applyVibecarbonSecrets({ kubeconfig, envLocal, s3Config, b
     stringData.S3_BUCKET = s3Config.bucket;
     stringData.S3_ENDPOINT = s3Config.endpoint;
     stringData.S3_REGION = s3Config.region;
+    // AWS-SDK-format credentials file for wal-g in the supabase-db pod. The
+    // supabase Helm chart can't inject secret env on the db container
+    // (environment.db is string-only), so instead of a post-helm env patch
+    // (which helm would strip every upgrade → a db restart every deploy) we
+    // mount THIS key as a file and point AWS_SHARED_CREDENTIALS_FILE at it
+    // (volume is helm-owned → never stripped → no per-deploy restart). See
+    // k8s/values/supabase.values.yaml deployment.db.volumes.
+    stringData.S3_CREDENTIALS_INI = `[default]\naws_access_key_id=${s3Config.accessKey}\naws_secret_access_key=${s3Config.secretKey}\n`;
   }
   if (backupBucketName) {
     stringData.S3_BACKUP_BUCKET = backupBucketName;
@@ -1255,6 +1313,8 @@ export async function installSupabase({
   domain,
   s3Config,
   envLocal,
+  dbImageTag,
+  backupBucketName,
 }) {
   const env = { ...process.env, KUBECONFIG: kubeconfig };
   // Render the values template — substitute every {{TOKEN}} the template
@@ -1265,10 +1325,35 @@ export async function installSupabase({
       `installSupabase: ${valuesTemplatePath} not found. Project must include the supabase values template.`,
     );
   }
+  if (!dbImageTag) {
+    throw new Error(
+      'installSupabase: dbImageTag is required (the pre-published wal-g db image, pulled from ghcr).',
+    );
+  }
+  // Split the full image ref into repository + tag for the chart's
+  // image.db.{repository,tag} (and the walg-restore init's `{{DB_IMAGE}}:{{DB_IMAGE_TAG}}`).
+  // dbImageTag is now the pre-published `ghcr.io/<org>/postgres:<pg>-walg<ver>`
+  // ref — split on the LAST colon so the registry host (and any future port
+  // colon) stays in the repository.
+  const lastColon = dbImageTag.lastIndexOf(':');
+  const dbImageRepo = dbImageTag.slice(0, lastColon);
+  const dbImageVer = dbImageTag.slice(lastColon + 1);
+  // WAL-G S3 prefix — dedicated backup bucket if provided, else the storage
+  // bucket (mirrors compose's S3_BACKUP_BUCKET:-S3_BUCKET fallback). Never
+  // leave it as s3:/// (wal-g would error).
+  const walgBucket = backupBucketName || s3Config?.bucket || '';
+  const walgS3Prefix = walgBucket
+    ? `s3://${walgBucket}/backups/${projectName || 'vibecarbon'}/walg`
+    : '';
   const rendered = readFileSync(valuesTemplatePath, 'utf-8')
     .replace(/\{\{DOMAIN\}\}/g, domain || 'localhost')
     .replace(/\{\{PROJECT_NAME\}\}/g, projectName || 'vibecarbon')
-    .replace(/\{\{S3_BACKUP_BUCKET\}\}/g, s3Config?.bucket ?? '')
+    .replace(/\{\{S3_BACKUP_BUCKET\}\}/g, walgBucket)
+    .replace(/\{\{WALG_S3_PREFIX\}\}/g, walgS3Prefix)
+    .replace(/\{\{S3_ENDPOINT\}\}/g, s3Config?.endpoint ?? '')
+    .replace(/\{\{S3_REGION\}\}/g, s3Config?.region ?? '')
+    .replace(/\{\{DB_IMAGE\}\}/g, dbImageRepo)
+    .replace(/\{\{DB_IMAGE_TAG\}\}/g, dbImageVer)
     // Studio dashboard creds — sourced from .env.local at deploy time. Empty
     // string fallback keeps yaml valid; chart inlines them into the
     // chart-generated supabase-dashboard secret.
@@ -1587,6 +1672,99 @@ export async function reloadPostgrest({ kubeconfig }) {
   }
 }
 
+/**
+ * Create the production app super-admin in `auth.users` via GoTrue's admin API.
+ *
+ * The k8s mirror of compose/index.js `createAdminUser`: identical GoTrue
+ * admin-API contract (POST the super_admin payload with the service-role
+ * bearer, idempotent on 422 via the shared admin-user helper), reached via
+ * `kubectl port-forward svc/supabase-supabase-auth` instead of an `ssh -L`
+ * tunnel. Goes direct to GoTrue (auth :9999, path `/admin/users`) — no Kong
+ * `/auth/v1` prefix to strip.
+ *
+ * Without this, a k8s deploy shipped a prod app the operator couldn't log
+ * into: ADMIN_EMAIL/ADMIN_PASSWORD only seeded the Supabase Studio dashboard
+ * basic-auth secret, never an app `auth.users` row.
+ *
+ * @param {{kubeconfig: string, envLocal: Record<string,string>, localPort?: number}} args
+ * @returns {Promise<{success: boolean, message: string}>}
+ */
+export async function provisionAdminUser({ kubeconfig, envLocal, localPort = 15000 }) {
+  const adminEmail = envLocal?.ADMIN_EMAIL;
+  const adminPassword = envLocal?.ADMIN_PASSWORD;
+  const serviceRoleKey = envLocal?.SERVICE_ROLE_KEY ?? envLocal?.SUPABASE_SERVICE_ROLE_KEY;
+  if (!adminEmail || !adminPassword || !serviceRoleKey) {
+    return { success: false, message: 'Admin credentials not found in .env.local' };
+  }
+
+  const env = { ...process.env, KUBECONFIG: kubeconfig };
+  // Background port-forward to GoTrue. Distinct localPort per cluster so the
+  // parallel HA fan-out (primary + standby on one operator host) doesn't race
+  // for the same bind — mirrors the 5000/5001 registry-tunnel split.
+  const pf = spawn(
+    'kubectl',
+    ['-n', 'vibecarbon', 'port-forward', 'svc/supabase-supabase-auth', `${localPort}:9999`],
+    { env, stdio: 'ignore' },
+  );
+  try {
+    const connected = await waitForGotrueHealth(`http://localhost:${localPort}/health`, {
+      attempts: 20,
+      intervalMs: 500,
+    });
+    if (!connected) {
+      return { success: false, message: 'Could not reach GoTrue via kubectl port-forward' };
+    }
+    return await postAdminUser({
+      adminUsersUrl: `http://localhost:${localPort}/admin/users`,
+      serviceRoleKey,
+      adminEmail,
+      adminPassword,
+    });
+  } finally {
+    pf.kill();
+  }
+}
+
+/**
+ * The four `ALTER SYSTEM` settings that turn on continuous WAL archiving for
+ * wal-g on the chart-installed supabase-db.
+ */
+export const WAL_ARCHIVING_SETTINGS = [
+  "ALTER SYSTEM SET archive_mode='on'",
+  "ALTER SYSTEM SET archive_command='bash /etc/postgresql/wal-archive.sh %p'",
+  "ALTER SYSTEM SET archive_timeout='900'",
+  "ALTER SYSTEM SET wal_level='replica'",
+];
+
+/**
+ * Build the `kubectl exec … psql …` argument list that enables WAL archiving.
+ *
+ * Each `ALTER SYSTEM` MUST be its own `-c` option. psql sends a single `-c`
+ * string as one simple-query request, and Postgres runs a multi-statement
+ * simple query inside ONE implicit transaction — where `ALTER SYSTEM` is
+ * rejected with "ALTER SYSTEM cannot run inside a transaction block". The
+ * documented remedy (and the compose/ha.js pattern) is to pass each statement
+ * as a separate `-c`, so every ALTER SYSTEM is its own implicit transaction.
+ *
+ * @param {string} dbPod - e.g. `supabase-supabase-db-0`
+ * @returns {string[]}
+ */
+export function enableWalArchivingPsqlArgs(dbPod) {
+  return [
+    '-n',
+    'vibecarbon',
+    'exec',
+    dbPod,
+    '--',
+    'psql',
+    '-U',
+    'supabase_admin',
+    '-d',
+    'postgres',
+    ...WAL_ARCHIVING_SETTINGS.flatMap((stmt) => ['-c', stmt]),
+  ];
+}
+
 /**
  * Apply cert-manager + vibecarbon-secrets + the project's k8s/infra +
  * k8s/base kustomizations, install supabase via helm, patch the app
@@ -1595,18 +1773,21 @@ export async function reloadPostgrest({ kubeconfig }) {
  * Local-first by default — no Flux on the deploy path; Flux is opt-in
  * via `vibecarbon configure cicd`.
  *
- * @param {{kubeconfig: string, projectDir: string, projectName: string, imageTag: string, backupImageTag?: string, envLocal: Record<string,string>, domain: string, s3Config?: {accessKey: string, secretKey: string, bucket: string, endpoint: string, region: string}, backupBucketName?: string|null, dnsProvider?: string|null, cloudflareApiToken?: string|null, hetznerApiToken?: string|null, apiToken?: string, region?: string, environment?: string, minWorkers?: number, maxWorkers?: number, workerServerType?: string, k3sToken?: string, masterIp?: string, sshKeyPath?: string, khPath?: string}} args
+ * @param {{kubeconfig: string, projectDir: string, projectName: string, imageTag: string, dbImageTag: string, envLocal: Record<string,string>, domain: string, s3Config?: {accessKey: string, secretKey: string, bucket: string, endpoint: string, region: string}, backupBucketName?: string|null, restore?: string|null, dnsProvider?: string|null, cloudflareApiToken?: string|null, hetznerApiToken?: string|null, apiToken?: string, region?: string, environment?: string, minWorkers?: number, maxWorkers?: number, workerServerType?: string, k3sToken?: string, masterIp?: string, sshKeyPath?: string, khPath?: string}} args
+ *   `restore` (when set, e.g. 'latest' or an ISO timestamp) skips applyMigrations
+ *   — the wal-g-restored DB already carries schema_migrations.
  */
 export async function applyK3sManifests({
   kubeconfig,
   projectDir,
   projectName,
   imageTag,
-  backupImageTag,
+  dbImageTag,
   envLocal,
   domain,
   s3Config,
   backupBucketName,
+  restore,
   dnsProvider,
   cloudflareApiToken,
   hetznerApiToken,
@@ -1658,42 +1839,73 @@ export async function applyK3sManifests({
   // Fan out the three deploys in parallel: kubectl wait blocks per-arg
   // sequentially, and the cainjector + webhook usually become Available
   // ~10-30s after the core deploy, so the serial form pays the longest
-  // arrival time three times in a row. Two-stage retry preserves the
-  // outer 180s+120s envelope of the previous --timeout=180s call.
+  // arrival time three times in a row.
+  //
+  // Each deploy POLLS to Available within a generous total budget rather than
+  // one or two fixed `kubectl wait` calls. On a fresh worker, cold image pulls
+  // can push cainjector/webhook readiness past a tight envelope even though the
+  // pod ultimately comes up healthy (observed 2026-06-01: k8s e2e failed
+  // `kubectl wait deploy/cert-manager-cainjector exited 1` while the pod was
+  // 1/1 Running moments later — the old 90s→120s=210s envelope just gave up
+  // first). Re-attempting `kubectl wait` in chunks until the budget also
+  // self-heals the fast-fail race where `wait` runs before the just-applied
+  // Deployment object has propagated (NotFound → exit 1 immediately). A
+  // genuinely stuck deploy still fails once the budget is exhausted.
   const certManagerDeploys = [
     'deploy/cert-manager',
     'deploy/cert-manager-webhook',
     'deploy/cert-manager-cainjector',
   ];
+  // 8-min budget: on a freshly provisioned cluster under load, cert-manager
+  // pods have been observed not even getting created until ~5 min after apply
+  // (control-plane/scheduler lag, not image pull), overrunning a tighter
+  // envelope even though the pods then come up healthy. This margin covers that
+  // cold-start tail; the poll below re-attempts within it rather than giving up.
+  const CERT_MANAGER_READY_BUDGET_MS = 480_000;
+  const waitDeployAvailableOnce = (deploy, timeoutSec) =>
+    new Promise((resolveFn, reject) => {
+      const child = spawn(
+        'kubectl',
+        [
+          '-n',
+          'cert-manager',
+          'wait',
+          '--for=condition=Available',
+          `--timeout=${timeoutSec}s`,
+          deploy,
+        ],
+        { stdio: 'inherit', env },
+      );
+      child.on('error', reject);
+      child.on('exit', (code) =>
+        code === 0 ? resolveFn() : reject(new Error(`kubectl wait ${deploy} exited ${code}`)),
+      );
+    });
+  const waitDeployAvailable = async (deploy) => {
+    const deadline = Date.now() + CERT_MANAGER_READY_BUDGET_MS;
+    let lastErr;
+    while (Date.now() < deadline) {
+      const remainingSec = Math.max(5, Math.ceil((deadline - Date.now()) / 1000));
+      try {
+        await waitDeployAvailableOnce(deploy, Math.min(60, remainingSec));
+        return;
+      } catch (err) {
+        lastErr = err;
+        if (Date.now() >= deadline) break;
+        // Brief backoff so a fast-failing wait (e.g. the Deployment object not
+        // yet propagated → immediate NotFound) doesn't busy-spin the budget.
+        await new Promise((r) => setTimeout(r, 3000));
+      }
+    }
+    throw (
+      lastErr ??
+      new Error(
+        `kubectl wait ${deploy} never became Available within ${CERT_MANAGER_READY_BUDGET_MS / 1000}s`,
+      )
+    );
+  };
   await perfAsync(`deploy.${perfPrefix}.certManager.wait`, () =>
-    Promise.all(
-      certManagerDeploys.map(async (deploy) => {
-        const waitOnce = (timeoutSec) =>
-          new Promise((resolveFn, reject) => {
-            const child = spawn(
-              'kubectl',
-              [
-                '-n',
-                'cert-manager',
-                'wait',
-                '--for=condition=Available',
-                `--timeout=${timeoutSec}s`,
-                deploy,
-              ],
-              { stdio: 'inherit', env },
-            );
-            child.on('error', reject);
-            child.on('exit', (code) =>
-              code === 0 ? resolveFn() : reject(new Error(`kubectl wait ${deploy} exited ${code}`)),
-            );
-          });
-        try {
-          await waitOnce(90);
-        } catch {
-          await waitOnce(120);
-        }
-      }),
-    ),
+    Promise.all(certManagerDeploys.map((deploy) => waitDeployAvailable(deploy))),
   );
   // 2b. Land the per-DNS-provider Secret in the cert-manager namespace
   //     BEFORE the kustomization that defines the issuers, so when the
@@ -1898,55 +2110,21 @@ export async function applyK3sManifests({
   if (!khPath) {
     throw new Error('applyK3sManifests: khPath required for registry push (Phase 6)');
   }
-  // Push app + backup images in parallel. Each opens its own SSH tunnel,
-  // so they need DISTINCT localTunnelPorts — pushImageToLocalRegistry's
-  // teardown is `pkill -f 'ssh.*-L.*<port>:localhost:5000'`, which would
-  // kill a sibling push's tunnel mid-operation if both used the same port.
-  // Offset the backup push by +100 so even the openTunnelOrWalk's 20-port
-  // fallback has 80 ports of headroom before colliding (plus HA's 5001
-  // standby cluster lands on +100=5101, well clear of primary's 5000+19).
-  // RCA: parallelizing without an offset would silently corrupt pushes
-  // (registry returns "blob unknown" on a teardown'd partial upload).
+  // Push the app image to the in-cluster registry so CA-spawned workers
+  // (which don't exist at sideload time) can pull it on first schedule.
+  // The db image is NOT pushed here: it's pinned to the static supabase
+  // node, reaches it via sideload, and the chart references it with
+  // imagePullPolicy: IfNotPresent — CA workers never run postgres.
   const appPushPort = localTunnelPort;
-  const backupPushPort = localTunnelPort + 100;
-  if (!masterIp) {
-    throw new Error('applyK3sManifests: masterIp required for registry push (Phase 6)');
-  }
-  if (!sshKeyPath) {
-    throw new Error('applyK3sManifests: sshKeyPath required for registry push (Phase 6)');
-  }
-  if (!khPath) {
-    throw new Error('applyK3sManifests: khPath required for registry push (Phase 6)');
-  }
-  const pushTasks = [
-    perfAsync(`deploy.${perfPrefix}.registryPush.app`, () =>
-      pushImageToLocalRegistry({
-        tag: imageTag,
-        masterIp,
-        sshKey: sshKeyPath,
-        khPath,
-        localTunnelPort: appPushPort,
-      }),
-    ),
-  ];
-  // 5b'''. Push the backup image too. The backup CronJob's pods may
-  //        eventually schedule onto a CA-spawned worker (no node
-  //        affinity pinning them to the static pool), so the same
-  //        registry-fallback story applies.
-  if (backupImageTag) {
-    pushTasks.push(
-      perfAsync(`deploy.${perfPrefix}.registryPush.backup`, () =>
-        pushImageToLocalRegistry({
-          tag: backupImageTag,
-          masterIp,
-          sshKey: sshKeyPath,
-          khPath,
-          localTunnelPort: backupPushPort,
-        }),
-      ),
-    );
-  }
-  await Promise.all(pushTasks);
+  await perfAsync(`deploy.${perfPrefix}.registryPush.app`, () =>
+    pushImageToLocalRegistry({
+      tag: imageTag,
+      masterIp,
+      sshKey: sshKeyPath,
+      khPath,
+      localTunnelPort: appPushPort,
+    }),
+  );
   // 5a'. Apply cluster-autoscaler manifests SEPARATELY. CA lives in
   //      kube-system but base/'s parent kustomization sets
   //      `namespace: vibecarbon`, whose namespace transformer would
@@ -1979,6 +2157,10 @@ export async function applyK3sManifests({
   // — DNS-01 vs HTTP-01 — cert-manager actually uses; see pickIssuerName.
   const acmeServer = envLocal?.ACME_CA_SERVER || process.env.ACME_CA_SERVER || '';
   const issuerName = pickIssuerName({ dnsProvider, acmeServer });
+  // DNS-01 issuers (cloudflare, hetzner) support wildcard SANs — one cert
+  // covers *.${domain} so every IngressRoute subdomain is included without
+  // separate per-router ACME orders. Manual (HTTP-01) falls back to apex-only.
+  const dnsNames = certificateDnsNames(domain, issuerName);
   await runKubectlWithRetry(
     [
       '-n',
@@ -1989,7 +2171,7 @@ export async function applyK3sManifests({
       '--type=merge',
       '-p',
       JSON.stringify({
-        spec: { dnsNames: [domain], issuerRef: { name: issuerName, kind: 'ClusterIssuer' } },
+        spec: { dnsNames, issuerRef: { name: issuerName, kind: 'ClusterIssuer' } },
       }),
     ],
     { env, description: 'applyK3sManifests: kubectl patch certificate/vibecarbon-tls' },
@@ -2148,53 +2330,109 @@ export async function applyK3sManifests({
     ['-n', 'vibecarbon', 'set', 'image', 'deployment/app', `app=${imageTag}`],
     { env, description: 'applyK3sManifests: kubectl set image deployment/app' },
   );
-  // 6b. Patch the backup CronJob image. base/backup/cronjob.yaml ships
-  //     image=ghcr.io/<owner>/<project>-backup:latest — never pushed in
-  //     direct/local mode, so the scheduled job pod fails ImagePullBackOff.
-  //     deployK3s sideloaded the backup image to every node (step 7b);
-  //     point the CronJob at it. imagePullPolicy: Always in the template
-  //     would defeat the sideload — switch to IfNotPresent at the same
-  //     time so kubelet uses the local copy.
-  if (backupImageTag) {
-    await runKubectlWithRetry(
-      ['-n', 'vibecarbon', 'set', 'image', 'cronjob/backup', `backup=${backupImageTag}`],
-      { env, description: 'applyK3sManifests: kubectl set image cronjob/backup' },
-    );
-    await runKubectlWithRetry(
-      [
-        '-n',
-        'vibecarbon',
-        'patch',
-        'cronjob/backup',
-        '--type=json',
-        '-p',
-        JSON.stringify([
-          {
-            op: 'replace',
-            path: '/spec/jobTemplate/spec/template/spec/containers/0/imagePullPolicy',
-            value: 'IfNotPresent',
-          },
-        ]),
-      ],
-      { env, description: 'applyK3sManifests: kubectl patch cronjob/backup imagePullPolicy' },
-    );
-  }
   // 7. Install Supabase via helm. Blocks until release converges (15m
-  //    timeout in chart). App pods may stay Pending on supabase URLs in
-  //    their config until kong/auth/db are Ready, but that's fine — they
-  //    won't churn ImagePullBackOff because step 6 already set them to
-  //    the sideloaded image.
+  //    timeout in chart). image.db points at the wal-g-equipped db image
+  //    (sideloaded in step 7b), and the chart mounts wal-archive.sh + the
+  //    non-secret WALG_* env from supabase.values.yaml.
   await perfAsync(`deploy.${perfPrefix}.installSupabase`, async () =>
-    installSupabase({ kubeconfig, projectDir, projectName, domain, s3Config, envLocal }),
+    installSupabase({
+      kubeconfig,
+      projectDir,
+      projectName,
+      domain,
+      s3Config,
+      envLocal,
+      dbImageTag,
+      backupBucketName,
+    }),
   );
+  // 7b. Enable continuous WAL archiving on the supabase-db. The chart can't
+  //     inject secret env (environment.db is string-only) or postgres archive
+  //     flags, so we (a) ALTER SYSTEM the archive params (persisted to
+  //     postgresql.auto.conf on the PVC), and (b) inject the SECRET S3 creds
+  //     via a strategic-merge env patch (merged by env name → idempotent;
+  //     re-applied every deploy because helm upgrade strips non-chart env).
+  //     The cred patch rolls the pod when it changes the template; on first
+  //     enable that rollout is what makes postgres pick up archive_mode.
+  //     Skipped without S3 (dev / no backups configured).
+  if (s3Config?.accessKey) {
+    await perfAsync(`deploy.${perfPrefix}.enableWalArchiving`, async () => {
+      const dbSts = 'supabase-supabase-db';
+      const dbPod = `${dbSts}-0`;
+      // S3 creds reach the db container as a mounted file (helm-owned, see
+      // supabase.values.yaml walg-s3-creds volume) — no env patch, so no
+      // db restart on warm deploys. Only the archive params need ALTER
+      // SYSTEM, and archive_mode flips only with a RESTART. So: check the
+      // RUNNING archive_mode; if already 'on' (persisted in
+      // postgresql.auto.conf from a prior enable), this is a warm deploy —
+      // nothing to do. If 'off', set the params and restart ONCE.
+      let archiveMode = '';
+      try {
+        archiveMode = (
+          await runKubectlWithRetry(
+            [
+              '-n',
+              'vibecarbon',
+              'exec',
+              dbPod,
+              '--',
+              'psql',
+              '-U',
+              'supabase_admin',
+              '-d',
+              'postgres',
+              '-tAc',
+              'SHOW archive_mode',
+            ],
+            {
+              env,
+              captureStdout: true,
+              description: 'applyK3sManifests: read archive_mode',
+            },
+          )
+        )
+          ?.toString()
+          .trim();
+      } catch {
+        // Treat an unreadable value as "not yet enabled" — the worst case is
+        // one extra ALTER SYSTEM + restart, which is idempotent.
+      }
+      if (archiveMode === 'on') {
+        console.error('[k3s] WAL archiving already enabled — skipping ALTER SYSTEM + restart.');
+        return;
+      }
+      await runKubectlWithRetry(enableWalArchivingPsqlArgs(dbPod), {
+        env,
+        description: 'applyK3sManifests: enable WAL archiving (ALTER SYSTEM)',
+      });
+      // archive_mode requires a restart (reload is insufficient). Restart the
+      // StatefulSet ONCE; persisted to postgresql.auto.conf on the PVC, so
+      // subsequent deploys hit the archiveMode==='on' fast path above.
+      await runKubectlWithRetry(
+        ['-n', 'vibecarbon', 'rollout', 'restart', `statefulset/${dbSts}`],
+        { env, description: 'applyK3sManifests: restart supabase-db to enable WAL archiving' },
+      );
+      await runKubectlWithRetry(
+        ['-n', 'vibecarbon', 'rollout', 'status', `statefulset/${dbSts}`, '--timeout=300s'],
+        { env, description: 'applyK3sManifests: wait supabase-db rollout (WAL archiving)' },
+      );
+    });
+  }
   // 8. Apply project SQL migrations against the chart-installed postgres.
-  //    Helm's `--wait` returned only after supabase-db was Ready, so the
-  //    pod is guaranteed up here. Without this, the app's
-  //    /api/health/ready returns 503 forever ("Could not find table
-  //    public.organizations").
-  await perfAsync(`deploy.${perfPrefix}.applyMigrations`, async () =>
-    applyMigrations({ kubeconfig, projectDir }),
-  );
+  //    SKIPPED when restoring: a wal-g restore brings the full DB including
+  //    supabase_migrations.schema_migrations, so re-running migrations would
+  //    hit duplicate-key errors (applyMigrations deliberately does not
+  //    swallow them). On a normal deploy, helm's `--wait` guarantees the db
+  //    pod is up; without migrations the app's /api/health/ready 503s.
+  if (restore) {
+    console.error(
+      `[k3s] restore=${restore} requested — skipping applyMigrations (restored DB carries schema_migrations).`,
+    );
+  } else {
+    await perfAsync(`deploy.${perfPrefix}.applyMigrations`, async () =>
+      applyMigrations({ kubeconfig, projectDir }),
+    );
+  }
   // 8b. Reload PostgREST's schema cache so it sees the tables applyMigrations
   //     just created. rest came up during helm --wait (before migrations) and
   //     no DDL-watch trigger exists to auto-NOTIFY — without this, /rest/v1/*
@@ -2202,6 +2440,28 @@ export async function applyK3sManifests({
   await perfAsync(`deploy.${perfPrefix}.reloadPostgrest`, async () =>
     reloadPostgrest({ kubeconfig }),
   );
+  // 8c. Create the production app super-admin via GoTrue's admin API. Mirrors
+  //     the compose path — without it the operator can't log into their own
+  //     deployed app (ADMIN_* only seeds the Studio dashboard secret, never an
+  //     auth.users row). Runs unconditionally (incl. restore: idempotent on
+  //     422, and a backup predating admin-creation still gets seeded). In HA
+  //     both clusters run this against their independently-writable DB at
+  //     deploy time (same as applyMigrations); the localPort is derived from
+  //     localTunnelPort so the parallel primary/standby port-forwards don't
+  //     race for the same bind. Non-fatal: a GoTrue blip warns + retries on
+  //     the next deploy rather than failing an otherwise-healthy one.
+  await perfAsync(`deploy.${perfPrefix}.createAdminUser`, async () => {
+    const adminResult = await provisionAdminUser({
+      kubeconfig,
+      envLocal,
+      localPort: 15000 + (localTunnelPort - 5000),
+    });
+    if (adminResult.success) {
+      console.log(`[k3s] ${adminResult.message}`);
+    } else {
+      console.warn(`[k3s] ${adminResult.message} — re-run \`vibecarbon deploy\` to retry.`);
+    }
+  });
   // 9. Wait for app rollout. The deployment template already has
   //    imagePullPolicy: IfNotPresent, so kubelet uses the sideloaded image
   //    without pulling. 300s wasn't enough on a cold deploy where the app
@@ -2469,37 +2729,23 @@ export async function deployK3s(options) {
   }
   const { tag: imageTag } = state.getStepResult('k3s-build');
 
-  // 7. Build the backup image up-front so we can fan out BOTH sideloads in
-  //    parallel. base/'s CronJob ships image=ghcr.io/<owner>/<project>-backup
-  //    — unreachable in direct/local mode (no GHCR push). Build the
-  //    <projectDir>/backup context locally, then later patch the CronJob
-  //    image in applyK3sManifests (step 5c).
-  const backupBuildInputs = { projectName: `${options.projectName}-backup` };
-  if (!state.shouldSkip('k3s-backup-build', backupBuildInputs)) {
-    state.startStep('k3s-backup-build', backupBuildInputs);
-    s.start('Building backup image');
-    const built = await buildAppImage(join(projectDir, 'backup'), `${options.projectName}-backup`);
-    state.completeStep('k3s-backup-build', {
-      tag: built.tag,
-      gitSha: built.gitSha,
-      isDirty: built.isDirty,
-    });
-    s.stop(`Backup image built: ${built.tag}`);
-  }
-  const { tag: backupImageTag } = state.getStepResult('k3s-backup-build');
+  // 7. The wal-g-equipped db image (supabase/postgres + wal-g) is now a
+  //    pre-published, multi-arch image pulled from ghcr — no per-deploy build
+  //    or sideload. The supabase Helm chart's image.db + the walg-restore init
+  //    container both reference it (see installSupabase + supabase.values.yaml).
+  const dbImageTag = resolveDbImageTag();
 
   // 7b. Sideload to every node (parallel SSH within sideloadK3s) AND fan out
-  //     the app+backup sideloads against each other. Previously sequential:
-  //     sideload.app then sideload.backup ≈ 1m4s + 57s on k8s-ha primary.
-  //     Both writes go through `docker save | ssh ... gunzip | k3s ctr import`
-  //     against disjoint image refs on the same target nodes — kernel disk
-  //     buffer + sshd are happy to multiplex two parallel streams. Cuts the
-  //     critical path to max(app, backup) instead of sum.
+  //     the app+db sideloads against each other. Both writes go through
+  //     `docker save | ssh ... gunzip | k3s ctr import` against disjoint
+  //     image refs on the same target nodes — kernel disk buffer + sshd
+  //     multiplex two parallel streams. Critical path = max(app, db).
+  //     The db image only NEEDS the supabase node (db is pinned there), but
+  //     sideloading to all nodes is harmless and keeps the target list shared.
   const sshTargets = [`root@${masterIp}`];
   if (supabaseIp) sshTargets.push(`root@${supabaseIp}`);
   for (const wIp of workerIps ?? []) sshTargets.push(`root@${wIp}`);
   const sideloadInputs = { imageTag, targets: sshTargets.join(',') };
-  const backupSideloadInputs = { backupImageTag, targets: sshTargets.join(',') };
   const sideloadTasks = [];
   if (!state.shouldSkip('k3s-sideload', sideloadInputs)) {
     state.startStep('k3s-sideload', sideloadInputs);
@@ -2509,23 +2755,15 @@ export async function deployK3s(options) {
       ).then(() => state.completeStep('k3s-sideload')),
     );
   }
-  if (!state.shouldSkip('k3s-backup-sideload', backupSideloadInputs)) {
-    state.startStep('k3s-backup-sideload', backupSideloadInputs);
-    sideloadTasks.push(
-      perfAsync(`deploy.${perfPrefix}.sideload.backup`, () =>
-        sideloadK3s({ tag: backupImageTag, sshTargets, sshKey: privateKeyPath, khPath }),
-      ).then(() => state.completeStep('k3s-backup-sideload')),
-    );
-  }
   if (sideloadTasks.length > 0) {
-    s.start(`Sideloading app + backup to ${sshTargets.length} node(s) in parallel`);
+    s.start(`Sideloading app to ${sshTargets.length} node(s)`);
     await Promise.all(sideloadTasks);
-    s.stop('Sideload complete (app + backup)');
+    s.stop('Sideload complete (app)');
   }
 
   // 8. Apply manifests + supabase Helm release + roll out the app.
-  if (!state.shouldSkip('k3s-apply', { imageTag, backupImageTag })) {
-    state.startStep('k3s-apply', { imageTag, backupImageTag });
+  if (!state.shouldSkip('k3s-apply', { imageTag, dbImageTag, restore: options.restore ?? '' })) {
+    state.startStep('k3s-apply', { imageTag, dbImageTag, restore: options.restore ?? '' });
     s.start('Applying k8s manifests + installing Supabase + rolling out app');
     const envLocal = loadEnvLocal(join(projectDir, '.env.local'));
     await perfAsync(`deploy.${perfPrefix}.applyManifests`, () =>
@@ -2535,11 +2773,12 @@ export async function deployK3s(options) {
         projectDir,
         projectName: options.projectName,
         imageTag,
-        backupImageTag,
+        dbImageTag,
         envLocal,
         domain: options.domain || 'localhost',
         s3Config: options.s3Config,
         backupBucketName: options.backupBucketName,
+        restore: options.restore,
         dnsProvider: options.dnsProvider,
         cloudflareApiToken: options.cloudflareApiToken,
         // Hetzner DNS-01 uses the SAME Cloud API token (api.hetzner.cloud)