vibecarbon 0.2.0 → 0.2.1

package/carbon/docker-compose.yml

@@ -144,7 +144,12 @@ services:
       # PITR gap, but the DB stays up — strictly better than the previous
       # behavior for a SaaS template. Grep `docker logs db` for
       # WAL_ARCHIVE_FAILED to detect silent backup regressions.
-      - archive_command=/etc/postgresql/wal-archive.sh %p
+      # Invoke via `bash` rather than relying on the script's exec bit: the
+      # create-time scaffold copy (copyFileSync) drops +x, and a 0644 script
+      # made postgres fail archiving with exit 126 ("not executable"), which
+      # silently re-pins WAL. bash-invoking it (like reconcile.sh) is immune to
+      # the file mode, so the fault-tolerant wrapper always runs.
+      - archive_command=bash /etc/postgresql/wal-archive.sh %p
       - -c
       # 60s was too aggressive — on an idle Supabase DB, that forces a
       # 16 MiB segment switch every minute regardless of write volume (PG14+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vibecarbon",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Create and manage production-ready Vibecarbon applications",
   "type": "module",
   "bin": "./src/cli.js",

package/src/create.js

@@ -780,6 +780,16 @@ async function bootstrap(cliArgs) {
     join(projectDir, 'volumes/db/set-passwords.sh'),
     variables,
   );
+  // wal-archive.sh is postgres's archive_command. It was missing from this copy
+  // list, so scaffolds shipped no source file — Docker then bind-mounted an
+  // empty DIRECTORY at /etc/postgresql/wal-archive.sh and postgres failed
+  // archiving with exit 126 ("is a directory"), re-pinning WAL. Copy it (the
+  // chmod block below makes it +x; the script has only shell $vars, no {{}}).
+  copyTemplate(
+    'volumes/db/wal-archive.sh',
+    join(projectDir, 'volumes/db/wal-archive.sh'),
+    variables,
+  );
   copyTemplate('volumes/db/jwt.sql', join(projectDir, 'volumes/db/jwt.sql'), variables);
   copyTemplate('volumes/db/realtime.sql', join(projectDir, 'volumes/db/realtime.sql'), variables);
   copyTemplate(
@@ -854,6 +864,11 @@ async function bootstrap(cliArgs) {
     'docker-entrypoint.sh',
     'volumes/kong/docker-entrypoint.sh',
     'volumes/db/set-passwords.sh',
+    // Postgres execs this directly as its archive_command, so it MUST be +x.
+    // copyTemplate (copyFileSync) drops the source's exec bit, and omitting it
+    // here shipped a 0644 script → archive_command failed with exit 126 ("not
+    // executable") → WAL pinned → disk-fill risk the wrapper exists to prevent.
+    'volumes/db/wal-archive.sh',
   ];
   for (const script of shellScripts) {
     const scriptPath = join(projectDir, script);

package/src/destroy.js

@@ -693,7 +693,29 @@ async function handleBackupBucket(envConfig, projectConfig, args, spinner) {
     }
     const s3Provider = new HetznerS3Provider(s3Creds.accessKey, s3Creds.secretKey, region);
     try {
-      const result = await s3Provider.emptyAndDeleteBucket(bucketName);
+      // Bound the purge so a slow/contended S3 endpoint can't consume the whole
+      // destroy budget. The Hetzner resources are already freed (Pulumi destroy
+      // ran before this), but a single S3 op can hang up to ~9 min under the
+      // SDK (maxAttempts 3 × 60s requestTimeout) × _send (maxAttempts 3) retry
+      // layers — observed on k8s teardown, where the storage bucket doubles as
+      // the Pulumi state backend (slow/contended), blowing the 600s
+      // final-destroy timeout and getting the whole step SIGKILLed. On timeout
+      // we leave the bucket for the orphan sweep / next destroy rather than hang.
+      const PURGE_TIMEOUT_MS = 180_000;
+      const result = await Promise.race([
+        s3Provider.emptyAndDeleteBucket(bucketName),
+        new Promise((_, reject) =>
+          setTimeout(
+            () =>
+              reject(
+                new Error(
+                  `backup bucket purge exceeded ${PURGE_TIMEOUT_MS / 1000}s — left for the orphan sweep / a retried destroy`,
+                ),
+              ),
+            PURGE_TIMEOUT_MS,
+          ),
+        ),
+      ]);
       spinner.stop(
         result.deleted
           ? result.objectsRemoved > 0

package/src/lib/deploy/compose/ha.js

@@ -1043,15 +1043,18 @@ export async function deployComposeHA(options) {
     ]),
   );
 
-  // Run migrations on primary only
+  // Run migrations on primary only. Let failures propagate — silently shipping
+  // an empty/partial schema is far worse than a visibly-failed deploy the
+  // operator can retry. The old swallowing try/catch ("non-fatal") masked
+  // exactly that: HA could deploy a schema-less DB that 500s on every DB-backed
+  // route with no error surfaced. runMigrations now waits for supabase_admin AND
+  // the storage schema (storage.buckets.public) before applying with
+  // ON_ERROR_STOP=1, so the first-run races that motivated the swallow are gone
+  // and a failure here is a real one that must abort the deploy.
   onProgress('Running database migrations on primary...');
-  try {
-    await perfAsync('deploy.ha.compose.migrations', () =>
-      runMigrations(primaryIp, sshKeyPath, projectName),
-    );
-  } catch {
-    // Migrations may partially fail on first run, non-fatal
-  }
+  await perfAsync('deploy.ha.compose.migrations', () =>
+    runMigrations(primaryIp, sshKeyPath, projectName),
+  );
 
   // Create admin user in production Supabase (same as single-server deploy)
   onProgress('Creating admin user...');

package/src/lib/deploy/image.js

@@ -92,11 +92,11 @@ export function inspectGitState(cwd) {
  * pod (Phase 1 wired the registries.yaml; Phase 6 wires the push).
  *
  * @param {string} projectDir - Project root with the Dockerfile.
- * @param {{projectName: string, timestamp?: string, rebuild?: boolean, tagPrefix?: string}} options
+ * @param {{projectName: string, timestamp?: string, rebuild?: boolean, tagPrefix?: string, buildArgs?: Record<string,string>}} options
  * @returns {{tag: string, gitSha: string, isDirty: boolean}}
  */
 export function buildLocalImage(projectDir, options) {
-  const { projectName, rebuild = false, tagPrefix } = options;
+  const { projectName, rebuild = false, tagPrefix, buildArgs = {} } = options;
   if (!projectName) {
     throw new Error('buildLocalImage: projectName is required');
   }
@@ -112,6 +112,14 @@ export function buildLocalImage(projectDir, options) {
 
   const args = ['build'];
   if (rebuild) args.push('--no-cache');
+  // Vite inlines import.meta.env.VITE_* at build time, so the browser bundle
+  // bakes whatever these ARGs resolve to. Without passing them the bundle ships
+  // empty VITE_SUPABASE_* and crashes at page load with "Missing Supabase
+  // environment variables" — k8s shipped exactly this because buildAppImage
+  // passed no build args (compose plumbs them via collectComposeBuildArgs).
+  for (const [k, v] of Object.entries(buildArgs)) {
+    if (v !== undefined && v !== null && v !== '') args.push('--build-arg', `${k}=${v}`);
+  }
   args.push('-t', tag, projectDir);
 
   execFileSync('docker', args, { stdio: 'inherit' });

package/src/lib/deploy/k8s/k3s.js

@@ -556,9 +556,25 @@ export function loadEnvLocal(envPath) {
  * @param {boolean} [rebuild]
  * @returns {Promise<{tag: string, gitSha: string, isDirty: boolean}>}
  */
-export async function buildAppImage(projectDir, projectName, rebuild = false) {
+export async function buildAppImage(projectDir, projectName, rebuild = false, domain = undefined) {
   const { buildLocalImage } = await import('../image.js');
-  return buildLocalImage(projectDir, { projectName, rebuild, tagPrefix: '10.0.1.1:5000' });
+  // App image: collect VITE_* build args (incl. VITE_SUPABASE_URL rewritten to
+  // https://api.<domain>) so the browser bundle ships with real config instead
+  // of empty strings. buildLocalImage previously passed NO build args, so the
+  // k8s SPA crashed on load with "Missing Supabase environment variables" —
+  // compose plumbs these via collectComposeBuildArgs, k8s did not. The backup
+  // image build (no frontend) calls this without a domain → no VITE args.
+  let buildArgs = {};
+  if (domain) {
+    const { collectComposeBuildArgs } = await import('../compose/build-args.js');
+    buildArgs = collectComposeBuildArgs(projectDir, { projectName, domain });
+  }
+  return buildLocalImage(projectDir, {
+    projectName,
+    rebuild,
+    tagPrefix: '10.0.1.1:5000',
+    buildArgs,
+  });
 }
 
 /**
@@ -2436,11 +2452,14 @@ export async function deployK3s(options) {
   }
 
   // 6. Build the app image locally
-  const buildInputs = { projectName: options.projectName };
+  // domain is part of the cache key + passed to the build: it's baked into the
+  // browser bundle (VITE_SUPABASE_URL=https://api.<domain>), so a domain change
+  // must force a rebuild rather than serving a stale-domain bundle.
+  const buildInputs = { projectName: options.projectName, domain: options.domain };
   if (!state.shouldSkip('k3s-build', buildInputs)) {
     state.startStep('k3s-build', buildInputs);
     s.start('Building app image (10.0.1.1:5000/...)');
-    const built = await buildAppImage(projectDir, options.projectName);
+    const built = await buildAppImage(projectDir, options.projectName, false, options.domain);
     state.completeStep('k3s-build', {
       tag: built.tag,
       gitSha: built.gitSha,

package/src/lib/deploy/remote-build.js

@@ -36,8 +36,13 @@ export async function buildRemote(ip, sshKeyPath, imageTag, cwd, buildArgs = {})
   // key path via env var rather than string-interpolating into the script
   // to keep shell-escaping concerns out of the picture (paths with spaces
   // or quotes would otherwise break the wrapper).
+  // ServerAliveInterval/CountMax: BuildKit holds a long-lived SSH session for
+  // the build; on a congested/cross-region link it can stall and the http2
+  // session helper drops with "error reading preface ... file already closed".
+  // Keepalives let ssh detect a dead peer (~60s) and fail the attempt cleanly
+  // so the retry below can re-establish, instead of hanging to the build timeout.
   const sshWrapper = `#!/bin/bash
-exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 "$@"
+exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 "$@"
 `;
   writeFileSync(sshWrapperPath, sshWrapper, { mode: 0o755 });
 
@@ -103,15 +108,31 @@ exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=no -o UserKn
     // (compose-ha 2026-05-09T00:38 e2 failure). Treat the false return as a
     // hard failure so the error path captures + reports it.
     const buildT = perfTimer('deploy.image.remoteBuild');
-    const buildOk = runCommand(args, {
-      cwd,
-      stdio: 'inherit',
-      env,
-      timeout: 600000,
-    });
+    // BuildKit-over-SSH is flaky on first connect to a freshly-booted (often
+    // cross-region) VPS: the session helper intermittently drops mid-build with
+    // `http2: server: error reading preface ... file already closed` / an SSH
+    // reset, and runCommand returns false. Retry a few times — each attempt
+    // re-establishes a fresh SSH/BuildKit session, and BuildKit's cache makes
+    // retries cheap. Without this a single transient drop failed the whole
+    // deploy (compose-ha standby) or left no image so `docker compose up` later
+    // hit "pull access denied" (compose scale, e2e loop run #1).
+    const BUILD_ATTEMPTS = 3;
+    let buildOk = false;
+    for (let attempt = 1; attempt <= BUILD_ATTEMPTS; attempt++) {
+      buildOk = runCommand(args, { cwd, stdio: 'inherit', env, timeout: 600000 });
+      if (buildOk !== false) break;
+      if (attempt < BUILD_ATTEMPTS) {
+        console.error(
+          `[remote-build] docker build on ${ip} failed (attempt ${attempt}/${BUILD_ATTEMPTS}) — retrying in ${3 * attempt}s`,
+        );
+        await new Promise((r) => setTimeout(r, 3000 * attempt));
+      }
+    }
     buildT.end();
     if (buildOk === false) {
-      throw new Error(`docker build failed on ${ip} for tag ${imageTag}`);
+      throw new Error(
+        `docker build failed on ${ip} for tag ${imageTag} after ${BUILD_ATTEMPTS} attempts`,
+      );
     }
 
     s.stop(`Image built natively: ${imageTag}`);

package/src/scale.js

@@ -439,9 +439,22 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
       // Reuses the same code that does direct-mode deploy builds.
       if (oldAppImage && isLocalOnlyImageTag(oldAppImage)) {
         s.start(`Building image ${oldAppImage} on new server...`);
-        await perfAsync('scale.sideloadImage', () =>
+        const built = await perfAsync('scale.sideloadImage', () =>
           buildRemote(newIp, sshKeyPath, oldAppImage, process.cwd()),
         );
+        // buildRemote returns false on failure (after its own retries). The app
+        // image is local-only, so if it isn't built on the new server, the
+        // `docker compose up` below pulls it and dies ~a minute later with
+        // "pull access denied / repository does not exist" — a confusing
+        // downstream symptom. Fail loudly here instead; this guard was missing
+        // (the HA deploy path has the equivalent check). RCA: e2e loop run #1.
+        if (!built) {
+          s.stop('Remote image build failed on new server', 1);
+          throw new Error(
+            `scale: failed to build local app image ${oldAppImage} on new server ${newIp} ` +
+              `(buildRemote exhausted its retries) — aborting before compose up.`,
+          );
+        }
         s.stop('Image built on new server');
       }