vibecarbon 0.2.0 → 0.3.0

package/carbon/.env.example

@@ -218,3 +218,31 @@ VITE_ENVIRONMENT="development"
 
 # App version shown in Super Admin (set automatically during deployment)
 VITE_APP_VERSION="0.1.0"
+
+# =============================================================================
+# CONNECTION POOLING (Supavisor — production)
+# =============================================================================
+# The internal Supabase stack (auth, rest, realtime, storage, meta) connects to
+# Postgres DIRECTLY and must stay that way — a transaction pooler breaks
+# realtime's replication slot, PostgREST's NOTIFY-based schema reload, and
+# migration advisory locks.
+#
+# Supavisor (started by docker-compose.prod.yml) is the pooler for EXTERNAL
+# direct-DB clients — e.g. a BI tool, a one-off script, or a serverless function
+# you connect to your database yourself. Nothing internal routes through it.
+#
+# Supavisor needs VAULT_ENC_KEY (vault encryption) and REALTIME_SECRET
+# (SECRET_KEY_BASE, also used by Realtime). `vibecarbon create` AUTO-GENERATES
+# both into .env.local alongside JWT_SECRET / POSTGRES_PASSWORD — you do not set
+# them by hand. The `:-default` fallbacks in docker-compose.prod.yml are dev-only
+# safety nets that a created project never uses.
+#
+# External connection strings (POOLER_TENANT_ID defaults to PROJECT_NAME; the
+# username carries the tenant id):
+#   Transaction mode (short, stateless queries / serverless):
+#     postgres://postgres.<PROJECT_NAME>:<POSTGRES_PASSWORD>@<host>:5432/postgres
+#   Session mode (needs prepared statements / LISTEN-NOTIFY / advisory locks):
+#     postgres://postgres.<PROJECT_NAME>:<POSTGRES_PASSWORD>@<host>:6543/postgres
+#
+# NOTE: the pooler ships configured but is not yet exercised by an automated
+# end-to-end test — validate connectivity on a real deploy before relying on it.

package/carbon/docker-compose.yml

@@ -144,7 +144,12 @@ services:
       # PITR gap, but the DB stays up — strictly better than the previous
       # behavior for a SaaS template. Grep `docker logs db` for
       # WAL_ARCHIVE_FAILED to detect silent backup regressions.
-      - archive_command=/etc/postgresql/wal-archive.sh %p
+      # Invoke via `bash` rather than relying on the script's exec bit: the
+      # create-time scaffold copy (copyFileSync) drops +x, and a 0644 script
+      # made postgres fail archiving with exit 126 ("not executable"), which
+      # silently re-pins WAL. bash-invoking it (like reconcile.sh) is immune to
+      # the file mode, so the fault-tolerant wrapper always runs.
+      - archive_command=bash /etc/postgresql/wal-archive.sh %p
       - -c
       # 60s was too aggressive — on an idle Supabase DB, that forces a
       # 16 MiB segment switch every minute regardless of write volume (PG14+
@@ -166,6 +171,14 @@ services:
       # failover — pg_promote retried 5 times against an unreachable
       # standby psql before aborting.
       - hot_standby=on
+      - -c
+      # Explicit connection ceiling (image default is 100). Real consumers —
+      # PostgREST pool + auth + storage + realtime + meta + Supavisor's server
+      # pool + replication + superuser_reserved — sum to ~65; 200 leaves
+      # headroom for the HA standby and scale-ups so the ceiling can't be hit.
+      # Connection pooling for EXTERNAL direct-DB clients is Supavisor (prod
+      # overlay); the internal stack connects directly. See docs/security.md.
+      - max_connections=200
     environment:
       POSTGRES_HOST: /var/run/postgresql
       PGPORT: 5432
@@ -325,6 +338,11 @@ services:
       PGRST_DB_USE_LEGACY_GUCS: "false"
       PGRST_APP_SETTINGS_JWT_SECRET: ${JWT_SECRET}
       PGRST_APP_SETTINGS_JWT_EXP: 3600
+      # Explicit DB connection pool (= PostgREST's default, made intentional)
+      # plus a fail-fast acquisition timeout so a saturated pool returns a clear
+      # error instead of hanging. Part of the max_connections budget on `db`.
+      PGRST_DB_POOL: "10"
+      PGRST_DB_POOL_ACQUISITION_TIMEOUT: "10"
     command: postgrest
     depends_on:
       db:

package/carbon/src/server/index.ts

@@ -6,6 +6,7 @@ import { logger as honoLogger } from 'hono/logger';
 import { secureHeaders } from 'hono/secure-headers';
 import { timeout } from 'hono/timeout';
 import { env } from './lib/env';
+import { decodeAalFromJwt } from './lib/jwt';
 import { logger } from './lib/logger';
 import { createRateLimiter } from './lib/rate-limiter';
 import { getSupabaseClient } from './lib/supabase';
@@ -138,6 +139,7 @@ app.use('/api/*', async (c, next) => {
 app.use('/api/*', async (c, next) => {
   if (c.req.path === '/api/health' || c.req.path === '/api/health/ready') {
     c.set('user', null);
+    c.set('aal', null);
     await next();
     return;
   }
@@ -154,11 +156,15 @@ app.use('/api/*', async (c, next) => {
     } = await supabase.auth.getUser();
     if (!error && user) {
       c.set('user', user);
+      // Token signature already verified by getUser() above — safe to read claims.
+      c.set('aal', decodeAalFromJwt(authHeader));
     } else {
       c.set('user', null);
+      c.set('aal', null);
     }
   } else {
     c.set('user', null);
+    c.set('aal', null);
   }
 
   await next();

package/carbon/src/server/lib/jwt.ts

@@ -0,0 +1,27 @@
+/** Supabase Authenticator Assurance Level. */
+export type Aal = 'aal1' | 'aal2';
+
+/**
+ * Read the `aal` (Authenticator Assurance Level) claim from a Supabase JWT.
+ *
+ * This does NOT verify the signature — callers must only pass a token that has
+ * already been verified upstream (the session middleware validates it via
+ * `supabase.auth.getUser()` before decoding here). It is a pure payload read.
+ *
+ * Returns 'aal1' | 'aal2', or null when the token is absent, malformed, or
+ * carries no recognizable aal claim.
+ */
+export function decodeAalFromJwt(token: string | undefined): Aal | null {
+  if (!token) return null;
+  const jwt = token.startsWith('Bearer ') ? token.slice('Bearer '.length) : token;
+  const parts = jwt.split('.');
+  if (parts.length !== 3) return null;
+  try {
+    const payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')) as {
+      aal?: unknown;
+    };
+    return payload.aal === 'aal1' || payload.aal === 'aal2' ? payload.aal : null;
+  } catch {
+    return null;
+  }
+}

package/carbon/src/server/lib/mfa-settings.ts

@@ -0,0 +1,40 @@
+import { logger } from './logger';
+import { supabaseAdmin } from './supabase';
+
+const TTL_MS = 60_000;
+let cache: { value: boolean; expiresAt: number } | null = null;
+
+/** Test-only: clear the cached global-MFA flag. */
+export function __resetMfaSettingsCache(): void {
+  cache = null;
+}
+
+/**
+ * Whether MFA is enabled application-wide (app_settings.mfa_enabled.enabled).
+ *
+ * Cached for 60s to keep the aal2 gate off the per-request DB hot path. On a
+ * read failure this FAILS OPEN — returns the last-known value, or false (gate
+ * inert) if none — because this control is opt-in and defaults off, so failing
+ * open only restores pre-feature behavior. A held aal2 session is always
+ * honored by the gate regardless of this value.
+ */
+export async function isMfaGloballyEnabled(): Promise<boolean> {
+  if (cache && cache.expiresAt > Date.now()) {
+    return cache.value;
+  }
+  try {
+    const { data, error } = await supabaseAdmin
+      .from('app_settings')
+      .select('value')
+      .eq('key', 'mfa_enabled')
+      .maybeSingle();
+    if (error) throw error;
+    const value = (data?.value ?? null) as { enabled?: boolean } | null;
+    const enabled = value?.enabled === true;
+    cache = { value: enabled, expiresAt: Date.now() + TTL_MS };
+    return enabled;
+  } catch (err) {
+    logger.warn({ err }, 'mfa-settings: failed to read app_settings.mfa_enabled; failing open');
+    return cache?.value ?? false;
+  }
+}

package/carbon/src/server/middleware/requireAal2.ts

@@ -0,0 +1,49 @@
+import type { Context, Next } from 'hono';
+import { isMfaGloballyEnabled } from '../lib/mfa-settings';
+import type { HonoVariables } from '../types';
+
+type Ctx = Context<{ Variables: HonoVariables }>;
+
+/**
+ * Shared decision for aal2 gating. Returns a Response to short-circuit with, or
+ * null when the request may proceed.
+ *
+ * Order: no user -> 401; already aal2 -> allow; MFA globally disabled -> allow
+ * (gate inert, respects the operator toggle); otherwise -> 403 mfa_required.
+ */
+export async function checkAal2(c: Ctx): Promise<Response | null> {
+  const user = c.get('user');
+  if (!user) {
+    return c.json({ error: 'Unauthorized' }, 401);
+  }
+
+  const aal = c.get('aal');
+  if (aal === 'aal2') {
+    return null;
+  }
+
+  if (!(await isMfaGloballyEnabled())) {
+    return null;
+  }
+
+  return c.json({ error: 'mfa_required', current_aal: aal ?? 'aal1', aal_required: 'aal2' }, 403);
+}
+
+/**
+ * Hono middleware: gate a route behind aal2.
+ *   v1Routes.delete('/me', requireAal2, handler)
+ */
+export async function requireAal2(c: Ctx, next: Next): Promise<Response | undefined> {
+  const blocked = await checkAal2(c);
+  if (blocked) return blocked;
+  await next();
+}
+
+/**
+ * Inline guard for conditional gating inside a handler (returns a Response to
+ * return early with, or null to proceed):
+ *   if (body.role === 'OWNER') { const b = await assertAal2(c); if (b) return b; }
+ */
+export function assertAal2(c: Ctx): Promise<Response | null> {
+  return checkAal2(c);
+}

package/carbon/src/server/routes/v1/billing.ts

@@ -6,6 +6,7 @@ import { getBillingProvider, getProviderPriceIds, isBillingConfigured } from '..
 import { env } from '../../lib/env';
 import { sanitizeError } from '../../lib/errors';
 import { supabaseAdmin } from '../../lib/supabase';
+import { requireAal2 } from '../../middleware/requireAal2';
 import type { HonoVariables } from '../../types';
 
 // Helper to access tables not yet in generated types
@@ -119,7 +120,16 @@ async function getOrCreateCustomer(
   // Create database record
   // NOTE: stripe_customer_id column stores the provider's customer ID regardless
   // of which provider is active. The column name is kept for backward compatibility.
-  const insertData =
+  // Single explicit shape (not a discriminated union of two object literals):
+  // PostgREST's insert overload rejects the union because the org branch's
+  // `user_id?: undefined` is incompatible with the user branch's `user_id: string`.
+  // `organizationId` is guaranteed defined in the org path by the guard above.
+  const insertData: {
+    user_id?: string;
+    organization_id?: string;
+    stripe_customer_id: string;
+    email: string;
+  } =
     type === 'user'
       ? { user_id: userId, stripe_customer_id: providerCustomerId, email }
       : { organization_id: organizationId, stripe_customer_id: providerCustomerId, email };
@@ -355,7 +365,7 @@ billingRoutes.get('/subscription', async (c) => {
 /**
  * Create a checkout session via the active billing provider
  */
-billingRoutes.post('/checkout', async (c) => {
+billingRoutes.post('/checkout', requireAal2, async (c) => {
   const user = c.get('user');
 
   if (!user) {
@@ -462,7 +472,7 @@ const setupSchema = z.object({
  * method collection differently (during checkout or via their portal).
  * For non-Stripe providers, this endpoint returns 501.
  */
-billingRoutes.post('/setup', async (c) => {
+billingRoutes.post('/setup', requireAal2, async (c) => {
   const user = c.get('user');
 
   if (!user) {
@@ -559,7 +569,7 @@ billingRoutes.post('/setup', async (c) => {
 /**
  * Create a customer portal session via the active billing provider
  */
-billingRoutes.post('/portal', async (c) => {
+billingRoutes.post('/portal', requireAal2, async (c) => {
   const user = c.get('user');
 
   if (!user) {

package/carbon/src/server/routes/v1/index.ts

@@ -7,6 +7,7 @@ import { sendEmail } from '../../lib/email';
 import { sanitizeError } from '../../lib/errors';
 import { logger } from '../../lib/logger';
 import { type getSupabaseClient, supabaseAdmin } from '../../lib/supabase';
+import { assertAal2, requireAal2 } from '../../middleware/requireAal2';
 import type { HonoVariables } from '../../types';
 
 // Helper to access tables not yet in generated types (notifications, etc.)
@@ -145,7 +146,7 @@ v1Routes.get('/me', async (c) => {
 });
 
 // Delete current user's account
-v1Routes.delete('/me', async (c) => {
+v1Routes.delete('/me', requireAal2, async (c) => {
   const user = c.get('user');
 
   if (!user) {
@@ -534,6 +535,15 @@ v1Routes.patch('/organizations/:orgId/members/:userId', async (c) => {
     return c.json({ error: 'Invalid JSON body' }, 400);
   }
 
+  // Promoting/transferring a member to OWNER is a high-privilege action —
+  // require an aal2 (MFA-validated) session. Checked here, before the authz
+  // lookups: an unauthorized caller is still rejected by the role checks below,
+  // and gating early keeps the check cheap and trivially testable.
+  if (body.role === 'OWNER') {
+    const blocked = await assertAal2(c);
+    if (blocked) return blocked;
+  }
+
   // Get current user's role
   const currentUserRole = await getUserOrgRole(supabase, user.id, orgId);
 
@@ -675,14 +685,15 @@ v1Routes.delete('/organizations/:orgId/members/:userId', async (c) => {
 });
 
 // Delete an organization (OWNER only)
-v1Routes.delete('/organizations/:orgId', async (c) => {
+v1Routes.delete('/organizations/:orgId', requireAal2, async (c) => {
   const user = c.get('user');
 
   if (!user) {
     return c.json({ error: 'Unauthorized' }, 401);
   }
 
-  const orgId = c.req.param('orgId');
+  // biome-ignore lint/style/noNonNullAssertion: orgId is always defined when this route matches
+  const orgId = c.req.param('orgId')!;
   const supabase = c.get('supabase');
 
   // Only the owner can delete

package/carbon/src/server/types.ts

@@ -8,4 +8,6 @@ import type { getSupabaseClient } from './lib/supabase';
 export type HonoVariables = {
   user: User | null;
   supabase: ReturnType<typeof getSupabaseClient>;
+  /** Authenticator Assurance Level of the current session, or null if unknown. */
+  aal: 'aal1' | 'aal2' | null;
 };

package/carbon/tests/integration/server/middleware/require-aal2.test.ts

@@ -0,0 +1,123 @@
+import { Hono } from 'hono';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import type { HonoVariables } from '@server/types';
+
+const { maybeSingleMock, warnMock } = vi.hoisted(() => ({
+  maybeSingleMock: vi.fn(),
+  warnMock: vi.fn(),
+}));
+
+vi.mock('@server/lib/supabase', () => ({
+  supabaseAdmin: {
+    from: () => ({ select: () => ({ eq: () => ({ maybeSingle: maybeSingleMock }) }) }),
+  },
+}));
+vi.mock('@server/lib/logger', () => ({
+  logger: { error: vi.fn(), info: vi.fn(), warn: warnMock, debug: vi.fn() },
+}));
+
+const { requireAal2, assertAal2 } = await import('@server/middleware/requireAal2');
+const { __resetMfaSettingsCache } = await import('@server/lib/mfa-settings');
+
+type Aal = 'aal1' | 'aal2' | null;
+
+// Build a fresh app whose injector middleware seeds user + aal BEFORE the
+// gated route (mountRoute can't be used here — it adds no pre-route middleware).
+function gatedApp(opts: { user: unknown; aal: Aal }) {
+  const app = new Hono<{ Variables: HonoVariables }>();
+  app.use('*', async (c, next) => {
+    // biome-ignore lint/suspicious/noExplicitAny: test injects a minimal user
+    c.set('user', opts.user as any);
+    c.set('aal', opts.aal);
+    await next();
+  });
+  app.delete('/gated', requireAal2, (c) => c.json({ ok: true }));
+  return app;
+}
+
+const aUser = { id: 'user-1', email: 'u@test.com' };
+
+function enableMfa() {
+  maybeSingleMock.mockResolvedValue({ data: { value: { enabled: true } }, error: null });
+}
+function disableMfa() {
+  maybeSingleMock.mockResolvedValue({ data: { value: { enabled: false } }, error: null });
+}
+
+beforeEach(() => {
+  __resetMfaSettingsCache();
+  maybeSingleMock.mockReset();
+  warnMock.mockReset();
+});
+
+describe('requireAal2', () => {
+  it('passes an aal2 session through (no settings read needed)', async () => {
+    const res = await gatedApp({ user: aUser, aal: 'aal2' }).request('/gated', { method: 'DELETE' });
+    expect(res.status).toBe(200);
+    expect(maybeSingleMock).not.toHaveBeenCalled();
+  });
+
+  it('blocks an aal1 session with 403 mfa_required when MFA is globally enabled', async () => {
+    enableMfa();
+    const res = await gatedApp({ user: aUser, aal: 'aal1' }).request('/gated', { method: 'DELETE' });
+    expect(res.status).toBe(403);
+    expect(await res.json()).toEqual({
+      error: 'mfa_required',
+      current_aal: 'aal1',
+      aal_required: 'aal2',
+    });
+  });
+
+  it('reports current_aal as aal1 when the session carries no aal claim', async () => {
+    enableMfa();
+    const res = await gatedApp({ user: aUser, aal: null }).request('/gated', { method: 'DELETE' });
+    expect(res.status).toBe(403);
+    expect(await res.json()).toEqual({
+      error: 'mfa_required',
+      current_aal: 'aal1',
+      aal_required: 'aal2',
+    });
+  });
+
+  it('passes an aal1 session through when MFA is globally disabled (gate inert)', async () => {
+    disableMfa();
+    const res = await gatedApp({ user: aUser, aal: 'aal1' }).request('/gated', { method: 'DELETE' });
+    expect(res.status).toBe(200);
+    expect(await res.json()).toEqual({ ok: true });
+  });
+
+  it('returns 401 when there is no authenticated user', async () => {
+    const res = await gatedApp({ user: null, aal: null }).request('/gated', { method: 'DELETE' });
+    expect(res.status).toBe(401);
+    expect(await res.json()).toEqual({ error: 'Unauthorized' });
+  });
+
+  it('fails open (allows) and warns when the settings read throws', async () => {
+    maybeSingleMock.mockRejectedValueOnce(new Error('db down'));
+    const res = await gatedApp({ user: aUser, aal: 'aal1' }).request('/gated', { method: 'DELETE' });
+    expect(res.status).toBe(200);
+    expect(warnMock).toHaveBeenCalledWith(
+      expect.objectContaining({ err: expect.any(Error) }),
+      expect.stringContaining('mfa-settings'),
+    );
+  });
+
+  it('assertAal2 returns a 403 inline for aal1 when MFA is enabled', async () => {
+    enableMfa();
+    const app = new Hono<{ Variables: HonoVariables }>();
+    app.use('*', async (c, next) => {
+      // biome-ignore lint/suspicious/noExplicitAny: minimal injected user
+      c.set('user', aUser as any);
+      c.set('aal', 'aal1');
+      await next();
+    });
+    app.post('/inline', async (c) => {
+      const blocked = await assertAal2(c);
+      if (blocked) return blocked;
+      return c.json({ ok: true });
+    });
+    const res = await app.request('/inline', { method: 'POST' });
+    expect(res.status).toBe(403);
+    expect((await res.json()).error).toBe('mfa_required');
+  });
+});

package/carbon/tests/integration/server/routes/billing-aal2.test.ts

@@ -0,0 +1,63 @@
+import { Hono } from 'hono';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { jsonPost } from '../../../_helpers/app';
+import type { HonoVariables } from '@server/types';
+
+const { maybeSingleMock } = vi.hoisted(() => ({ maybeSingleMock: vi.fn() }));
+
+vi.mock('@server/lib/supabase', () => ({
+  supabaseAdmin: {
+    from: () => ({ select: () => ({ eq: () => ({ maybeSingle: maybeSingleMock }) }) }),
+  },
+}));
+vi.mock('@server/lib/logger', () => ({
+  logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn(), debug: vi.fn() },
+}));
+// Billing is "configured" so the gate (which runs first) is what we observe,
+// not a 503. getBillingProvider is never reached because the gate short-circuits.
+vi.mock('@server/billing', () => ({
+  isBillingConfigured: () => true,
+  getBillingProvider: () => ({}),
+  getProviderPriceIds: () => ({}),
+}));
+
+const { billingRoutes } = await import('@server/routes/v1/billing');
+const { __resetMfaSettingsCache } = await import('@server/lib/mfa-settings');
+
+function appWith(aal: 'aal1' | 'aal2') {
+  const app = new Hono<{ Variables: HonoVariables }>();
+  app.use('*', async (c, next) => {
+    // biome-ignore lint/suspicious/noExplicitAny: minimal injected user
+    c.set('user', { id: 'u1', email: 'u@test.com' } as any);
+    c.set('aal', aal);
+    await next();
+  });
+  app.route('/api/v1/billing', billingRoutes);
+  return app;
+}
+
+beforeEach(() => {
+  __resetMfaSettingsCache();
+  maybeSingleMock.mockReset();
+  maybeSingleMock.mockResolvedValue({ data: { value: { enabled: true } }, error: null });
+});
+
+describe('billing endpoints are aal2-gated when MFA is globally enabled', () => {
+  for (const path of ['/checkout', '/portal', '/setup']) {
+    it(`POST ${path} returns 403 mfa_required for an aal1 session`, async () => {
+      const res = await appWith('aal1').request(
+        `/api/v1/billing${path}`,
+        jsonPost({ type: 'user' }),
+      );
+      expect(res.status).toBe(403);
+      expect((await res.json()).error).toBe('mfa_required');
+    });
+  }
+
+  it('POST /checkout is NOT gated for an aal2 session (gate passes through)', async () => {
+    const res = await appWith('aal2').request('/api/v1/billing/checkout', jsonPost({ type: 'user' }));
+    // Gate satisfied → handler runs. Whatever it returns, it is NOT the mfa_required 403.
+    const body = await res.json();
+    expect(body.error).not.toBe('mfa_required');
+  });
+});

package/carbon/tests/integration/server/routes/v1-aal2.test.ts

@@ -0,0 +1,102 @@
+import { Hono } from 'hono';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import type { HonoVariables } from '@server/types';
+
+const { maybeSingleMock } = vi.hoisted(() => ({ maybeSingleMock: vi.fn() }));
+
+// supabaseAdmin is used both for the app_settings read (gate) and by handlers.
+// The gate short-circuits gated paths before handlers run, so an app_settings-
+// shaped chain is enough. The PATCH "skip" case proceeds into the handler,
+// which calls getUserOrgRole on the USER client (c.get('supabase')) — injected
+// separately below.
+vi.mock('@server/lib/supabase', () => ({
+  supabaseAdmin: {
+    from: () => ({ select: () => ({ eq: () => ({ maybeSingle: maybeSingleMock }) }) }),
+  },
+}));
+vi.mock('@server/lib/logger', () => ({
+  logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn(), debug: vi.fn() },
+}));
+vi.mock('@server/lib/email', () => ({ sendEmail: vi.fn() }));
+
+const { v1Routes } = await import('@server/routes/v1/index');
+const { __resetMfaSettingsCache } = await import('@server/lib/mfa-settings');
+
+// User-scoped supabase stub whose membership lookups resolve to "no role", so a
+// request that passes the gate cleanly 404s instead of crashing.
+const noRoleSupabase = {
+  from: () => ({
+    select: () => ({ eq: () => ({ eq: () => ({ single: async () => ({ data: null }) }) }) }),
+  }),
+};
+
+function appWith(aal: 'aal1' | 'aal2', user: unknown = { id: 'u1', email: 'u@test.com' }) {
+  const app = new Hono<{ Variables: HonoVariables }>();
+  app.use('*', async (c, next) => {
+    // biome-ignore lint/suspicious/noExplicitAny: minimal injected context
+    c.set('user', user as any);
+    c.set('aal', aal);
+    // biome-ignore lint/suspicious/noExplicitAny: minimal user-client stub
+    c.set('supabase', noRoleSupabase as any);
+    await next();
+  });
+  app.route('/api/v1', v1Routes);
+  return app;
+}
+
+function jsonPatch(body: unknown): RequestInit {
+  return { method: 'PATCH', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) };
+}
+
+beforeEach(() => {
+  __resetMfaSettingsCache();
+  maybeSingleMock.mockReset();
+  maybeSingleMock.mockResolvedValue({ data: { value: { enabled: true } }, error: null });
+});
+
+describe('account + org endpoints are aal2-gated when MFA is globally enabled', () => {
+  it('DELETE /me returns 403 mfa_required for aal1', async () => {
+    const res = await appWith('aal1').request('/api/v1/me', { method: 'DELETE' });
+    expect(res.status).toBe(403);
+    expect((await res.json()).error).toBe('mfa_required');
+  });
+
+  it('DELETE /organizations/:id returns 403 mfa_required for aal1', async () => {
+    const res = await appWith('aal1').request('/api/v1/organizations/org-1', { method: 'DELETE' });
+    expect(res.status).toBe(403);
+    expect((await res.json()).error).toBe('mfa_required');
+  });
+
+  it('PATCH member role -> OWNER returns 403 mfa_required for aal1 (transfer gated)', async () => {
+    const res = await appWith('aal1').request(
+      '/api/v1/organizations/org-1/members/u2',
+      jsonPatch({ role: 'OWNER' }),
+    );
+    expect(res.status).toBe(403);
+    expect((await res.json()).error).toBe('mfa_required');
+  });
+
+  it('PATCH member role -> ADMIN is NOT blocked by the MFA gate for aal1', async () => {
+    const res = await appWith('aal1').request(
+      '/api/v1/organizations/org-1/members/u2',
+      jsonPatch({ role: 'ADMIN' }),
+    );
+    // Proceeds past the gate into the handler, which 404s on the null-role
+    // membership lookup. The point: it is NOT the mfa_required 403.
+    const body = await res.json();
+    expect(body.error).not.toBe('mfa_required');
+  });
+
+  it('aal2 session passes the gate and reaches the handler on DELETE /organizations/:id', async () => {
+    // Same route as the aal1 -> 403 test above. With aal2 the gate is satisfied
+    // and the handler runs; getUserOrgRole (on the null-role stub) then returns
+    // 404 "not found / access denied". Asserting the concrete 404 — not merely
+    // "not 403" — proves the handler actually executed past the gate. The aal1
+    // companion test (403 mfa_required on this same route) is what guards
+    // against the middleware being accidentally removed: if it were, aal1 would
+    // also reach the handler and return 404 instead of 403.
+    const del = await appWith('aal2').request('/api/v1/organizations/org-1', { method: 'DELETE' });
+    expect(del.status).toBe(404);
+    expect((await del.json()).error).not.toBe('mfa_required');
+  });
+});

package/carbon/tests/unit/server/jwt.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, it } from 'vitest';
+import { decodeAalFromJwt } from '@server/lib/jwt';
+
+// Build a JWT with the given payload. Signature is irrelevant — decodeAalFromJwt
+// never verifies it (the real session middleware already did via getUser()).
+function makeJwt(payload: Record<string, unknown>): string {
+  const b64 = (o: unknown) => Buffer.from(JSON.stringify(o)).toString('base64url');
+  return `${b64({ alg: 'HS256', typ: 'JWT' })}.${b64(payload)}.fakesig`;
+}
+
+describe('decodeAalFromJwt', () => {
+  it('returns aal2 for an aal2 token', () => {
+    expect(decodeAalFromJwt(makeJwt({ aal: 'aal2', sub: 'u1' }))).toBe('aal2');
+  });
+
+  it('returns aal1 for an aal1 token', () => {
+    expect(decodeAalFromJwt(makeJwt({ aal: 'aal1', sub: 'u1' }))).toBe('aal1');
+  });
+
+  it('returns null when the aal claim is absent', () => {
+    expect(decodeAalFromJwt(makeJwt({ sub: 'u1' }))).toBeNull();
+  });
+
+  it('returns null for an unexpected aal value', () => {
+    expect(decodeAalFromJwt(makeJwt({ aal: 'aal3' }))).toBeNull();
+  });
+
+  it('returns null for a non-JWT string (wrong segment count)', () => {
+    expect(decodeAalFromJwt('not-a-jwt')).toBeNull();
+  });
+
+  it('returns null for malformed base64/JSON without throwing', () => {
+    expect(decodeAalFromJwt('aaa.@@@notbase64@@@.bbb')).toBeNull();
+  });
+
+  it('returns null for undefined/empty input', () => {
+    expect(decodeAalFromJwt(undefined)).toBeNull();
+    expect(decodeAalFromJwt('')).toBeNull();
+  });
+
+  it('tolerates a leading "Bearer " prefix', () => {
+    expect(decodeAalFromJwt(`Bearer ${makeJwt({ aal: 'aal2' })}`)).toBe('aal2');
+  });
+});

package/carbon/tests/unit/server/mfa-settings.test.ts

@@ -0,0 +1,64 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { maybeSingleMock, warnMock } = vi.hoisted(() => ({
+  maybeSingleMock: vi.fn(),
+  warnMock: vi.fn(),
+}));
+
+// supabaseAdmin.from('app_settings').select('value').eq('key','mfa_enabled').maybeSingle()
+vi.mock('@server/lib/supabase', () => ({
+  supabaseAdmin: {
+    from: () => ({ select: () => ({ eq: () => ({ maybeSingle: maybeSingleMock }) }) }),
+  },
+}));
+
+vi.mock('@server/lib/logger', () => ({
+  logger: { error: vi.fn(), info: vi.fn(), warn: warnMock, debug: vi.fn() },
+}));
+
+const { isMfaGloballyEnabled, __resetMfaSettingsCache } = await import('@server/lib/mfa-settings');
+
+beforeEach(() => {
+  __resetMfaSettingsCache();
+  maybeSingleMock.mockReset();
+  warnMock.mockReset();
+});
+
+describe('isMfaGloballyEnabled', () => {
+  it('returns true when the setting is { enabled: true }', async () => {
+    maybeSingleMock.mockResolvedValueOnce({ data: { value: { enabled: true } }, error: null });
+    expect(await isMfaGloballyEnabled()).toBe(true);
+  });
+
+  it('returns false when the setting is { enabled: false }', async () => {
+    maybeSingleMock.mockResolvedValueOnce({ data: { value: { enabled: false } }, error: null });
+    expect(await isMfaGloballyEnabled()).toBe(false);
+  });
+
+  it('returns false when the row is missing', async () => {
+    maybeSingleMock.mockResolvedValueOnce({ data: null, error: null });
+    expect(await isMfaGloballyEnabled()).toBe(false);
+  });
+
+  it('caches within the TTL — a second call issues no query', async () => {
+    maybeSingleMock.mockResolvedValueOnce({ data: { value: { enabled: true } }, error: null });
+    expect(await isMfaGloballyEnabled()).toBe(true);
+    expect(await isMfaGloballyEnabled()).toBe(true);
+    expect(maybeSingleMock).toHaveBeenCalledTimes(1);
+  });
+
+  it('__resetMfaSettingsCache forces a re-query', async () => {
+    maybeSingleMock.mockResolvedValueOnce({ data: { value: { enabled: true } }, error: null });
+    expect(await isMfaGloballyEnabled()).toBe(true);
+    __resetMfaSettingsCache();
+    maybeSingleMock.mockResolvedValueOnce({ data: { value: { enabled: false } }, error: null });
+    expect(await isMfaGloballyEnabled()).toBe(false);
+    expect(maybeSingleMock).toHaveBeenCalledTimes(2);
+  });
+
+  it('fails open to false (gate inert) and warns when the read throws', async () => {
+    maybeSingleMock.mockRejectedValueOnce(new Error('db down'));
+    expect(await isMfaGloballyEnabled()).toBe(false);
+    expect(warnMock).toHaveBeenCalledTimes(1);
+  });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vibecarbon",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Create and manage production-ready Vibecarbon applications",
   "type": "module",
   "bin": "./src/cli.js",

package/src/create.js

@@ -780,6 +780,16 @@ async function bootstrap(cliArgs) {
     join(projectDir, 'volumes/db/set-passwords.sh'),
     variables,
   );
+  // wal-archive.sh is postgres's archive_command. It was missing from this copy
+  // list, so scaffolds shipped no source file — Docker then bind-mounted an
+  // empty DIRECTORY at /etc/postgresql/wal-archive.sh and postgres failed
+  // archiving with exit 126 ("is a directory"), re-pinning WAL. Copy it (the
+  // chmod block below makes it +x; the script has only shell $vars, no {{}}).
+  copyTemplate(
+    'volumes/db/wal-archive.sh',
+    join(projectDir, 'volumes/db/wal-archive.sh'),
+    variables,
+  );
   copyTemplate('volumes/db/jwt.sql', join(projectDir, 'volumes/db/jwt.sql'), variables);
   copyTemplate('volumes/db/realtime.sql', join(projectDir, 'volumes/db/realtime.sql'), variables);
   copyTemplate(
@@ -854,6 +864,11 @@ async function bootstrap(cliArgs) {
     'docker-entrypoint.sh',
     'volumes/kong/docker-entrypoint.sh',
     'volumes/db/set-passwords.sh',
+    // Postgres execs this directly as its archive_command, so it MUST be +x.
+    // copyTemplate (copyFileSync) drops the source's exec bit, and omitting it
+    // here shipped a 0644 script → archive_command failed with exit 126 ("not
+    // executable") → WAL pinned → disk-fill risk the wrapper exists to prevent.
+    'volumes/db/wal-archive.sh',
   ];
   for (const script of shellScripts) {
     const scriptPath = join(projectDir, script);

package/src/destroy.js

@@ -693,7 +693,29 @@ async function handleBackupBucket(envConfig, projectConfig, args, spinner) {
     }
     const s3Provider = new HetznerS3Provider(s3Creds.accessKey, s3Creds.secretKey, region);
     try {
-      const result = await s3Provider.emptyAndDeleteBucket(bucketName);
+      // Bound the purge so a slow/contended S3 endpoint can't consume the whole
+      // destroy budget. The Hetzner resources are already freed (Pulumi destroy
+      // ran before this), but a single S3 op can hang up to ~9 min under the
+      // SDK (maxAttempts 3 × 60s requestTimeout) × _send (maxAttempts 3) retry
+      // layers — observed on k8s teardown, where the storage bucket doubles as
+      // the Pulumi state backend (slow/contended), blowing the 600s
+      // final-destroy timeout and getting the whole step SIGKILLed. On timeout
+      // we leave the bucket for the orphan sweep / next destroy rather than hang.
+      const PURGE_TIMEOUT_MS = 180_000;
+      const result = await Promise.race([
+        s3Provider.emptyAndDeleteBucket(bucketName),
+        new Promise((_, reject) =>
+          setTimeout(
+            () =>
+              reject(
+                new Error(
+                  `backup bucket purge exceeded ${PURGE_TIMEOUT_MS / 1000}s — left for the orphan sweep / a retried destroy`,
+                ),
+              ),
+            PURGE_TIMEOUT_MS,
+          ),
+        ),
+      ]);
       spinner.stop(
         result.deleted
           ? result.objectsRemoved > 0

package/src/lib/deploy/compose/ha.js

@@ -1043,15 +1043,18 @@ export async function deployComposeHA(options) {
     ]),
   );
 
-  // Run migrations on primary only
+  // Run migrations on primary only. Let failures propagate — silently shipping
+  // an empty/partial schema is far worse than a visibly-failed deploy the
+  // operator can retry. The old swallowing try/catch ("non-fatal") masked
+  // exactly that: HA could deploy a schema-less DB that 500s on every DB-backed
+  // route with no error surfaced. runMigrations now waits for supabase_admin AND
+  // the storage schema (storage.buckets.public) before applying with
+  // ON_ERROR_STOP=1, so the first-run races that motivated the swallow are gone
+  // and a failure here is a real one that must abort the deploy.
   onProgress('Running database migrations on primary...');
-  try {
-    await perfAsync('deploy.ha.compose.migrations', () =>
-      runMigrations(primaryIp, sshKeyPath, projectName),
-    );
-  } catch {
-    // Migrations may partially fail on first run, non-fatal
-  }
+  await perfAsync('deploy.ha.compose.migrations', () =>
+    runMigrations(primaryIp, sshKeyPath, projectName),
+  );
 
   // Create admin user in production Supabase (same as single-server deploy)
   onProgress('Creating admin user...');

package/src/lib/deploy/image.js

@@ -92,11 +92,11 @@ export function inspectGitState(cwd) {
  * pod (Phase 1 wired the registries.yaml; Phase 6 wires the push).
  *
  * @param {string} projectDir - Project root with the Dockerfile.
- * @param {{projectName: string, timestamp?: string, rebuild?: boolean, tagPrefix?: string}} options
+ * @param {{projectName: string, timestamp?: string, rebuild?: boolean, tagPrefix?: string, buildArgs?: Record<string,string>}} options
  * @returns {{tag: string, gitSha: string, isDirty: boolean}}
  */
 export function buildLocalImage(projectDir, options) {
-  const { projectName, rebuild = false, tagPrefix } = options;
+  const { projectName, rebuild = false, tagPrefix, buildArgs = {} } = options;
   if (!projectName) {
     throw new Error('buildLocalImage: projectName is required');
   }
@@ -112,6 +112,14 @@ export function buildLocalImage(projectDir, options) {
 
   const args = ['build'];
   if (rebuild) args.push('--no-cache');
+  // Vite inlines import.meta.env.VITE_* at build time, so the browser bundle
+  // bakes whatever these ARGs resolve to. Without passing them the bundle ships
+  // empty VITE_SUPABASE_* and crashes at page load with "Missing Supabase
+  // environment variables" — k8s shipped exactly this because buildAppImage
+  // passed no build args (compose plumbs them via collectComposeBuildArgs).
+  for (const [k, v] of Object.entries(buildArgs)) {
+    if (v !== undefined && v !== null && v !== '') args.push('--build-arg', `${k}=${v}`);
+  }
   args.push('-t', tag, projectDir);
 
   execFileSync('docker', args, { stdio: 'inherit' });

package/src/lib/deploy/k8s/k3s.js

@@ -556,9 +556,25 @@ export function loadEnvLocal(envPath) {
  * @param {boolean} [rebuild]
  * @returns {Promise<{tag: string, gitSha: string, isDirty: boolean}>}
  */
-export async function buildAppImage(projectDir, projectName, rebuild = false) {
+export async function buildAppImage(projectDir, projectName, rebuild = false, domain = undefined) {
   const { buildLocalImage } = await import('../image.js');
-  return buildLocalImage(projectDir, { projectName, rebuild, tagPrefix: '10.0.1.1:5000' });
+  // App image: collect VITE_* build args (incl. VITE_SUPABASE_URL rewritten to
+  // https://api.<domain>) so the browser bundle ships with real config instead
+  // of empty strings. buildLocalImage previously passed NO build args, so the
+  // k8s SPA crashed on load with "Missing Supabase environment variables" —
+  // compose plumbs these via collectComposeBuildArgs, k8s did not. The backup
+  // image build (no frontend) calls this without a domain → no VITE args.
+  let buildArgs = {};
+  if (domain) {
+    const { collectComposeBuildArgs } = await import('../compose/build-args.js');
+    buildArgs = collectComposeBuildArgs(projectDir, { projectName, domain });
+  }
+  return buildLocalImage(projectDir, {
+    projectName,
+    rebuild,
+    tagPrefix: '10.0.1.1:5000',
+    buildArgs,
+  });
 }
 
 /**
@@ -2436,11 +2452,14 @@ export async function deployK3s(options) {
   }
 
   // 6. Build the app image locally
-  const buildInputs = { projectName: options.projectName };
+  // domain is part of the cache key + passed to the build: it's baked into the
+  // browser bundle (VITE_SUPABASE_URL=https://api.<domain>), so a domain change
+  // must force a rebuild rather than serving a stale-domain bundle.
+  const buildInputs = { projectName: options.projectName, domain: options.domain };
   if (!state.shouldSkip('k3s-build', buildInputs)) {
     state.startStep('k3s-build', buildInputs);
     s.start('Building app image (10.0.1.1:5000/...)');
-    const built = await buildAppImage(projectDir, options.projectName);
+    const built = await buildAppImage(projectDir, options.projectName, false, options.domain);
     state.completeStep('k3s-build', {
       tag: built.tag,
       gitSha: built.gitSha,

package/src/lib/deploy/remote-build.js

@@ -36,8 +36,13 @@ export async function buildRemote(ip, sshKeyPath, imageTag, cwd, buildArgs = {})
   // key path via env var rather than string-interpolating into the script
   // to keep shell-escaping concerns out of the picture (paths with spaces
   // or quotes would otherwise break the wrapper).
+  // ServerAliveInterval/CountMax: BuildKit holds a long-lived SSH session for
+  // the build; on a congested/cross-region link it can stall and the http2
+  // session helper drops with "error reading preface ... file already closed".
+  // Keepalives let ssh detect a dead peer (~60s) and fail the attempt cleanly
+  // so the retry below can re-establish, instead of hanging to the build timeout.
   const sshWrapper = `#!/bin/bash
-exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 "$@"
+exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 "$@"
 `;
   writeFileSync(sshWrapperPath, sshWrapper, { mode: 0o755 });
 
@@ -103,15 +108,31 @@ exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=no -o UserKn
     // (compose-ha 2026-05-09T00:38 e2 failure). Treat the false return as a
     // hard failure so the error path captures + reports it.
     const buildT = perfTimer('deploy.image.remoteBuild');
-    const buildOk = runCommand(args, {
-      cwd,
-      stdio: 'inherit',
-      env,
-      timeout: 600000,
-    });
+    // BuildKit-over-SSH is flaky on first connect to a freshly-booted (often
+    // cross-region) VPS: the session helper intermittently drops mid-build with
+    // `http2: server: error reading preface ... file already closed` / an SSH
+    // reset, and runCommand returns false. Retry a few times — each attempt
+    // re-establishes a fresh SSH/BuildKit session, and BuildKit's cache makes
+    // retries cheap. Without this a single transient drop failed the whole
+    // deploy (compose-ha standby) or left no image so `docker compose up` later
+    // hit "pull access denied" (compose scale, e2e loop run #1).
+    const BUILD_ATTEMPTS = 3;
+    let buildOk = false;
+    for (let attempt = 1; attempt <= BUILD_ATTEMPTS; attempt++) {
+      buildOk = runCommand(args, { cwd, stdio: 'inherit', env, timeout: 600000 });
+      if (buildOk !== false) break;
+      if (attempt < BUILD_ATTEMPTS) {
+        console.error(
+          `[remote-build] docker build on ${ip} failed (attempt ${attempt}/${BUILD_ATTEMPTS}) — retrying in ${3 * attempt}s`,
+        );
+        await new Promise((r) => setTimeout(r, 3000 * attempt));
+      }
+    }
     buildT.end();
     if (buildOk === false) {
-      throw new Error(`docker build failed on ${ip} for tag ${imageTag}`);
+      throw new Error(
+        `docker build failed on ${ip} for tag ${imageTag} after ${BUILD_ATTEMPTS} attempts`,
+      );
     }
 
     s.stop(`Image built natively: ${imageTag}`);

package/src/scale.js

@@ -439,9 +439,22 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
       // Reuses the same code that does direct-mode deploy builds.
       if (oldAppImage && isLocalOnlyImageTag(oldAppImage)) {
         s.start(`Building image ${oldAppImage} on new server...`);
-        await perfAsync('scale.sideloadImage', () =>
+        const built = await perfAsync('scale.sideloadImage', () =>
           buildRemote(newIp, sshKeyPath, oldAppImage, process.cwd()),
         );
+        // buildRemote returns false on failure (after its own retries). The app
+        // image is local-only, so if it isn't built on the new server, the
+        // `docker compose up` below pulls it and dies ~a minute later with
+        // "pull access denied / repository does not exist" — a confusing
+        // downstream symptom. Fail loudly here instead; this guard was missing
+        // (the HA deploy path has the equivalent check). RCA: e2e loop run #1.
+        if (!built) {
+          s.stop('Remote image build failed on new server', 1);
+          throw new Error(
+            `scale: failed to build local app image ${oldAppImage} on new server ${newIp} ` +
+              `(buildRemote exhausted its retries) — aborting before compose up.`,
+          );
+        }
         s.stop('Image built on new server');
       }