vibecarbon 0.3.1 → 0.5.0

package/carbon/backup/compose-backup.sh

@@ -2,139 +2,63 @@
 set -e
 
 # ============================================================================
-# Vibecarbon Compose Backup Script
+# Vibecarbon Compose Backup Script — single source of truth for compose backups
 #
-# Runs via cron on the VPS. Creates a full backup archive containing all
-# PostgreSQL databases and storage files, uploads to S3, and cleans up
-# old backups based on retention policy.
+# Runs via cron on the VPS (and on-demand from the CLI). Triggers a wal-g base
+# backup from inside the db container (which already has WALG_S3_PREFIX + AWS_*
+# configured from .env) and prunes old base backups via `wal-g delete retain`.
 #
-# Environment Variables (loaded from .backup-env):
-#   PROJECT_NAME           - Project name for backup naming
-#   BACKUP_RETENTION_DAYS  - Days to keep backups (default: 30)
-#   S3_BACKUP_BUCKET       - S3 bucket for backup storage
-#   S3_ACCESS_KEY          - S3 access key
-#   S3_SECRET_KEY          - S3 secret key
-#   S3_ENDPOINT            - S3 endpoint URL
-#   S3_REGION              - S3 region
+# Invoked as `cd <project-dir> && RETAIN=<n> bash backup/compose-backup.sh`, so
+# it operates relative to the current working directory — no PROJECT_NAME
+# needed. Both the cron path and the on-demand path use the SAME invocation
+# (src/lib/deploy/compose/index.js composeBackupCmd), so the quoting/behavior
+# can never drift.
+#
+# Environment Variables:
+#   RETAIN  - Number of full base backups to keep (default: 7)
 # ============================================================================
 
-# Prevent concurrent backups
+# Prevent concurrent backups (cron + a manual run racing each other).
 LOCKFILE="/tmp/vibecarbon-backup.lock"
 exec 200>"${LOCKFILE}"
 flock -n 200 || { echo "Backup already running — skipping"; exit 0; }
 
-PROJECT="${PROJECT_NAME:?PROJECT_NAME is required}"
-REMOTE_DIR="/opt/${PROJECT}"
-TIMESTAMP=$(date +%Y%m%d_%H%M%S)
-WORK_DIR="/tmp/backup_${TIMESTAMP}"
-ARCHIVE_NAME="${PROJECT}_${TIMESTAMP}_full.tar.gz"
-RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-30}"
+RETAIN="${RETAIN:-7}"
 
-echo "=== Vibecarbon Compose Backup ==="
+echo "=== Vibecarbon Compose Backup (wal-g) ==="
 echo "Started at $(date)"
-echo "Retention: ${RETENTION_DAYS} days"
-
-mkdir -p "${WORK_DIR}"
-
-# ============================================================================
-# Step 1: Dump postgres database
-# ============================================================================
-
-echo "Backing up postgres database..."
-cd "${REMOTE_DIR}" && docker compose exec -T db pg_dump -U postgres postgres \
-  | gzip > "${WORK_DIR}/postgres.sql.gz"
-DB_SIZE=$(du -h "${WORK_DIR}/postgres.sql.gz" | cut -f1)
-echo "postgres: ${DB_SIZE}"
-
-# ============================================================================
-# Step 2: Dump optional service databases (n8n, metabase)
-# ============================================================================
+echo "Retain: ${RETAIN} full base backups"
 
-for EXTRA_DB in n8n metabase; do
-  DB_EXISTS=$(cd "${REMOTE_DIR}" && docker compose exec -T db \
-    psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='${EXTRA_DB}'" 2>/dev/null || echo "")
-  if [ "${DB_EXISTS}" = "1" ]; then
-    echo "Backing up ${EXTRA_DB} database..."
-    cd "${REMOTE_DIR}" && docker compose exec -T db pg_dump -U postgres "${EXTRA_DB}" \
-      | gzip > "${WORK_DIR}/${EXTRA_DB}.sql.gz"
-    EXTRA_SIZE=$(du -h "${WORK_DIR}/${EXTRA_DB}.sql.gz" | cut -f1)
-    echo "${EXTRA_DB}: ${EXTRA_SIZE}"
+# wal-g connects to PG via libpq; PGUSER=supabase_admin is REQUIRED (only the
+# superuser may call pg_backup_start; unset, libpq defaults to OS user root →
+# "role root does not exist"). The pg_is_in_recovery()='f' guard makes the
+# backup a no-op on a standby (exit 0), so only the primary pushes to S3.
+#
+# The S3-configured guard exits 0 BEFORE wal-g runs when no S3 backup target is
+# configured. docker-compose.yml renders WALG_S3_PREFIX as
+#   s3://${S3_BACKUP_BUCKET:-${S3_BUCKET:-}}/backups/${PROJECT_NAME}/walg
+# so an unconfigured deploy yields the empty-bucket form `s3:///backups/...`
+# (and empty AWS_ACCESS_KEY_ID). Without this guard an always-installed cron
+# would `set -e`-fail nightly into backup.log on no-S3 deploys. Checking inside
+# the container is robust: it tests the actual env wal-g sees (source of truth),
+# covers both the cron and on-demand paths, and needs no deploy-time S3 plumbing.
+docker compose exec -T db bash -c '
+  export PGUSER=supabase_admin PGHOST=localhost PGPORT=5432 PGDATABASE=postgres
+  case "${WALG_S3_PREFIX:-}" in
+    ""|s3:///*)
+      echo "scheduled backup skipped: no S3 backup target configured (empty WALG_S3_PREFIX)."
+      exit 0
+      ;;
+  esac
+  if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then
+    echo "scheduled backup skipped: no S3 credentials configured (empty AWS_ACCESS_KEY_ID/SECRET)."
+    exit 0
   fi
-done
-
-# ============================================================================
-# Step 3: Storage files (file backend only)
-# ============================================================================
-
-STORAGE_COUNT=$(cd "${REMOTE_DIR}" && docker compose exec -T storage \
-  sh -c 'find /var/lib/storage -type f 2>/dev/null | wc -l' 2>/dev/null || echo "0")
-if [ "${STORAGE_COUNT}" -gt 0 ] 2>/dev/null; then
-  echo "Backing up ${STORAGE_COUNT} storage files..."
-  cd "${REMOTE_DIR}" && docker compose exec -T storage \
-    tar czf - -C / var/lib/storage > "${WORK_DIR}/storage.tar.gz"
-  STORAGE_SIZE=$(du -h "${WORK_DIR}/storage.tar.gz" | cut -f1)
-  echo "Storage: ${STORAGE_SIZE}"
-fi
-
-# ============================================================================
-# Step 4: Create combined archive
-# ============================================================================
-
-echo "Creating archive: ${ARCHIVE_NAME}"
-tar czf "${REMOTE_DIR}/backups/${ARCHIVE_NAME}" -C "${WORK_DIR}" .
-TOTAL_SIZE=$(du -h "${REMOTE_DIR}/backups/${ARCHIVE_NAME}" | cut -f1)
-echo "Total archive size: ${TOTAL_SIZE}"
-
-# ============================================================================
-# Step 5: Upload to S3 (if configured)
-# ============================================================================
-
-if [ -n "${S3_BACKUP_BUCKET}" ] && [ -n "${S3_ACCESS_KEY}" ] && [ -n "${S3_SECRET_KEY}" ] && [ -n "${S3_ENDPOINT}" ]; then
-  echo "Uploading to S3: s3://${S3_BACKUP_BUCKET}/backups/${ARCHIVE_NAME}"
-
-  export AWS_ACCESS_KEY_ID="${S3_ACCESS_KEY}"
-  export AWS_SECRET_ACCESS_KEY="${S3_SECRET_KEY}"
-  export AWS_DEFAULT_REGION="${S3_REGION:-us-east-1}"
-
-  aws s3 cp "${REMOTE_DIR}/backups/${ARCHIVE_NAME}" \
-    "s3://${S3_BACKUP_BUCKET}/backups/${ARCHIVE_NAME}" \
-    --endpoint-url "${S3_ENDPOINT}"
-
-  echo "Upload complete"
-
-  # S3 retention cleanup
-  echo "Cleaning up S3 backups older than ${RETENTION_DAYS} days..."
-  CUTOFF_DATE=$(date -d "-${RETENTION_DAYS} days" +%Y%m%d 2>/dev/null || echo "")
-  if [ -n "${CUTOFF_DATE}" ]; then
-    aws s3api list-objects-v2 \
-      --bucket "${S3_BACKUP_BUCKET}" \
-      --prefix "backups/${PROJECT}_" \
-      --endpoint-url "${S3_ENDPOINT}" \
-      --query "Contents[].Key" \
-      --output text 2>/dev/null | tr '\t' '\n' | while read -r key; do
-        [ -z "${key}" ] || [ "${key}" = "None" ] && continue
-        FILE_DATE=$(echo "${key}" | sed -n 's/.*_\([0-9]\{8\}\)_[0-9]\{6\}_full\.tar\.gz$/\1/p')
-        if [ -n "${FILE_DATE}" ] && [ "${FILE_DATE}" -lt "${CUTOFF_DATE}" ] 2>/dev/null; then
-          echo "Deleting expired backup: ${key}"
-          aws s3 rm "s3://${S3_BACKUP_BUCKET}/${key}" --endpoint-url "${S3_ENDPOINT}"
-        fi
-      done
+  if [ "$(psql -U supabase_admin -d postgres -tAc "SELECT pg_is_in_recovery()")" != "f" ]; then
+    echo "supabase-db is in recovery (standby) — skipping base backup."
+    exit 0
   fi
-else
-  echo "No S3 configured — backup stored locally only"
-fi
-
-# ============================================================================
-# Step 6: Local retention cleanup
-# ============================================================================
-
-find "${REMOTE_DIR}/backups" -name "${PROJECT}_*_full.tar.gz" -mtime "+${RETENTION_DAYS}" -delete 2>/dev/null || true
-
-# ============================================================================
-# Cleanup
-# ============================================================================
-
-rm -rf "${WORK_DIR}"
+  wal-g backup-push "$PGDATA" && wal-g delete retain FULL "'"${RETAIN}"'" --confirm
+'
 
 echo "=== Backup completed at $(date) ==="
-echo "Archive: ${ARCHIVE_NAME} (${TOTAL_SIZE})"

package/carbon/db/Dockerfile

@@ -1,20 +1,31 @@
 # syntax=docker/dockerfile:1
 FROM supabase/postgres:15.8.1.085
 
-# Install WAL-G v3.0.5 — kept in lockstep with carbon/backup/Dockerfile.
+# Install WAL-G v3.0.5. This image IS the backup tooling now (both compose
+# and k8s): wal-g runs co-located with PGDATA in this container for continuous
+# WAL archiving, base backups, and restore — no separate backup image.
 # Earlier v3.0.3 URL 404'd in 2026-04-26 (alpine variant retired upstream),
 # and curl without --fail piped a 404 HTML body into tar which surfaced
 # as "not in gzip format" instead of an honest HTTP error. --fail makes
 # curl exit non-zero on 4xx/5xx so the failure is unmistakable.
 #
+# Build target MUST match the base image's glibc: supabase/postgres:15.8.1.085
+# is Ubuntu 20.04 (glibc 2.31). The ubuntu-22.04 wal-g build is linked against
+# glibc 2.35 and dies at runtime with `version 'GLIBC_2.3{2,3,4}' not found`
+# — which breaks BOTH base backups and WAL archiving (the archive_command
+# shells out to wal-g), and is invisible at deploy time because we only assert
+# archive_mode=on, never that wal-g actually runs. The ubuntu-20.04 build links
+# against glibc 2.31 and runs on this base. (RCA 2026-05-30: e2e k8s/k8s-ha
+# backup step — wal-g exited 1 with the GLIBC error inside the db pod.)
+#
 # Single RUN with `apt-get purge -y --auto-remove curl` collapses the
 # install + cleanup into one layer so the curl binary doesn't end up
 # in the published image (saving ~5MB and reducing attack surface).
 RUN apt-get update && apt-get install -y --no-install-recommends \
         curl \
         ca-certificates \
-    && curl -fL https://github.com/wal-g/wal-g/releases/download/v3.0.5/wal-g-pg-ubuntu-22.04-amd64.tar.gz | tar -xz \
-    && mv wal-g-pg-ubuntu-22.04-amd64 /usr/local/bin/wal-g \
+    && curl -fL https://github.com/wal-g/wal-g/releases/download/v3.0.5/wal-g-pg-ubuntu-20.04-amd64.tar.gz | tar -xz \
+    && mv wal-g-pg-ubuntu-20.04-amd64 /usr/local/bin/wal-g \
     && chmod +x /usr/local/bin/wal-g \
     && apt-get purge -y --auto-remove curl \
     && rm -rf /var/lib/apt/lists/*

package/carbon/docker-compose.dns01.prod.yml

@@ -0,0 +1,42 @@
+# DNS-01 ACME override — applied ONLY for managed-DNS deploys (cloudflare / hetzner).
+#
+# Activated by the deploy bundler (src/lib/deploy/bundle.js -> composeFileFlags)
+# when the deploy manages DNS via an API. It is appended AFTER
+# docker-compose.prod.yml, so the `command:` below REPLACES the base Traefik
+# command — Compose replaces sequences, it does not merge them. That means the
+# full command list must be restated here; keep every non-ACME flag in sync
+# with docker-compose.prod.yml. Only the two `httpchallenge` lines are swapped
+# for the three `dnschallenge` lines.
+#
+# Why: cloudflare/hetzner expose an API lego (Traefik's ACME client) uses to
+# solve the DNS-01 challenge by writing a TXT record. This removes the HTTP-01
+# race against A-record propagation, so the deploy no longer needs the
+# DNS-propagation poll or the Traefik cert self-heal. `manual` DNS keeps the
+# base HTTP-01 path (this override is not applied for it).
+#
+# Tokens: lego picks the provider from ACME_DNS_PROVIDER and reads its token.
+# Hetzner uses the consolidated Cloud API token HETZNER_API_TOKEN (lego v4.27+,
+# Traefik >= 3.6.6; we pin v3.6.11) — the same token as server ops. Cloudflare
+# uses CF_DNS_API_TOKEN. Both are passed through; only the active provider's
+# token is read. All three are populated in .env by the bundler.
+services:
+  traefik:
+    command:
+      - "--api.dashboard=true"
+      - "--providers.docker=true"
+      - "--providers.docker.endpoint=tcp://docker-socket-proxy:2375"
+      - "--providers.docker.exposedbydefault=false"
+      - "--providers.file.directory=/etc/traefik/dynamic"
+      - "--entrypoints.web.address=:80"
+      - "--entrypoints.websecure.address=:443"
+      - "--certificatesresolvers.letsencrypt.acme.dnschallenge=true"
+      - "--certificatesresolvers.letsencrypt.acme.dnschallenge.provider=${ACME_DNS_PROVIDER}"
+      - "--certificatesresolvers.letsencrypt.acme.dnschallenge.resolvers=1.1.1.1:53,8.8.8.8:53"
+      - "--certificatesresolvers.letsencrypt.acme.email={{ADMIN_EMAIL}}"
+      - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json"
+      - "--certificatesresolvers.letsencrypt.acme.caserver=${ACME_CA_SERVER:-https://acme-v02.api.letsencrypt.org/directory}"
+      - "--log.level=WARN"
+    environment:
+      ACME_DNS_PROVIDER: ${ACME_DNS_PROVIDER:-}
+      CF_DNS_API_TOKEN: ${CF_DNS_API_TOKEN:-}
+      HETZNER_API_TOKEN: ${HETZNER_API_TOKEN:-}

package/carbon/docker-compose.metabase.prod.yml

@@ -15,11 +15,10 @@ services:
     labels:
       # Production Traefik routing with SSL
       - "traefik.enable=true"
-      # HTTPS router with super_admin auth
+      # HTTPS router with super_admin auth — cert from default store
       - "traefik.http.routers.metabase.rule=Host(`metabase.${DOMAIN}`)"
       - "traefik.http.routers.metabase.entrypoints=websecure"
       - "traefik.http.routers.metabase.tls=true"
-      - "traefik.http.routers.metabase.tls.certresolver=letsencrypt"
       - "traefik.http.routers.metabase.middlewares=super-admin-auth@file"
       - "traefik.http.services.metabase.loadbalancer.server.port=3000"
       # HTTP to HTTPS redirect

package/carbon/docker-compose.n8n.prod.yml

@@ -18,11 +18,10 @@ services:
     labels:
       # Production Traefik routing with SSL
       - "traefik.enable=true"
-      # HTTPS router with super_admin auth
+      # HTTPS router with super_admin auth — cert from default store
       - "traefik.http.routers.n8n.rule=Host(`n8n.${DOMAIN}`)"
       - "traefik.http.routers.n8n.entrypoints=websecure"
       - "traefik.http.routers.n8n.tls=true"
-      - "traefik.http.routers.n8n.tls.certresolver=letsencrypt"
       - "traefik.http.routers.n8n.middlewares=super-admin-auth@file"
       - "traefik.http.services.n8n.loadbalancer.server.port=5678"
       # HTTP to HTTPS redirect

package/carbon/docker-compose.prod.yml

@@ -26,7 +26,12 @@ services:
       - "traefik.http.routers.app-http.entrypoints=web"
       - "traefik.http.routers.app-http.middlewares=redirect-to-https"
       - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
-      # HTTPS router with TLS
+      # HTTPS router for the app at the bare apex (${DOMAIN}). The apex gets its
+      # OWN single-domain cert via certresolver — it is intentionally NOT in the
+      # default wildcard store cert, because apex + `*.${DOMAIN}` share the
+      # `_acme-challenge.${DOMAIN}` TXT name and some DNS providers (Hetzner)
+      # can't hold both challenge values at once. The subdomain routers below use
+      # `tls=true` (no resolver) and serve the `*.${DOMAIN}` default cert via SNI.
       - "traefik.http.routers.app.rule=Host(`${DOMAIN:-localhost}`)"
       - "traefik.http.routers.app.entrypoints=websecure"
       - "traefik.http.routers.app.tls=true"
@@ -109,10 +114,9 @@ services:
       - "traefik.http.routers.dashboard-http.rule=Host(`traefik.${DOMAIN}`)"
       - "traefik.http.routers.dashboard-http.entrypoints=web"
       - "traefik.http.routers.dashboard-http.middlewares=redirect-to-https"
-      # HTTPS dashboard with super_admin auth
+      # HTTPS dashboard with super_admin auth — cert from default store
       - "traefik.http.routers.dashboard.entrypoints=websecure"
       - "traefik.http.routers.dashboard.tls=true"
-      - "traefik.http.routers.dashboard.tls.certresolver=letsencrypt"
       - "traefik.http.routers.dashboard.middlewares=super-admin-auth@file"
     deploy:
       resources:
@@ -127,11 +131,10 @@ services:
   studio:
     labels:
       - "traefik.enable=true"
-      # HTTPS router with admin auth
+      # HTTPS router with admin auth — cert from default store
       - "traefik.http.routers.studio.rule=Host(`studio.${DOMAIN}`)"
       - "traefik.http.routers.studio.entrypoints=websecure"
       - "traefik.http.routers.studio.tls=true"
-      - "traefik.http.routers.studio.tls.certresolver=letsencrypt"
       - "traefik.http.routers.studio.middlewares=admin-auth@file"
       - "traefik.http.services.studio.loadbalancer.server.port=3000"
 
@@ -260,11 +263,10 @@ services:
       - "traefik.http.routers.kong-http.rule=Host(`api.${DOMAIN:-localhost}`)"
       - "traefik.http.routers.kong-http.entrypoints=web"
       - "traefik.http.routers.kong-http.middlewares=redirect-to-https"
-      # HTTPS router for Supabase API
+      # HTTPS router for Supabase API — cert from default store
       - "traefik.http.routers.kong.rule=Host(`api.${DOMAIN:-localhost}`)"
       - "traefik.http.routers.kong.entrypoints=websecure"
       - "traefik.http.routers.kong.tls=true"
-      - "traefik.http.routers.kong.tls.certresolver=letsencrypt"
       - "traefik.http.services.kong.loadbalancer.server.port=8000"
     deploy:
       resources:

package/carbon/k8s/base/backup/configmap-walg.yaml

@@ -0,0 +1,47 @@
+# Fault-tolerant WAL-archive wrapper mounted into the supabase-db container at
+# /etc/postgresql/wal-archive.sh (see k8s/values/supabase.values.yaml
+# deployment.db.volumes/volumeMounts). Postgres' archive_command invokes it as
+# `bash /etc/postgresql/wal-archive.sh %p` (enabled via ALTER SYSTEM in
+# applyK3sManifests). wal-g reads WALG_S3_PREFIX / AWS_* from the db
+# container's env.
+#
+# IMPORTANT: keep this script body byte-for-byte in sync with
+# carbon/volumes/db/wal-archive.sh (the compose mount). Same retry + exit-0
+# semantics: a persistent S3 outage degrades PITR for the failed segment but
+# keeps the DB up instead of filling the disk.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vibecarbon-wal-archive
+  namespace: vibecarbon
+  labels:
+    app: vibecarbon-backup
+    component: backup
+data:
+  wal-archive.sh: |
+    #!/bin/bash
+    # Fault-tolerant archive_command for Postgres + wal-g.
+    # Invoked by postgres as: wal-archive.sh <wal-file-path>
+    # (%p — absolute path to the WAL segment). Retries with backoff, then
+    # logs WAL_ARCHIVE_FAILED and exits 0 so PG recycles the segment rather
+    # than pinning it and filling the disk. Keep in sync with
+    # carbon/volumes/db/wal-archive.sh.
+    set -u
+    WAL_PATH="${1:?archive_command requires a WAL file path}"
+    RETRIES="${WAL_ARCHIVE_RETRIES:-3}"
+    SLEEP_BASE="${WAL_ARCHIVE_SLEEP:-2}"
+
+    attempt=1
+    while [ "$attempt" -le "$RETRIES" ]; do
+        if /usr/local/bin/wal-g wal-push "$WAL_PATH"; then
+            exit 0
+        fi
+        if [ "$attempt" -lt "$RETRIES" ]; then
+            sleep_for=$((SLEEP_BASE ** attempt))
+            sleep "$sleep_for"
+        fi
+        attempt=$((attempt + 1))
+    done
+
+    echo "WAL_ARCHIVE_FAILED: wal-g wal-push '$WAL_PATH' failed after $RETRIES attempts; allowing PG to recycle (PITR gap for this segment)" >&2
+    exit 0

package/carbon/k8s/base/backup/cronjob.yaml

@@ -1,3 +1,17 @@
+# Nightly wal-g base backup.
+#
+# wal-g backup-push needs direct PGDATA filesystem access, which only the
+# supabase-db pod has. So instead of a separate backup container that talks to
+# postgres over the network (the old pg_dump design), this CronJob runs a tiny
+# kubectl image and `kubectl exec`s wal-g INSIDE the db pod — where wal-g, the
+# WALG_*/AWS_* env, and PGDATA already live (see k8s/values/supabase.values.yaml
+# + the WAL-archiving setup in applyK3sManifests). No bespoke backup image, no
+# sideload, no registry push.
+#
+# HA-safe: the command guards on pg_is_in_recovery()=f, so it only takes a base
+# backup on the PRIMARY. On the standby (in recovery) it no-ops, and after a
+# failover the new primary's CronJob transparently starts taking base backups —
+# no per-cluster config.
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -23,98 +37,59 @@ spec:
           restartPolicy: OnFailure
           serviceAccountName: backup-sa
           containers:
-          - name: backup
-            image: ghcr.io/{{GITHUB_OWNER}}/{{PROJECT_NAME}}-backup:latest
-            imagePullPolicy: Always
-            env:
-            - name: PGHOST
-              valueFrom:
-                configMapKeyRef:
-                  name: vibecarbon-config
-                  key: DB_HOST
-            - name: PGPORT
-              valueFrom:
-                configMapKeyRef:
-                  name: vibecarbon-config
-                  key: DB_PORT
-            - name: PGUSER
-              value: supabase_admin
-            - name: PGDATABASE
-              valueFrom:
-                configMapKeyRef:
-                  name: vibecarbon-config
-                  key: DB_NAME
-            - name: PGPASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: vibecarbon-secrets
-                  key: DB_PASSWORD
-            - name: S3_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: vibecarbon-secrets
-                  key: S3_ACCESS_KEY
-                  optional: true
-            - name: S3_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: vibecarbon-secrets
-                  key: S3_SECRET_KEY
-                  optional: true
-            - name: S3_BUCKET
-              valueFrom:
-                secretKeyRef:
-                  name: vibecarbon-secrets
-                  key: S3_BUCKET
-                  optional: true
-            - name: S3_ENDPOINT
-              valueFrom:
-                secretKeyRef:
-                  name: vibecarbon-secrets
-                  key: S3_ENDPOINT
-                  optional: true
-            - name: S3_REGION
-              valueFrom:
-                secretKeyRef:
-                  name: vibecarbon-secrets
-                  key: S3_REGION
-                  optional: true
-            - name: S3_BACKUP_BUCKET
-              valueFrom:
-                secretKeyRef:
-                  name: vibecarbon-secrets
-                  key: S3_BACKUP_BUCKET
-                  optional: true
-            - name: BACKUP_RETENTION_DAYS
-              valueFrom:
-                configMapKeyRef:
-                  name: vibecarbon-config
-                  key: BACKUP_RETENTION_DAYS
-                  optional: true
-            - name: STORAGE_BACKEND
-              value: file
-            # Use pg_dump (network protocol) instead of WAL-G in this
-            # CronJob — WAL-G's backup-push needs direct filesystem access
-            # to PGDATA, which only the supabase-db pod has. The backup
-            # pod runs separately and connects via PGHOST, so it has to
-            # use the libpq-based pg_dump path. Switching WAL-G on for
-            # k8s would require a sidecar inside supabase-db.
-            - name: BACKUP_MODE
-              value: pg_dump
-            - name: PROJECT_NAME
-              valueFrom:
-                configMapKeyRef:
-                  name: vibecarbon-config
-                  key: PROJECT_NAME
-            resources:
-              requests:
-                cpu: 100m
-                memory: 256Mi
-              limits:
-                cpu: 500m
-                memory: 512Mi
-            securityContext:
-              allowPrivilegeEscalation: false
+            - name: backup
+              # Pinned public kubectl image — no build/sideload/push. Bump the
+              # tag deliberately; `latest` would risk a silent client/server
+              # skew on the scheduled run. Needs bash (the script below uses
+              # `set -euo pipefail`) + kubectl on PATH, so the distroless
+              # registry.k8s.io/kubectl is unusable. Was bitnami/kubectl:1.31
+              # until Bitnami sunset their public Docker Hub catalog (2025) and
+              # the tag vanished (`not found` → ImagePullBackOff); alpine/k8s
+              # is the maintained drop-in (alpine base, bash + kubectl).
+              image: alpine/k8s:1.31.12
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: BACKUP_RETENTION_DAYS
+                  valueFrom:
+                    configMapKeyRef:
+                      name: vibecarbon-config
+                      key: BACKUP_RETENTION_DAYS
+                      optional: true
+              command:
+                - /bin/bash
+                - -c
+                - |
+                  set -euo pipefail
+                  RETAIN="${BACKUP_RETENTION_DAYS:-7}"
+                  # Exec wal-g inside the db pod. The guard makes this a no-op on
+                  # a standby (in recovery) so only the primary takes base backups.
+                  kubectl exec -n vibecarbon statefulset/supabase-supabase-db -- bash -c '
+                    set -euo pipefail
+                    # wal-g backup-push connects to Postgres (pg_backup_start) using
+                    # libpq env. Unset, PGUSER defaults to the container OS user
+                    # "root" → "role \"root\" does not exist". And only the cluster
+                    # superuser may call pg_backup_start — supabase_admin is the sole
+                    # superuser ("postgres" gets "permission denied for function
+                    # pg_backup_start"). So pin the libpq connection explicitly.
+                    # (wal-push/archiving and backup-fetch/restore do not connect to
+                    # PG, which is why this only bit backup-push. RCA 2026-05-30.)
+                    export PGUSER=supabase_admin PGHOST=localhost PGPORT=5432 PGDATABASE=postgres
+                    if [ "$(psql -U supabase_admin -d postgres -tAc "SELECT pg_is_in_recovery()")" != "f" ]; then
+                      echo "supabase-db is in recovery (standby) — skipping base backup."
+                      exit 0
+                    fi
+                    wal-g backup-push "$PGDATA"
+                    wal-g delete retain FULL '"${RETAIN}"' --confirm
+                  '
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: 250m
+                  memory: 128Mi
+              securityContext:
+                allowPrivilegeEscalation: false
           securityContext:
             seccompProfile:
               type: RuntimeDefault

package/carbon/k8s/base/backup/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
+  - configmap-walg.yaml
   - cronjob.yaml
   - rbac.yaml
   - network-policy.yaml

package/carbon/k8s/base/backup/network-policy.yaml

@@ -1,3 +1,9 @@
+# The backup CronJob pod no longer talks to postgres:5432 or S3 directly — it
+# runs `kubectl exec` into the supabase-db pod (wal-g runs there). So its only
+# egress need is the Kubernetes apiserver. DNS + intra-namespace egress come
+# from the cluster-baseline policies (network-policies.yaml, podSelector {}).
+# The wal-g → S3 egress now happens from the supabase-db pod, allowed by the
+# supabase-db-s3-egress policy in network-policies.yaml.
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -10,22 +16,11 @@ spec:
   policyTypes:
     - Egress
   egress:
-    # Connect to PostgreSQL for pg_dump
-    - to:
-        - podSelector:
-            matchLabels:
-              app: vibecarbon-postgres
-      ports:
-        - protocol: TCP
-          port: 5432
-    # Upload to S3 (external HTTPS)
+    # Kubernetes apiserver (kubernetes.default → ClusterIP 10.43.0.1:443) so
+    # `kubectl exec` works. Mirrors the app/traefik apiserver-egress pattern.
     - to:
         - ipBlock:
-            cidr: 0.0.0.0/0
-            except:
-              - 10.0.0.0/8
-              - 172.16.0.0/12
-              - 192.168.0.0/16
+            cidr: 10.43.0.1/32
       ports:
         - protocol: TCP
           port: 443

package/carbon/k8s/base/backup/rbac.yaml

@@ -5,3 +5,43 @@ metadata:
   labels:
     app: vibecarbon-backup
     component: backup
+---
+# The backup CronJob runs `kubectl exec` into the supabase-db pod to invoke
+# wal-g backup-push (wal-g needs local PGDATA access). That requires
+# pods/exec in the vibecarbon namespace. `pods` get/list is needed so
+# `kubectl exec statefulset/...` can resolve the pod.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: backup-exec
+  namespace: vibecarbon
+  labels:
+    app: vibecarbon-backup
+    component: backup
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+  - apiGroups: ["apps"]
+    resources: ["statefulsets"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: backup-exec
+  namespace: vibecarbon
+  labels:
+    app: vibecarbon-backup
+    component: backup
+subjects:
+  - kind: ServiceAccount
+    name: backup-sa
+    namespace: vibecarbon
+roleRef:
+  kind: Role
+  name: backup-exec
+  apiGroup: rbac.authorization.k8s.io