vibecarbon 0.1.0

package/carbon/cloud-init/k3s/master-init.sh

@@ -0,0 +1,215 @@
+#!/bin/bash
+set -euo pipefail
+
+# Note: This script runs AS PART of cloud-init (user_data), so we cannot wait for
+# cloud-init to complete (that would cause a deadlock). The script starts immediately.
+
+# Wait for any dpkg locks held by unattended-upgrades (runs on first boot).
+# Without this, apt-get fails intermittently with "Could not get lock /var/lib/dpkg/lock".
+for i in $(seq 1 30); do
+  if ! fuser /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1; then
+    break
+  fi
+  echo "Waiting for dpkg lock to be released... ($i/30)"
+  sleep 10
+done
+
+# Update apt lists + install the packages we need. `apt-get upgrade` was
+# removed from the critical path — unattended-upgrades handles security
+# patches async. Saves 30-60s per deploy.
+apt-get update -qq
+apt-get install -y -qq curl jq docker.io docker-buildx
+
+# Validate k3s version is set
+K3S_TARGET_VERSION="${k3s_version}"
+if [ -z "$K3S_TARGET_VERSION" ]; then
+  echo "ERROR: k3s_version not set"
+  exit 1
+fi
+echo "Installing k3s version: $K3S_TARGET_VERSION"
+
+# IDENTICAL block in master-init.sh / worker-init.sh / supabase-init.sh — keep in sync.
+# (Three-way duplication is acceptable here because the cloud-init scripts are
+# rendered independently per role; extracting a shared snippet would require
+# restructuring the renderScript() pipeline. Out of scope for Phase 1.)
+# Configure containerd to mirror pulls for 10.0.1.1:5000 to the local registry
+# we run on master. Cluster-autoscaler-spawned workers pull the app image from
+# here on-demand (sideload only reaches workers that exist at sideload time).
+# k3s only reads /etc/rancher/k3s/registries.yaml on agent start, so this MUST
+# land before the k3s install step below.
+#
+# Plain http://10.0.1.1:5000 — the registry runs HTTP on the private network.
+# Do NOT add `insecure_skip_verify: true` here: that flag only affects HTTPS
+# endpoints, and recent k3s versions place it incorrectly when present
+# (k3s-io/k3s#13215), which can break containerd config generation.
+mkdir -p /etc/rancher/k3s
+cat > /etc/rancher/k3s/registries.yaml << 'REGEOF'
+mirrors:
+  "10.0.1.1:5000":
+    endpoint:
+      - "http://10.0.1.1:5000"
+REGEOF
+
+# Configure Floating IP on the OS so the kernel accepts packets destined for it.
+# Hetzner routes the Floating IP to this server at the network level, but without
+# a local address binding the kernel drops the packets.
+# Use `ip addr add` instead of netplan — a separate netplan file that redefines
+# ethernets.eth0 can override Hetzner's cloud-init config and drop the primary IP.
+ip addr add ${floating_ip}/32 dev eth0 2>/dev/null || true
+
+# Persist across reboots via a long-running systemd service (netplan-safe).
+# Must be Type=simple (not oneshot) so it survives DHCP renewals — when networkd
+# re-applies the cloud-init netplan config it can flush manually-added addresses.
+# The loop re-adds the floating IP within 30 seconds if that happens.
+cat > /etc/systemd/system/floating-ip.service << FIPEOF
+[Unit]
+Description=Maintain Hetzner Floating IP
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/bin/bash -c 'while true; do if ! ip addr show eth0 | grep -q "${floating_ip}"; then ip addr add ${floating_ip}/32 dev eth0 && echo "Floating IP re-added"; fi; sleep 30; done'
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+FIPEOF
+systemctl daemon-reload
+systemctl enable floating-ip.service
+
+# Get node IPs for k3s configuration
+PUBLIC_IP=$(curl -s --max-time 10 http://169.254.169.254/hetzner/v1/metadata/public-ipv4)
+
+# Wait for the private network interface (10.0.x.x) to be assigned.
+# Hetzner attaches the private network after the server boots, so the interface
+# may not exist yet when cloud-init runs. grep exits 1 if no match, which kills
+# the script under set -euo pipefail — so we retry with || true.
+PRIVATE_IFACE=""
+RETRY_COUNT=0
+MAX_RETRIES=120
+DHCP_TRIGGERED=false
+echo "Waiting for private network interface..."
+while [ -z "$PRIVATE_IFACE" ] && [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+  PRIVATE_IFACE=$(ip -4 addr show | grep 'inet 10\.0\.' | awk '{print $NF}' || true)
+  if [ -z "$PRIVATE_IFACE" ]; then
+    # See worker-init.sh for the dhcpcd-race RCA. Same recovery applies here.
+    if [ "$DHCP_TRIGGERED" = "false" ] && [ $RETRY_COUNT -ge 6 ] && command -v dhcpcd >/dev/null 2>&1; then
+      CANDIDATE=$(ip -o link show 2>/dev/null | awk -F': ' '/^[0-9]+: en[a-z]+[0-9]+s[0-9]+/ {print $2}' | grep -v '^eth' | head -1 || true)
+      if [ -n "$CANDIDATE" ]; then
+        DHCP_TRIGGERED=true
+        echo "Private NIC $CANDIDATE has no lease after 30s — triggering dhcpcd"
+        ip link set "$CANDIDATE" up 2>&1 || true
+        dhcpcd -1 "$CANDIDATE" 2>&1 | sed 's/^/  dhcpcd: /' || true
+      fi
+    fi
+    if [ $((RETRY_COUNT % 12)) -eq 0 ]; then
+      echo "Private interface not ready yet... ($RETRY_COUNT/$MAX_RETRIES)"
+    fi
+    RETRY_COUNT=$((RETRY_COUNT + 1))
+    sleep 5
+  fi
+done
+if [ -z "$PRIVATE_IFACE" ]; then
+  echo "FATAL: Private network interface not found after $MAX_RETRIES retries"
+  echo "Hetzner private network may not be attached to this server."
+  exit 1
+fi
+echo "Private interface: $PRIVATE_IFACE"
+PRIVATE_IP=$(ip -4 addr show "$PRIVATE_IFACE" 2>/dev/null | grep -oP 'inet \K[\d.]+' || echo "")
+
+# Install k3s master with pre-shared token.
+# --node-ip must be the PUBLIC IP so Hetzner CCM can match it against the Hetzner API
+# and remove the node.cloudprovider.kubernetes.io/uninitialized taint.
+# --flannel-iface routes pod-to-pod traffic over the private network.
+#
+# The k3s install script itself fetches https://github.com/k3s-io/k3s/releases/... and
+# GitHub routinely returns 504 for brief windows. The outer curl has --retry 3, but
+# GitHub failures happen inside the install script where we can't hand --retry through.
+# Wrap the whole pipeline in a bash retry loop — up to 5 attempts with 20s backoff —
+# so a transient CDN hiccup doesn't kill a fresh provision.
+K3S_INSTALL_OK=false
+for attempt in 1 2 3 4 5; do
+  if curl -sfL --max-time 120 --retry 3 --retry-delay 10 https://get.k3s.io | INSTALL_K3S_VERSION="${k3s_version}" sh -s - server \
+      --cluster-init \
+      --token "${k3s_token}" \
+      --tls-san "$PUBLIC_IP" \
+      --tls-san "${floating_ip}" \
+      --node-ip "$PUBLIC_IP" \
+      --node-external-ip "$PUBLIC_IP" \
+      --advertise-address "$PUBLIC_IP" \
+      --flannel-iface "$PRIVATE_IFACE" \
+      --disable traefik \
+      --disable servicelb \
+      --disable-cloud-controller \
+      --flannel-backend=wireguard-native \
+      --kubelet-arg="cloud-provider=external" \
+      --write-kubeconfig-mode=644; then
+    K3S_INSTALL_OK=true
+    break
+  fi
+  echo "k3s install attempt $attempt failed (likely transient GitHub/CDN issue), retrying in 20s..."
+  sleep 20
+done
+if [ "$K3S_INSTALL_OK" != "true" ]; then
+  echo "k3s install failed after 5 attempts — check GitHub CDN status" >&2
+  exit 1
+fi
+
+# Wait for k3s to be ready (max 3 minutes — if it hasn't started by then, something is wrong)
+K3S_READY=false
+for i in $(seq 1 36); do
+  if kubectl get nodes 2>/dev/null | grep -q "Ready"; then
+    K3S_READY=true
+    break
+  fi
+  echo "Waiting for k3s to be ready... ($i/36)"
+  sleep 5
+done
+if [ "$K3S_READY" != "true" ]; then
+  echo "FATAL: k3s did not become ready within 3 minutes"
+  echo "k3s service status:"
+  systemctl status k3s --no-pager 2>&1 || true
+  journalctl -u k3s --no-pager -n 20 2>&1 || true
+  exit 1
+fi
+
+# Marker for vibecarbon's deployK3s waiter — set AFTER hcloud Secret + CCM/CSI
+# install below complete (see end of script). Keeps the early-readiness signal
+# distinct from "fully provisioned and ready for app install."
+
+# Create secret for Hetzner Cloud Controller Manager
+kubectl -n kube-system create secret generic hcloud \
+  --from-literal=token="${hcloud_token}" \
+  --from-literal=network="${network_id}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# Create secret for Hetzner CSI Driver
+kubectl -n kube-system create secret generic hcloud-csi \
+  --from-literal=token="${hcloud_token}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# Install Hetzner Cloud Controller Manager
+# Use ccm.yaml (not ccm-networks.yaml) since k3s manages pod networking via flannel
+# Retry up to 3 times — kubectl apply can fail if API server is briefly unavailable
+for i in 1 2 3; do
+  kubectl apply -f https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.20.0/ccm.yaml && break
+  echo "CCM install attempt $i failed, retrying in 10s..."
+  sleep 10
+done
+
+# Install Hetzner CSI Driver
+for i in 1 2 3; do
+  kubectl apply -f https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.9.0/deploy/kubernetes/hcloud-csi.yml && break
+  echo "CSI install attempt $i failed, retrying in 10s..."
+  sleep 10
+done
+
+echo "k3s master installation complete!"
+
+# Final marker: vibecarbon's deployK3s polls for /tmp/k3s-ready before fetching
+# kubeconfig + applying app manifests. Written here so the marker only fires
+# after k3s + hcloud secrets + CCM + CSI are all in place — i.e., the cluster
+# is ready for real workloads, not just "k3s daemon is up."
+touch /tmp/k3s-ready

package/carbon/cloud-init/k3s/supabase-init.sh

@@ -0,0 +1,167 @@
+#!/bin/bash
+set -euo pipefail
+
+# Note: This script runs AS PART of cloud-init (user_data), so we cannot wait for
+# cloud-init to complete (that would cause a deadlock). The script starts immediately.
+
+# Wait for any dpkg locks held by unattended-upgrades (runs on first boot).
+for i in $(seq 1 30); do
+  if ! fuser /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1; then
+    break
+  fi
+  echo "Waiting for dpkg lock to be released... ($i/30)"
+  sleep 10
+done
+
+# Update apt lists + install just what we need. `apt-get upgrade` was removed
+# from this path — we don't need every security update applied before the node
+# joins the cluster (unattended-upgrades handles that async in the background),
+# and it used to add 30-60s to every deploy.
+apt-get update -qq
+apt-get install -y -qq curl jq
+
+# Validate k3s version is set
+K3S_TARGET_VERSION="${k3s_version}"
+if [ -z "$K3S_TARGET_VERSION" ]; then
+  echo "ERROR: k3s_version not set"
+  exit 1
+fi
+echo "Installing k3s version: $K3S_TARGET_VERSION"
+
+# IDENTICAL block in master-init.sh / worker-init.sh / supabase-init.sh — keep in sync.
+# (Three-way duplication is acceptable here because the cloud-init scripts are
+# rendered independently per role; extracting a shared snippet would require
+# restructuring the renderScript() pipeline. Out of scope for Phase 1.)
+# Configure containerd to mirror pulls for 10.0.1.1:5000 to the local registry
+# running on master. Cluster-autoscaler-spawned workers pull the app image from
+# here on-demand (sideload only reaches workers that exist at sideload time).
+# k3s only reads /etc/rancher/k3s/registries.yaml on agent start, so this MUST
+# land before the k3s install step below.
+#
+# Plain http://10.0.1.1:5000 — the registry runs HTTP on the private network.
+# Do NOT add `insecure_skip_verify: true` here: that flag only affects HTTPS
+# endpoints, and recent k3s versions place it incorrectly when present
+# (k3s-io/k3s#13215), which can break containerd config generation.
+mkdir -p /etc/rancher/k3s
+cat > /etc/rancher/k3s/registries.yaml << 'REGEOF'
+mirrors:
+  "10.0.1.1:5000":
+    endpoint:
+      - "http://10.0.1.1:5000"
+REGEOF
+
+# Get node IPs for k3s configuration
+PUBLIC_IP=$(curl -s --max-time 10 http://169.254.169.254/hetzner/v1/metadata/public-ipv4)
+
+# Wait for the private network interface (10.0.x.x) to be assigned.
+# Hetzner attaches the private network after the server boots, so the interface
+# may not exist yet when cloud-init runs. grep exits 1 if no match, which kills
+# the script under set -euo pipefail — so we retry with || true.
+#
+# This must happen BEFORE the master-readyz probe below, because that probe
+# uses the master's private IP (10.0.1.1) — unreachable until our private NIC
+# is up. See worker-init.sh for the full RCA.
+PRIVATE_IFACE=""
+RETRY_COUNT=0
+MAX_RETRIES=120
+DHCP_TRIGGERED=false
+echo "Waiting for private network interface..."
+while [ -z "$PRIVATE_IFACE" ] && [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+  PRIVATE_IFACE=$(ip -4 addr show | grep 'inet 10\.0\.' | awk '{print $NF}' || true)
+  if [ -z "$PRIVATE_IFACE" ]; then
+    if [ "$DHCP_TRIGGERED" = "false" ] && [ $RETRY_COUNT -ge 6 ] && command -v dhcpcd >/dev/null 2>&1; then
+      CANDIDATE=$(ip -o link show 2>/dev/null | awk -F': ' '/^[0-9]+: en[a-z]+[0-9]+s[0-9]+/ {print $2}' | grep -v '^eth' | head -1 || true)
+      if [ -n "$CANDIDATE" ]; then
+        DHCP_TRIGGERED=true
+        echo "Private NIC $CANDIDATE has no lease after 30s — triggering dhcpcd"
+        ip link set "$CANDIDATE" up 2>&1 || true
+        dhcpcd -1 "$CANDIDATE" 2>&1 | sed 's/^/  dhcpcd: /' || true
+      fi
+    fi
+    if [ $((RETRY_COUNT % 12)) -eq 0 ]; then
+      echo "Private interface not ready yet... ($RETRY_COUNT/$MAX_RETRIES)"
+    fi
+    RETRY_COUNT=$((RETRY_COUNT + 1))
+    sleep 5
+  fi
+done
+if [ -z "$PRIVATE_IFACE" ]; then
+  echo "FATAL: Private network interface not found after $MAX_RETRIES retries"
+  exit 1
+fi
+echo "Private interface: $PRIVATE_IFACE"
+PRIVATE_IP=$(ip -4 addr show "$PRIVATE_IFACE" 2>/dev/null | grep -oP 'inet \K[\d.]+' || echo "")
+
+# Wait for master k3s API to be ready before joining (uses private IP — must
+# come AFTER private-NIC bring-up above).
+MAX_RETRIES=120
+RETRY_COUNT=0
+
+echo "Waiting for master k3s API to be ready..."
+while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+  if curl -sk --max-time 5 "https://${master_ip}:6443/readyz" >/dev/null 2>&1; then
+    echo "Master k3s API is ready!"
+    break
+  fi
+
+  if [ $((RETRY_COUNT % 12)) -eq 0 ]; then
+    echo "Master not ready yet... ($RETRY_COUNT/$MAX_RETRIES)"
+  fi
+  RETRY_COUNT=$((RETRY_COUNT + 1))
+  sleep 5
+done
+
+if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
+  echo "Warning: Master may not be fully ready, attempting to join anyway..."
+fi
+
+# Install k3s agent as dedicated Supabase node.
+# --node-ip must be the PUBLIC IP so Hetzner CCM can match it (see master-init.sh).
+# Bash retry loop around the whole pipeline — the k3s install script fetches
+# from github.com which returns transient 504s under load. 5 attempts, 20s backoff.
+K3S_INSTALL_OK=false
+for attempt in 1 2 3 4 5; do
+  if curl -sfL --max-time 120 --retry 3 --retry-delay 10 https://get.k3s.io | INSTALL_K3S_VERSION="${k3s_version}" K3S_URL="https://${master_ip}:6443" K3S_TOKEN="${k3s_token}" sh -s - agent \
+      --node-ip "$PUBLIC_IP" \
+      --node-external-ip "$PUBLIC_IP" \
+      --flannel-iface "$PRIVATE_IFACE" \
+      --kubelet-arg="cloud-provider=external" \
+      --node-label="dedicated=supabase" \
+      --node-label="node-pool=supabase-pool" \
+      --node-taint="dedicated=supabase:NoSchedule"; then
+    K3S_INSTALL_OK=true
+    break
+  fi
+  echo "k3s supabase install attempt $attempt failed (likely transient GitHub/CDN issue), retrying in 20s..."
+  sleep 20
+done
+if [ "$K3S_INSTALL_OK" != "true" ]; then
+  echo "k3s supabase install failed after 5 attempts — check GitHub CDN status" >&2
+  exit 1
+fi
+
+echo "k3s supabase node installation complete!"
+
+# Pre-warm containerd image cache in the background so the Supabase HelmRelease
+# doesn't pay the full ~400MB postgres pull on its critical path. We only
+# pre-pull stable-name images that supabase-community pins conservatively;
+# version drift just means a cache miss (not wrong data).
+#
+# Runs async so the node can be marked Ready immediately — the main deploy
+# doesn't block on this. If the pull is still in-flight when the chart
+# schedules a pod, containerd deduplicates the concurrent pulls anyway.
+nohup bash -c '
+  for i in $(seq 1 60); do
+    [ -S /run/k3s/containerd/containerd.sock ] && break
+    sleep 5
+  done
+  # Heaviest base images in the Supabase stack (~400MB combined). Pinned
+  # tags match supabase-community chart defaults at time of authoring.
+  for img in \
+    docker.io/supabase/postgres:15.1.1.78 \
+    docker.io/library/kong:2.8.1 \
+  ; do
+    /usr/local/bin/k3s crictl pull "$img" 2>&1 || echo "Pre-pull skipped: $img"
+  done
+' > /var/log/supabase-image-prewarm.log 2>&1 &
+

package/carbon/cloud-init/k3s/worker-init.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+set -euo pipefail
+
+# Note: This script runs AS PART of cloud-init (user_data), so we cannot wait for
+# cloud-init to complete (that would cause a deadlock). The script starts immediately.
+
+# Wait for any dpkg locks held by unattended-upgrades (runs on first boot).
+for i in $(seq 1 30); do
+  if ! fuser /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1; then
+    break
+  fi
+  echo "Waiting for dpkg lock to be released... ($i/30)"
+  sleep 10
+done
+
+# Update apt lists + install the packages we need. `apt-get upgrade` was
+# removed from the critical path — unattended-upgrades handles security
+# patches async. Saves 30-60s per deploy.
+apt-get update -qq
+apt-get install -y -qq curl jq
+
+# Validate k3s version is set
+K3S_TARGET_VERSION="${k3s_version}"
+if [ -z "$K3S_TARGET_VERSION" ]; then
+  echo "ERROR: k3s_version not set"
+  exit 1
+fi
+echo "Installing k3s version: $K3S_TARGET_VERSION"
+
+# IDENTICAL block in master-init.sh / worker-init.sh / supabase-init.sh — keep in sync.
+# (Three-way duplication is acceptable here because the cloud-init scripts are
+# rendered independently per role; extracting a shared snippet would require
+# restructuring the renderScript() pipeline. Out of scope for Phase 1.)
+# Configure containerd to mirror pulls for 10.0.1.1:5000 to the local registry
+# running on master. Cluster-autoscaler-spawned workers pull the app image from
+# here on-demand (sideload only reaches workers that exist at sideload time).
+# k3s only reads /etc/rancher/k3s/registries.yaml on agent start, so this MUST
+# land before the k3s install step below.
+#
+# Plain http://10.0.1.1:5000 — the registry runs HTTP on the private network.
+# Do NOT add `insecure_skip_verify: true` here: that flag only affects HTTPS
+# endpoints, and recent k3s versions place it incorrectly when present
+# (k3s-io/k3s#13215), which can break containerd config generation.
+mkdir -p /etc/rancher/k3s
+cat > /etc/rancher/k3s/registries.yaml << 'REGEOF'
+mirrors:
+  "10.0.1.1:5000":
+    endpoint:
+      - "http://10.0.1.1:5000"
+REGEOF
+
+# Get node IPs for k3s configuration
+PUBLIC_IP=$(curl -s --max-time 10 http://169.254.169.254/hetzner/v1/metadata/public-ipv4)
+
+# Wait for the private network interface (10.0.x.x) to be assigned.
+# Hetzner attaches the private network after the server boots, so the interface
+# may not exist yet when cloud-init runs. grep exits 1 if no match, which kills
+# the script under set -euo pipefail — so we retry with || true.
+#
+# This must happen BEFORE the master-readyz probe below, because that probe
+# uses the master's private IP (10.0.1.1) — unreachable until our private NIC
+# is up. Workers stuck in master-readyz with no private NIC was the failure
+# mode in k8s-ha 2026-05-01 deploy run (yir52p, primary-worker-1): hung 600s
+# in "Master not ready yet..." while enp7s0 sat in state DOWN with no IP.
+PRIVATE_IFACE=""
+RETRY_COUNT=0
+MAX_RETRIES=120
+DHCP_TRIGGERED=false
+echo "Waiting for private network interface..."
+while [ -z "$PRIVATE_IFACE" ] && [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+  PRIVATE_IFACE=$(ip -4 addr show | grep 'inet 10\.0\.' | awk '{print $NF}' || true)
+  if [ -z "$PRIVATE_IFACE" ]; then
+    # After 30s with no 10.0.x.x address, actively trigger dhcpcd on any
+    # unmanaged enp* NIC. Hetzner ships dhcpcd-base + auto-starts dhcpcd at
+    # boot, but it races with the post-boot Hetzner private-network
+    # attachment. When dhcpcd loses the race the interface stays DOWN and no
+    # IP is ever requested — running `dhcpcd -1 <iface>` once obtains the
+    # lease idempotently. Skip if already triggered or dhcpcd missing.
+    if [ "$DHCP_TRIGGERED" = "false" ] && [ $RETRY_COUNT -ge 6 ] && command -v dhcpcd >/dev/null 2>&1; then
+      CANDIDATE=$(ip -o link show 2>/dev/null | awk -F': ' '/^[0-9]+: en[a-z]+[0-9]+s[0-9]+/ {print $2}' | grep -v '^eth' | head -1 || true)
+      if [ -n "$CANDIDATE" ]; then
+        DHCP_TRIGGERED=true
+        echo "Private NIC $CANDIDATE has no lease after 30s — triggering dhcpcd"
+        ip link set "$CANDIDATE" up 2>&1 || true
+        dhcpcd -1 "$CANDIDATE" 2>&1 | sed 's/^/  dhcpcd: /' || true
+      fi
+    fi
+    if [ $((RETRY_COUNT % 12)) -eq 0 ]; then
+      echo "Private interface not ready yet... ($RETRY_COUNT/$MAX_RETRIES)"
+    fi
+    RETRY_COUNT=$((RETRY_COUNT + 1))
+    sleep 5
+  fi
+done
+if [ -z "$PRIVATE_IFACE" ]; then
+  echo "FATAL: Private network interface not found after $MAX_RETRIES retries"
+  exit 1
+fi
+echo "Private interface: $PRIVATE_IFACE"
+PRIVATE_IP=$(ip -4 addr show "$PRIVATE_IFACE" 2>/dev/null | grep -oP 'inet \K[\d.]+' || echo "")
+
+# Wait for master k3s API to be ready before joining (uses private IP — must
+# come AFTER private-NIC bring-up above).
+MAX_RETRIES=120
+RETRY_COUNT=0
+
+echo "Waiting for master k3s API to be ready..."
+while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+  if curl -sk --max-time 5 "https://${master_ip}:6443/readyz" >/dev/null 2>&1; then
+    echo "Master k3s API is ready!"
+    break
+  fi
+
+  if [ $((RETRY_COUNT % 12)) -eq 0 ]; then
+    echo "Master not ready yet... ($RETRY_COUNT/$MAX_RETRIES)"
+  fi
+  RETRY_COUNT=$((RETRY_COUNT + 1))
+  sleep 5
+done
+
+if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
+  echo "Warning: Master may not be fully ready, attempting to join anyway..."
+fi
+
+# Install k3s agent with pre-shared token.
+# --node-ip must be the PUBLIC IP so Hetzner CCM can match it (see master-init.sh).
+# Bash retry loop around the whole pipeline — the k3s install script fetches
+# from github.com which returns transient 504s under load. 5 attempts, 20s backoff.
+K3S_INSTALL_OK=false
+for attempt in 1 2 3 4 5; do
+  if curl -sfL --max-time 120 --retry 3 --retry-delay 10 https://get.k3s.io | INSTALL_K3S_VERSION="${k3s_version}" K3S_URL="https://${master_ip}:6443" K3S_TOKEN="${k3s_token}" sh -s - agent \
+      --node-ip "$PUBLIC_IP" \
+      --node-external-ip "$PUBLIC_IP" \
+      --flannel-iface "$PRIVATE_IFACE" \
+      --kubelet-arg="cloud-provider=external"; then
+    K3S_INSTALL_OK=true
+    break
+  fi
+  echo "k3s worker install attempt $attempt failed (likely transient GitHub/CDN issue), retrying in 20s..."
+  sleep 20
+done
+if [ "$K3S_INSTALL_OK" != "true" ]; then
+  echo "k3s worker install failed after 5 attempts — check GitHub CDN status" >&2
+  exit 1
+fi
+
+echo "k3s worker installation complete!"

package/carbon/components.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "https://ui.shadcn.com/schema.json",
+  "style": "base-maia",
+  "rsc": false,
+  "tsx": true,
+  "tailwind": {
+    "config": "",
+    "css": "src/client/index.css",
+    "baseColor": "gray",
+    "cssVariables": true,
+    "prefix": ""
+  },
+  "iconLibrary": "lucide",
+  "aliases": {
+    "components": "@/components",
+    "utils": "@/lib/utils",
+    "ui": "@/components/ui",
+    "lib": "@/lib",
+    "hooks": "@/hooks"
+  },
+  "menuColor": "default",
+  "menuAccent": "bold",
+  "registries": {}
+}

package/carbon/content/blog/authentication-guide.mdx

@@ -0,0 +1,34 @@
+---
+title: Authentication Guide
+description: A complete guide to authentication features including OAuth, magic links, and MFA.
+date: "2025-01-20"
+author: Team
+---
+
+# Authentication Guide
+
+Your application comes with a full authentication system powered by Supabase Auth.
+
+## Supported Methods
+
+- **Email & Password** — Traditional sign-up and sign-in
+- **OAuth Providers** — Google, Microsoft, GitHub, Apple, Discord
+- **Magic Links** — Passwordless sign-in via email
+- **Multi-Factor Authentication** — TOTP-based MFA for added security
+
+## Enabling OAuth Providers
+
+OAuth providers are configured in your Supabase dashboard and toggled via environment variables:
+
+```bash
+GITHUB_ENABLED=true
+APPLE_ENABLED=true
+DISCORD_ENABLED=true
+MAGIC_LINK_ENABLED=true
+```
+
+The sign-in page automatically shows buttons for enabled providers.
+
+## Row-Level Security
+
+All database access is secured with Row-Level Security (RLS) policies. Users can only access data they're authorized to see, enforced at the database level.

package/carbon/content/blog/getting-started.mdx

@@ -0,0 +1,41 @@
+---
+title: Getting Started
+description: Learn how to set up your project and start building your SaaS application.
+date: "2025-01-15"
+author: Team
+---
+
+# Getting Started
+
+Welcome to your new SaaS application! This guide will help you get up and running quickly.
+
+## Prerequisites
+
+Before you begin, make sure you have:
+
+- **Node.js 22+** installed
+- **Docker** running for local development
+- **pnpm** as your package manager
+
+## Quick Start
+
+1. Start the development environment:
+
+```bash
+pnpm dev:start
+```
+
+This command will:
+- Start Docker containers (Supabase, PostgreSQL, etc.)
+- Run database migrations
+- Launch the API server and Vite dev server
+
+2. Open your browser to `http://app.localhost`
+
+3. Sign in with your admin credentials
+
+## Next Steps
+
+- Configure your [Stripe billing](/settings/billing) to start accepting payments
+- Set up [OAuth providers](/settings/security) for social login
+- Invite your [team members](/dashboard) to collaborate

package/carbon/content/changelog/v0-1-0.mdx

@@ -0,0 +1,40 @@
+---
+title: "Initial Release"
+description: "The first release of {{PROJECT_NAME}} with core features including authentication, billing, and infrastructure."
+date: "2025-01-15"
+version: "0.1.0"
+---
+
+## What's New
+
+### Authentication
+- Email/password sign-up and sign-in
+- OAuth providers (Google, Microsoft, GitHub, Apple, Discord)
+- Magic link authentication
+- Multi-factor authentication (TOTP)
+- Account lockout protection
+
+### Billing
+- Stripe integration with subscription management
+- Free, Startup, and Pro plans
+- Customer portal for self-service billing
+- Webhook-driven subscription lifecycle
+
+### Infrastructure
+- Self-hosted Supabase (PostgreSQL, Auth, REST API, Realtime, Storage)
+- Docker Compose for local development
+- Kubernetes manifests with Kustomize overlays
+- High-availability deployment with PostgreSQL replication
+- Traefik reverse proxy with automatic SSL
+
+### Admin Panel
+- Super admin dashboard with service health monitoring
+- User and organization management
+- System-wide notification management
+- User impersonation for support
+
+### Developer Experience
+- Full TypeScript across client and server
+- Biome for linting and formatting
+- Vitest test suite
+- AI editor rules (Claude Code, Cursor, Windsurf, Copilot)