vibecarbon 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +663 -0
- package/README.md +188 -0
- package/carbon/.claude/hooks/task-completed-gate.sh +19 -0
- package/carbon/.claude/hooks/teammate-idle-gate.sh +29 -0
- package/carbon/.claude/settings.json +29 -0
- package/carbon/.cursor/rules/vibecarbon.mdc +6 -0
- package/carbon/.dockerignore +57 -0
- package/carbon/.env.example +219 -0
- package/carbon/.github/copilot-instructions.md +3 -0
- package/carbon/.github/workflows/deploy.yml +346 -0
- package/carbon/.github/workflows/vibecarbon-build.yml +81 -0
- package/carbon/.windsurfrules +74 -0
- package/carbon/AGENTS.md +422 -0
- package/carbon/CLAUDE.md +3 -0
- package/carbon/DEVELOPMENT.md +187 -0
- package/carbon/Dockerfile +98 -0
- package/carbon/LICENSE +21 -0
- package/carbon/PRODUCTION.md +364 -0
- package/carbon/README.md +193 -0
- package/carbon/TESTING.md +138 -0
- package/carbon/backup/Dockerfile +75 -0
- package/carbon/backup/backup.sh +95 -0
- package/carbon/backup/compose-backup.sh +140 -0
- package/carbon/biome.json +83 -0
- package/carbon/cloud-init/docker-ce-setup.yaml +55 -0
- package/carbon/cloud-init/k3s/master-init.sh +215 -0
- package/carbon/cloud-init/k3s/supabase-init.sh +167 -0
- package/carbon/cloud-init/k3s/worker-init.sh +147 -0
- package/carbon/components.json +24 -0
- package/carbon/content/blog/authentication-guide.mdx +34 -0
- package/carbon/content/blog/getting-started.mdx +41 -0
- package/carbon/content/changelog/v0-1-0.mdx +40 -0
- package/carbon/content/docs/analytics.mdx +90 -0
- package/carbon/content/docs/authentication.mdx +231 -0
- package/carbon/content/docs/background-jobs.mdx +116 -0
- package/carbon/content/docs/cli.mdx +630 -0
- package/carbon/content/docs/database.mdx +236 -0
- package/carbon/content/docs/deployment.mdx +227 -0
- package/carbon/content/docs/development.mdx +238 -0
- package/carbon/content/docs/environments.mdx +84 -0
- package/carbon/content/docs/getting-started.mdx +112 -0
- package/carbon/content/docs/legal/privacy-policy.mdx +89 -0
- package/carbon/content/docs/legal/terms-of-service.mdx +99 -0
- package/carbon/content/docs/optional-services.mdx +160 -0
- package/carbon/db/Dockerfile +23 -0
- package/carbon/docker-compose.dev-init.yml +5 -0
- package/carbon/docker-compose.metabase.override.yml +14 -0
- package/carbon/docker-compose.metabase.prod.yml +28 -0
- package/carbon/docker-compose.metabase.yml +84 -0
- package/carbon/docker-compose.n8n.override.yml +14 -0
- package/carbon/docker-compose.n8n.prod.yml +31 -0
- package/carbon/docker-compose.n8n.yml +97 -0
- package/carbon/docker-compose.observability.override.yml +18 -0
- package/carbon/docker-compose.observability.prod.yml +28 -0
- package/carbon/docker-compose.observability.yml +125 -0
- package/carbon/docker-compose.override.yml +28 -0
- package/carbon/docker-compose.prod.yml +294 -0
- package/carbon/docker-compose.yml +508 -0
- package/carbon/docker-entrypoint.sh +12 -0
- package/carbon/ha/activate-standby.sh +52 -0
- package/carbon/ha/primary-init.sql +36 -0
- package/carbon/ha/standby-init.sh +74 -0
- package/carbon/k8s/LICENSE +99 -0
- package/carbon/k8s/README.md +272 -0
- package/carbon/k8s/base/app/deployment.yaml +130 -0
- package/carbon/k8s/base/app/hpa.yaml +44 -0
- package/carbon/k8s/base/app/kustomization.yaml +10 -0
- package/carbon/k8s/base/app/network-policy.yaml +124 -0
- package/carbon/k8s/base/app/pdb.yaml +14 -0
- package/carbon/k8s/base/app/rbac.yaml +36 -0
- package/carbon/k8s/base/app/service.yaml +16 -0
- package/carbon/k8s/base/backup/cronjob.yaml +120 -0
- package/carbon/k8s/base/backup/kustomization.yaml +7 -0
- package/carbon/k8s/base/backup/network-policy.yaml +31 -0
- package/carbon/k8s/base/backup/rbac.yaml +7 -0
- package/carbon/k8s/base/cluster-autoscaler/deployment.yaml +103 -0
- package/carbon/k8s/base/cluster-autoscaler/kustomization.yaml +17 -0
- package/carbon/k8s/base/cluster-autoscaler/rbac.yaml +109 -0
- package/carbon/k8s/base/config/configmap.yaml +48 -0
- package/carbon/k8s/base/config/kustomization.yaml +8 -0
- package/carbon/k8s/base/hetzner-ccm/kustomization.yaml +9 -0
- package/carbon/k8s/base/hetzner-csi/kustomization.yaml +9 -0
- package/carbon/k8s/base/kustomization.yaml +43 -0
- package/carbon/k8s/base/namespace.yaml +7 -0
- package/carbon/k8s/base/network-policies.yaml +95 -0
- package/carbon/k8s/base/registry/kustomization.yaml +5 -0
- package/carbon/k8s/base/registry/local-registry.yaml +193 -0
- package/carbon/k8s/base/traefik/certificate.yaml +21 -0
- package/carbon/k8s/base/traefik/configmap.yaml +13 -0
- package/carbon/k8s/base/traefik/deployment.yaml +165 -0
- package/carbon/k8s/base/traefik/ingressroute.yaml +141 -0
- package/carbon/k8s/base/traefik/kustomization.yaml +11 -0
- package/carbon/k8s/base/traefik/middleware.yaml +109 -0
- package/carbon/k8s/base/traefik/network-policy.yaml +92 -0
- package/carbon/k8s/base/traefik/service.yaml +38 -0
- package/carbon/k8s/flux/README.md +49 -0
- package/carbon/k8s/flux/clusters/primary/vibecarbon.yaml +128 -0
- package/carbon/k8s/flux/clusters/standby/vibecarbon.yaml +83 -0
- package/carbon/k8s/gitops/cert-manager-webhook-hetzner/helm-release.yaml +38 -0
- package/carbon/k8s/gitops/cert-manager-webhook-hetzner/helm-repository.yaml +16 -0
- package/carbon/k8s/gitops/cert-manager-webhook-hetzner/kustomization.yaml +10 -0
- package/carbon/k8s/gitops/supabase/helm-release.yaml +66 -0
- package/carbon/k8s/gitops/supabase/helm-repository.yaml +18 -0
- package/carbon/k8s/gitops/supabase/kustomization.yaml +33 -0
- package/carbon/k8s/infra/cert-manager-resources/cluster-issuers-cloudflare.yaml +51 -0
- package/carbon/k8s/infra/cert-manager-resources/cluster-issuers-hetzner.yaml +59 -0
- package/carbon/k8s/infra/cert-manager-resources/cluster-issuers-manual.yaml +43 -0
- package/carbon/k8s/infra/cert-manager-resources/kustomization.yaml +15 -0
- package/carbon/k8s/infra/kustomization.yaml +21 -0
- package/carbon/k8s/infra/traefik-crds/kustomization.yaml +10 -0
- package/carbon/k8s/overlays/local/configmap.yaml +23 -0
- package/carbon/k8s/overlays/local/ingressroute-studio.yaml +104 -0
- package/carbon/k8s/overlays/local/kustomization.yaml +215 -0
- package/carbon/k8s/overlays/local/namespace.yaml +8 -0
- package/carbon/k8s/overlays/local/secrets.yaml +32 -0
- package/carbon/k8s/overlays/production/kustomization.yaml +34 -0
- package/carbon/k8s/test-local.sh +318 -0
- package/carbon/k8s/values/supabase.values.yaml +133 -0
- package/carbon/package.json +154 -0
- package/carbon/runtime.Dockerfile +17 -0
- package/carbon/scripts/_dev-jwt.mjs +45 -0
- package/carbon/scripts/check-rls.ts +53 -0
- package/carbon/scripts/dev-init.js +191 -0
- package/carbon/scripts/dev.js +191 -0
- package/carbon/scripts/docker-down.js +63 -0
- package/carbon/scripts/docker-logs.js +59 -0
- package/carbon/scripts/docker-up.js +222 -0
- package/carbon/scripts/generate-dev-configs.sh +131 -0
- package/carbon/scripts/generate-rss.ts +116 -0
- package/carbon/scripts/generate-sitemap.ts +102 -0
- package/carbon/scripts/k8s-apply.js +71 -0
- package/carbon/scripts/k8s-delete.js +75 -0
- package/carbon/scripts/lib/manifest.js +176 -0
- package/carbon/scripts/secret-scan.mjs +278 -0
- package/carbon/scripts/validate-dev-configs.sh +101 -0
- package/carbon/src/client/App.tsx +202 -0
- package/carbon/src/client/assets/hyperformant-dark.svg +29 -0
- package/carbon/src/client/assets/hyperformant-light.svg +29 -0
- package/carbon/src/client/assets/logos/aider.svg +1 -0
- package/carbon/src/client/assets/logos/antigravity.svg +1 -0
- package/carbon/src/client/assets/logos/bolt.svg +1 -0
- package/carbon/src/client/assets/logos/claude-code.svg +1 -0
- package/carbon/src/client/assets/logos/copilot.svg +1 -0
- package/carbon/src/client/assets/logos/cursor.svg +1 -0
- package/carbon/src/client/assets/logos/gemini-cli.svg +1 -0
- package/carbon/src/client/assets/logos/lovable.svg +1 -0
- package/carbon/src/client/assets/logos/openai-codex.svg +1 -0
- package/carbon/src/client/assets/logos/v0.svg +1 -0
- package/carbon/src/client/assets/logos/windsurf.svg +1 -0
- package/carbon/src/client/assets/vibecarbon-icon.svg +19 -0
- package/carbon/src/client/assets/vibecarbon-logo-dark.svg +29 -0
- package/carbon/src/client/assets/vibecarbon-logo-light.svg +29 -0
- package/carbon/src/client/components/AIIntegrationSection.tsx +120 -0
- package/carbon/src/client/components/AppSidebar.tsx +760 -0
- package/carbon/src/client/components/ArchitectureSection.tsx +46 -0
- package/carbon/src/client/components/CTAFooter.tsx +59 -0
- package/carbon/src/client/components/ComparisonSection.tsx +132 -0
- package/carbon/src/client/components/ContentPanel.tsx +66 -0
- package/carbon/src/client/components/ContentSkeleton.tsx +21 -0
- package/carbon/src/client/components/ErrorBoundary.tsx +46 -0
- package/carbon/src/client/components/FAQSection.tsx +76 -0
- package/carbon/src/client/components/FileUpload.tsx +210 -0
- package/carbon/src/client/components/HeaderActions.tsx +17 -0
- package/carbon/src/client/components/Hero.tsx +608 -0
- package/carbon/src/client/components/ImpersonationBanner.tsx +31 -0
- package/carbon/src/client/components/LanguageSwitcher.tsx +67 -0
- package/carbon/src/client/components/Logo.tsx +87 -0
- package/carbon/src/client/components/LogoStrip.tsx +76 -0
- package/carbon/src/client/components/MetricsStrip.tsx +48 -0
- package/carbon/src/client/components/Nav.tsx +195 -0
- package/carbon/src/client/components/NewsletterSignup.tsx +147 -0
- package/carbon/src/client/components/NotificationDrawer.tsx +171 -0
- package/carbon/src/client/components/PageHeader.tsx +24 -0
- package/carbon/src/client/components/PlanGate.tsx +36 -0
- package/carbon/src/client/components/PricingSection.tsx +371 -0
- package/carbon/src/client/components/SEO.tsx +34 -0
- package/carbon/src/client/components/SmoothScroll.tsx +45 -0
- package/carbon/src/client/components/TechStackSection.tsx +165 -0
- package/carbon/src/client/components/WorkflowSection.tsx +604 -0
- package/carbon/src/client/components/admin/DockerLogs.tsx +276 -0
- package/carbon/src/client/components/admin/PerformanceMetrics.tsx +230 -0
- package/carbon/src/client/components/admin/ServiceCard.tsx +45 -0
- package/carbon/src/client/components/admin/ServicesStatus.tsx +296 -0
- package/carbon/src/client/components/architecture/ArchitectureDiagram.tsx +413 -0
- package/carbon/src/client/components/auth/AuthProvider.tsx +466 -0
- package/carbon/src/client/components/effects/FilmGrainOverlay.tsx +35 -0
- package/carbon/src/client/components/effects/FresnelEdge.tsx +49 -0
- package/carbon/src/client/components/effects/GlowTracker.tsx +46 -0
- package/carbon/src/client/components/effects/SparkBurst.tsx +145 -0
- package/carbon/src/client/components/effects/VGlowEffect.tsx +83 -0
- package/carbon/src/client/components/effects/index.ts +2 -0
- package/carbon/src/client/components/layouts/SidebarLayout.tsx +20 -0
- package/carbon/src/client/components/scroll/ScrollSection.tsx +76 -0
- package/carbon/src/client/components/scroll/ScrollytellingProvider.tsx +81 -0
- package/carbon/src/client/components/scroll/index.ts +2 -0
- package/carbon/src/client/components/ui/accordion.tsx +71 -0
- package/carbon/src/client/components/ui/alert-dialog.tsx +162 -0
- package/carbon/src/client/components/ui/alert.tsx +73 -0
- package/carbon/src/client/components/ui/aspect-ratio.tsx +22 -0
- package/carbon/src/client/components/ui/avatar.tsx +91 -0
- package/carbon/src/client/components/ui/badge.tsx +50 -0
- package/carbon/src/client/components/ui/breadcrumb.tsx +100 -0
- package/carbon/src/client/components/ui/button-group.tsx +78 -0
- package/carbon/src/client/components/ui/button.tsx +120 -0
- package/carbon/src/client/components/ui/calendar.tsx +182 -0
- package/carbon/src/client/components/ui/card.tsx +85 -0
- package/carbon/src/client/components/ui/carousel.tsx +227 -0
- package/carbon/src/client/components/ui/chart.tsx +357 -0
- package/carbon/src/client/components/ui/checkbox.tsx +27 -0
- package/carbon/src/client/components/ui/collapsible.tsx +15 -0
- package/carbon/src/client/components/ui/command.tsx +178 -0
- package/carbon/src/client/components/ui/context-menu.tsx +233 -0
- package/carbon/src/client/components/ui/dialog.tsx +132 -0
- package/carbon/src/client/components/ui/drawer.tsx +118 -0
- package/carbon/src/client/components/ui/dropdown-menu.tsx +242 -0
- package/carbon/src/client/components/ui/empty.tsx +94 -0
- package/carbon/src/client/components/ui/field.tsx +226 -0
- package/carbon/src/client/components/ui/hover-card.tsx +44 -0
- package/carbon/src/client/components/ui/input-group.tsx +146 -0
- package/carbon/src/client/components/ui/input-otp.tsx +83 -0
- package/carbon/src/client/components/ui/input.tsx +20 -0
- package/carbon/src/client/components/ui/item.tsx +188 -0
- package/carbon/src/client/components/ui/kbd.tsx +26 -0
- package/carbon/src/client/components/ui/label.tsx +21 -0
- package/carbon/src/client/components/ui/menubar.tsx +250 -0
- package/carbon/src/client/components/ui/navigation-menu.tsx +155 -0
- package/carbon/src/client/components/ui/pagination.tsx +102 -0
- package/carbon/src/client/components/ui/popover.tsx +75 -0
- package/carbon/src/client/components/ui/progress.tsx +66 -0
- package/carbon/src/client/components/ui/radio-group.tsx +36 -0
- package/carbon/src/client/components/ui/resizable.tsx +46 -0
- package/carbon/src/client/components/ui/scroll-area.tsx +48 -0
- package/carbon/src/client/components/ui/select.tsx +186 -0
- package/carbon/src/client/components/ui/separator.tsx +21 -0
- package/carbon/src/client/components/ui/sheet.tsx +124 -0
- package/carbon/src/client/components/ui/sidebar.tsx +702 -0
- package/carbon/src/client/components/ui/skeleton.tsx +13 -0
- package/carbon/src/client/components/ui/slider.tsx +57 -0
- package/carbon/src/client/components/ui/sonner.tsx +45 -0
- package/carbon/src/client/components/ui/spinner.tsx +15 -0
- package/carbon/src/client/components/ui/switch.tsx +30 -0
- package/carbon/src/client/components/ui/table.tsx +89 -0
- package/carbon/src/client/components/ui/tabs.tsx +73 -0
- package/carbon/src/client/components/ui/textarea.tsx +18 -0
- package/carbon/src/client/components/ui/toggle-group.tsx +87 -0
- package/carbon/src/client/components/ui/toggle.tsx +44 -0
- package/carbon/src/client/components/ui/tooltip.tsx +56 -0
- package/carbon/src/client/hooks/api/index.ts +14 -0
- package/carbon/src/client/hooks/api/useAuthSettings.ts +80 -0
- package/carbon/src/client/hooks/api/useSubscription.ts +87 -0
- package/carbon/src/client/hooks/use-mobile.ts +21 -0
- package/carbon/src/client/hooks/useMousePosition.ts +91 -0
- package/carbon/src/client/hooks/useNotifications.tsx +124 -0
- package/carbon/src/client/hooks/useOrganizationMembers.ts +127 -0
- package/carbon/src/client/hooks/useOrganizations.tsx +230 -0
- package/carbon/src/client/hooks/usePackageManager.ts +69 -0
- package/carbon/src/client/hooks/useReducedMotion.ts +20 -0
- package/carbon/src/client/hooks/useRunningServices.ts +114 -0
- package/carbon/src/client/hooks/useScrollProgress.ts +56 -0
- package/carbon/src/client/index.css +467 -0
- package/carbon/src/client/index.html +56 -0
- package/carbon/src/client/lib/admin-services.ts +151 -0
- package/carbon/src/client/lib/api.ts +32 -0
- package/carbon/src/client/lib/blog.ts +35 -0
- package/carbon/src/client/lib/changelog.ts +33 -0
- package/carbon/src/client/lib/docs-search.ts +101 -0
- package/carbon/src/client/lib/docs.ts +37 -0
- package/carbon/src/client/lib/i18n.ts +32 -0
- package/carbon/src/client/lib/supabase.ts +72 -0
- package/carbon/src/client/lib/tailwind-colors.ts +357 -0
- package/carbon/src/client/lib/theme.ts +117 -0
- package/carbon/src/client/lib/utils.ts +22 -0
- package/carbon/src/client/locales/de.json +529 -0
- package/carbon/src/client/locales/en.json +461 -0
- package/carbon/src/client/locales/es.json +529 -0
- package/carbon/src/client/locales/fr.json +529 -0
- package/carbon/src/client/locales/pt.json +529 -0
- package/carbon/src/client/main.tsx +56 -0
- package/carbon/src/client/mdx.d.ts +13 -0
- package/carbon/src/client/pages/ApiDocs.tsx +76 -0
- package/carbon/src/client/pages/AuthCallback.tsx +34 -0
- package/carbon/src/client/pages/Blog.tsx +167 -0
- package/carbon/src/client/pages/Changelog.tsx +171 -0
- package/carbon/src/client/pages/Charts.tsx +388 -0
- package/carbon/src/client/pages/Checkout.tsx +227 -0
- package/carbon/src/client/pages/Contact.tsx +174 -0
- package/carbon/src/client/pages/Dashboard.tsx +368 -0
- package/carbon/src/client/pages/Docs.tsx +372 -0
- package/carbon/src/client/pages/ForgotPassword.tsx +111 -0
- package/carbon/src/client/pages/Home.tsx +187 -0
- package/carbon/src/client/pages/Legal.tsx +100 -0
- package/carbon/src/client/pages/Login.tsx +408 -0
- package/carbon/src/client/pages/MFAVerify.tsx +156 -0
- package/carbon/src/client/pages/NotFound.tsx +21 -0
- package/carbon/src/client/pages/Onboarding.tsx +246 -0
- package/carbon/src/client/pages/ResetPassword.tsx +200 -0
- package/carbon/src/client/pages/UIComponents.tsx +390 -0
- package/carbon/src/client/pages/admin/ContactSubmissions.tsx +220 -0
- package/carbon/src/client/pages/admin/Dashboard.tsx +24 -0
- package/carbon/src/client/pages/admin/Infrastructure.tsx +257 -0
- package/carbon/src/client/pages/admin/Jobs.tsx +225 -0
- package/carbon/src/client/pages/admin/Logs.tsx +18 -0
- package/carbon/src/client/pages/admin/Newsletter.tsx +300 -0
- package/carbon/src/client/pages/admin/Notifications.tsx +603 -0
- package/carbon/src/client/pages/admin/Organizations.tsx +306 -0
- package/carbon/src/client/pages/admin/Settings.tsx +314 -0
- package/carbon/src/client/pages/admin/Theme.tsx +465 -0
- package/carbon/src/client/pages/admin/Users.tsx +365 -0
- package/carbon/src/client/pages/api-docs.css +156 -0
- package/carbon/src/client/pages/organizations/Details.tsx +200 -0
- package/carbon/src/client/pages/organizations/Members.tsx +402 -0
- package/carbon/src/client/pages/settings/Billing.tsx +473 -0
- package/carbon/src/client/pages/settings/Profile.tsx +160 -0
- package/carbon/src/client/pages/settings/Security.tsx +341 -0
- package/carbon/src/client/public/favicon.svg +19 -0
- package/carbon/src/client/public/robots.txt +8 -0
- package/carbon/src/server/billing/index.ts +104 -0
- package/carbon/src/server/billing/provider.ts +81 -0
- package/carbon/src/server/billing/providers/paddle.ts +314 -0
- package/carbon/src/server/billing/providers/polar.ts +325 -0
- package/carbon/src/server/billing/providers/stripe.ts +233 -0
- package/carbon/src/server/emails/templates.ts +116 -0
- package/carbon/src/server/index.ts +554 -0
- package/carbon/src/server/lib/auth.ts +6 -0
- package/carbon/src/server/lib/email.ts +64 -0
- package/carbon/src/server/lib/env.ts +112 -0
- package/carbon/src/server/lib/errors.ts +21 -0
- package/carbon/src/server/lib/logger.ts +17 -0
- package/carbon/src/server/lib/rate-limiter.ts +288 -0
- package/carbon/src/server/lib/request.ts +34 -0
- package/carbon/src/server/lib/stripe.ts +42 -0
- package/carbon/src/server/lib/supabase.ts +65 -0
- package/carbon/src/server/middleware/requirePlan.ts +80 -0
- package/carbon/src/server/routes/_internal/services-status.ts +958 -0
- package/carbon/src/server/routes/_internal/verify-role.ts +185 -0
- package/carbon/src/server/routes/health.ts +48 -0
- package/carbon/src/server/routes/v1/admin/contact.ts +128 -0
- package/carbon/src/server/routes/v1/admin/jobs.ts +171 -0
- package/carbon/src/server/routes/v1/admin/newsletter.ts +237 -0
- package/carbon/src/server/routes/v1/auth.ts +390 -0
- package/carbon/src/server/routes/v1/billing.ts +718 -0
- package/carbon/src/server/routes/v1/contact.ts +93 -0
- package/carbon/src/server/routes/v1/index.ts +1333 -0
- package/carbon/src/server/routes/v1/newsletter.ts +181 -0
- package/carbon/src/server/routes/v1/performance.ts +157 -0
- package/carbon/src/server/routes/v1/stats.ts +170 -0
- package/carbon/src/server/routes/v1/theme.ts +106 -0
- package/carbon/src/server/routes/webhooks/billing.ts +376 -0
- package/carbon/src/server/routes/webhooks/stripe.ts +276 -0
- package/carbon/src/server/types.ts +11 -0
- package/carbon/src/shared/pricing.ts +155 -0
- package/carbon/src/shared/types.ts +338 -0
- package/carbon/supabase/migrations/00001_init.sql +717 -0
- package/carbon/supabase/migrations/00002_theme_settings.sql +13 -0
- package/carbon/supabase/migrations/00003_pg_cron.sql +121 -0
- package/carbon/supabase/migrations/00004_contact_newsletter.sql +81 -0
- package/carbon/supabase/migrations/00005_localization_languages.sql +22 -0
- package/carbon/supabase/seed.sql +16 -0
- package/carbon/tests/_helpers/app.ts +45 -0
- package/carbon/tests/_helpers/env.ts +37 -0
- package/carbon/tests/_helpers/factories.ts +69 -0
- package/carbon/tests/_helpers/jwt.ts +23 -0
- package/carbon/tests/_helpers/setup-integration.ts +20 -0
- package/carbon/tests/_helpers/setup-rtl.ts +12 -0
- package/carbon/tests/component/ErrorBoundary.test.tsx +53 -0
- package/carbon/tests/component/use-auth-settings.test.tsx +119 -0
- package/carbon/tests/integration/server/routes/contact.test.ts +162 -0
- package/carbon/tests/integration/server/routes/health.test.ts +61 -0
- package/carbon/tests/structural/i18n-parity.test.ts +42 -0
- package/carbon/tests/unit/client/utils.test.ts +49 -0
- package/carbon/tests/unit/shared/pricing.test.ts +93 -0
- package/carbon/tsconfig.json +27 -0
- package/carbon/tsconfig.server.json +27 -0
- package/carbon/tsconfig.test.json +9 -0
- package/carbon/vite.config.ts +110 -0
- package/carbon/vitest.config.ts +74 -0
- package/carbon/volumes/db/jwt.sql +57 -0
- package/carbon/volumes/db/metabase-init.sh +29 -0
- package/carbon/volumes/db/n8n-init.sh +25 -0
- package/carbon/volumes/db/realtime.sql +33 -0
- package/carbon/volumes/db/roles.sql +93 -0
- package/carbon/volumes/db/set-passwords.sh +12 -0
- package/carbon/volumes/db/super-admin.dev.sql +113 -0
- package/carbon/volumes/db/super-admin.generated.sql +113 -0
- package/carbon/volumes/db/super-admin.sql +114 -0
- package/carbon/volumes/grafana/dashboards/logs.json +179 -0
- package/carbon/volumes/grafana/dashboards/overview.json +523 -0
- package/carbon/volumes/grafana/dashboards/postgresql.json +337 -0
- package/carbon/volumes/grafana/dashboards.dev/logs.json +179 -0
- package/carbon/volumes/grafana/dashboards.dev/overview.json +156 -0
- package/carbon/volumes/grafana/dashboards.dev/postgresql.json +337 -0
- package/carbon/volumes/grafana/provisioning/dashboards/dashboards.dev.yml +16 -0
- package/carbon/volumes/grafana/provisioning/dashboards/dashboards.yml +16 -0
- package/carbon/volumes/grafana/provisioning/datasources/datasources.yml +43 -0
- package/carbon/volumes/kong/docker-entrypoint.sh +9 -0
- package/carbon/volumes/kong/kong.yml +208 -0
- package/carbon/volumes/loki/loki-config.yml +58 -0
- package/carbon/volumes/n8n/hooks.js +131 -0
- package/carbon/volumes/n8n/scripts/setup.sh +66 -0
- package/carbon/volumes/prometheus/prometheus.yml +43 -0
- package/carbon/volumes/promtail/promtail-config.yml +64 -0
- package/carbon/volumes/traefik/middlewares.dev.yml +83 -0
- package/carbon/volumes/traefik/middlewares.yml +95 -0
- package/carbon/volumes/traefik/vite-dev.yml +20 -0
- package/package.json +95 -0
- package/src/access.js +354 -0
- package/src/activate.js +187 -0
- package/src/add.js +718 -0
- package/src/backup.js +786 -0
- package/src/cli.js +350 -0
- package/src/configure.js +967 -0
- package/src/console.js +155 -0
- package/src/create.js +1499 -0
- package/src/deploy.js +311 -0
- package/src/destroy.js +2033 -0
- package/src/diagnose.js +735 -0
- package/src/down.js +80 -0
- package/src/failover.js +1032 -0
- package/src/lib/backup-s3.js +179 -0
- package/src/lib/build.js +33 -0
- package/src/lib/checksum.js +28 -0
- package/src/lib/ci-setup.js +666 -0
- package/src/lib/cli/help.js +129 -0
- package/src/lib/cli/parse-flags.js +160 -0
- package/src/lib/cli/select-action.js +65 -0
- package/src/lib/cli/select-environment.js +108 -0
- package/src/lib/cli/tty-guard.js +75 -0
- package/src/lib/cloudflare.js +447 -0
- package/src/lib/colors.js +52 -0
- package/src/lib/command.js +361 -0
- package/src/lib/config.js +359 -0
- package/src/lib/cost.js +103 -0
- package/src/lib/deploy/bundle.js +231 -0
- package/src/lib/deploy/compose/acme-verify.js +121 -0
- package/src/lib/deploy/compose/build-args.js +78 -0
- package/src/lib/deploy/compose/ha.js +1874 -0
- package/src/lib/deploy/compose/index.js +1294 -0
- package/src/lib/deploy/compose/reconcile.js +74 -0
- package/src/lib/deploy/github.js +382 -0
- package/src/lib/deploy/image.js +191 -0
- package/src/lib/deploy/index.js +119 -0
- package/src/lib/deploy/k8s/LICENSE +99 -0
- package/src/lib/deploy/k8s/gitops-deploy.js +298 -0
- package/src/lib/deploy/k8s/ha/index.js +1000 -0
- package/src/lib/deploy/k8s/index.js +62 -0
- package/src/lib/deploy/k8s/k3s.js +2515 -0
- package/src/lib/deploy/orchestrator.js +1401 -0
- package/src/lib/deploy/prompts.js +545 -0
- package/src/lib/deploy/remote-build.js +130 -0
- package/src/lib/deploy/state.js +120 -0
- package/src/lib/deploy/utils.js +328 -0
- package/src/lib/deploy-logger.js +93 -0
- package/src/lib/dns-propagation.js +48 -0
- package/src/lib/environment.js +58 -0
- package/src/lib/fetch-retry.js +103 -0
- package/src/lib/github-environments.js +335 -0
- package/src/lib/hetzner-dns.js +377 -0
- package/src/lib/hetzner-guided-setup.js +275 -0
- package/src/lib/host-keys.js +37 -0
- package/src/lib/iac/cloud-init.js +52 -0
- package/src/lib/iac/index.js +325 -0
- package/src/lib/iac/programs/hetzner-compose.js +130 -0
- package/src/lib/iac/programs/hetzner-k8s.js +320 -0
- package/src/lib/kubectl.js +81 -0
- package/src/lib/licensing/index.js +259 -0
- package/src/lib/licensing/tiers.js +181 -0
- package/src/lib/licensing/validator.js +171 -0
- package/src/lib/merge-package-json.js +90 -0
- package/src/lib/operator-ip.js +381 -0
- package/src/lib/package-manager.js +111 -0
- package/src/lib/perf.js +71 -0
- package/src/lib/project-guard.js +39 -0
- package/src/lib/project.js +334 -0
- package/src/lib/providers/base.js +276 -0
- package/src/lib/providers/hetzner-s3.js +656 -0
- package/src/lib/providers/hetzner.js +755 -0
- package/src/lib/providers/index.js +164 -0
- package/src/lib/s3.js +510 -0
- package/src/lib/secret-scan.js +583 -0
- package/src/lib/secrets.js +63 -0
- package/src/lib/server-types.js +195 -0
- package/src/lib/shell.js +91 -0
- package/src/lib/ssh.js +241 -0
- package/src/lib/tracker.js +242 -0
- package/src/lib/upgrade-policy.js +170 -0
- package/src/lib/validators.js +105 -0
- package/src/lib/version.js +5 -0
- package/src/remove.js +292 -0
- package/src/reset.js +97 -0
- package/src/restore.js +871 -0
- package/src/scale.js +1734 -0
- package/src/shell.js +222 -0
- package/src/status.js +981 -0
- package/src/up.js +264 -0
- package/src/upgrade.js +721 -0
package/src/scale.js
ADDED
|
@@ -0,0 +1,1734 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vibecarbon Scale Command
|
|
3
|
+
*
|
|
4
|
+
* Adjust instance types post-deploy without re-running the full deploy
|
|
5
|
+
* flow. Interactive-by-default; bare `vibecarbon scale` walks the
|
|
6
|
+
* operator through env → role → server-type prompts. Power users can
|
|
7
|
+
* seed env via `-env <name>` (or positional) and server type via
|
|
8
|
+
* `-type <id>` for fully-scripted scaling.
|
|
9
|
+
*
|
|
10
|
+
* Form rule: vibecarbon uses single-dash flags only — see
|
|
11
|
+
* memory:feedback_cli_single_dash_flags. Worker-bound flags
|
|
12
|
+
* (--min-workers / --max-workers) are gone; the interactive bounds
|
|
13
|
+
* prompt covers that need without tying up two top-level letters.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { existsSync, readFileSync, unlinkSync } from 'node:fs';
|
|
17
|
+
import os from 'node:os';
|
|
18
|
+
import { join } from 'node:path';
|
|
19
|
+
import * as p from '@clack/prompts';
|
|
20
|
+
import { getGHCRCredentials } from './lib/build.js';
|
|
21
|
+
import { renderHelp } from './lib/cli/help.js';
|
|
22
|
+
import { parseFlags } from './lib/cli/parse-flags.js';
|
|
23
|
+
import { requireTTYOrFlags } from './lib/cli/tty-guard.js';
|
|
24
|
+
import { c, printBanner } from './lib/colors.js';
|
|
25
|
+
import { runCommand } from './lib/command.js';
|
|
26
|
+
import { loadCredentials, saveProjectConfig } from './lib/config.js';
|
|
27
|
+
import { buildCostBreakdown, formatCostLines } from './lib/cost.js';
|
|
28
|
+
import {
|
|
29
|
+
backupCompose,
|
|
30
|
+
dockerLoginOnServer,
|
|
31
|
+
isLocalOnlyImageTag,
|
|
32
|
+
pullComposeImages,
|
|
33
|
+
restoreCompose,
|
|
34
|
+
setupComposeBackupCron,
|
|
35
|
+
setupServer,
|
|
36
|
+
setupServerFiles,
|
|
37
|
+
startComposeStack,
|
|
38
|
+
waitForSSH,
|
|
39
|
+
} from './lib/deploy/compose/index.js';
|
|
40
|
+
import { buildRemote } from './lib/deploy/remote-build.js';
|
|
41
|
+
import { DEFAULT_WORKER_MAX, DEFAULT_WORKER_MIN } from './lib/deploy/utils.js';
|
|
42
|
+
import { waitForDNSPropagation } from './lib/dns-propagation.js';
|
|
43
|
+
import { getApiToken, getS3Credentials } from './lib/hetzner-guided-setup.js';
|
|
44
|
+
import { requireLicense } from './lib/licensing/index.js';
|
|
45
|
+
import { ensureOperatorIpAccess } from './lib/operator-ip.js';
|
|
46
|
+
import { perfAsync } from './lib/perf.js';
|
|
47
|
+
import { assertInProjectDir } from './lib/project-guard.js';
|
|
48
|
+
import { HetznerProvider } from './lib/providers/hetzner.js';
|
|
49
|
+
import { buildComposeTypeOptions, buildSimpleTypeOptions } from './lib/server-types.js';
|
|
50
|
+
import { parseDotenv } from './lib/shell.js';
|
|
51
|
+
import { scpDownload, scpUpload, sshRun, sshRunScript } from './lib/ssh.js';
|
|
52
|
+
import { VERSION } from './lib/version.js';
|
|
53
|
+
|
|
54
|
+
// ============================================================================
|
|
55
|
+
// COMMAND SPEC — single source of truth for argv parsing AND help output.
|
|
56
|
+
// ============================================================================
|
|
57
|
+
|
|
58
|
+
/** @type {import('./lib/cli/parse-flags.js').CommandSpec & { summary?: string, description?: string, examples?: Array<{ command: string, description?: string }> }} */
|
|
59
|
+
const SPEC = {
|
|
60
|
+
name: 'scale',
|
|
61
|
+
summary: 'Adjust instance types post-deploy',
|
|
62
|
+
description: [
|
|
63
|
+
'What you can change:',
|
|
64
|
+
' Compose / Compose HA VPS server type (blue-green; zero downtime)',
|
|
65
|
+
' Kubernetes Worker / master / supabase server types (in-place)',
|
|
66
|
+
' Kubernetes Worker bounds (autoscaler floor + ceiling)',
|
|
67
|
+
].join('\n'),
|
|
68
|
+
positional: [
|
|
69
|
+
{
|
|
70
|
+
name: 'env',
|
|
71
|
+
optional: true,
|
|
72
|
+
description: 'Environment to scale (skips the env prompt)',
|
|
73
|
+
},
|
|
74
|
+
],
|
|
75
|
+
flags: [
|
|
76
|
+
{ name: 'h', boolean: true, description: 'Show this help' },
|
|
77
|
+
{ name: 'v', boolean: true, description: 'Show version' },
|
|
78
|
+
{ name: 'y', boolean: true, description: 'Skip confirmations (required with -type)' },
|
|
79
|
+
{ name: 'env', value: '<name>', description: 'Environment seed (alternative to positional)' },
|
|
80
|
+
{
|
|
81
|
+
name: 'type',
|
|
82
|
+
value: '<id>',
|
|
83
|
+
description:
|
|
84
|
+
'Server type to scale to (e.g. cx33). Compose: resizes the VPS. K8s: resizes all roles.',
|
|
85
|
+
},
|
|
86
|
+
],
|
|
87
|
+
examples: [
|
|
88
|
+
{ command: 'vibecarbon scale', description: 'prompts for env and changes' },
|
|
89
|
+
{ command: 'vibecarbon scale prod', description: 'env seeded; prompts for changes' },
|
|
90
|
+
{
|
|
91
|
+
command: 'vibecarbon scale -env prod -type cx33 -y',
|
|
92
|
+
description: 'scripted: resize all roles to cx33',
|
|
93
|
+
},
|
|
94
|
+
],
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
// ============================================================================
|
|
98
|
+
// COMPOSE SCALING
|
|
99
|
+
// ============================================================================
|
|
100
|
+
|
|
101
|
+
async function scaleCompose(environment, envConfig, projectConfig, options = {}) {
|
|
102
|
+
const servers = envConfig.servers || [];
|
|
103
|
+
const projectName = projectConfig.projectName;
|
|
104
|
+
|
|
105
|
+
// `s.id` is the Hetzner-assigned numeric server id, persisted by the
|
|
106
|
+
// orchestrator on every API-provisioned compose deploy. Servers deployed
|
|
107
|
+
// via --ssh against a pre-existing VPS legitimately have no id and can't
|
|
108
|
+
// be resized through the Hetzner API — the user has to resize via the
|
|
109
|
+
// provider console.
|
|
110
|
+
//
|
|
111
|
+
// Pre-release scope: there is intentionally no fallback to `hetznerName`
|
|
112
|
+
// lookup or to .vibecarbon.json shapes that predate the id-persistence
|
|
113
|
+
// fix. If you see this message after a fresh `vibecarbon deploy` against
|
|
114
|
+
// an automated provider, your .vibecarbon.json is from before the fix
|
|
115
|
+
// landed — re-deploy to refresh the shape (it's pre-release; re-deploys
|
|
116
|
+
// are cheap).
|
|
117
|
+
const resizableServers = servers.filter((s) => s.id);
|
|
118
|
+
if (resizableServers.length === 0) {
|
|
119
|
+
p.log.info('No servers with a Hetzner id found in .vibecarbon.json.');
|
|
120
|
+
p.log.info('If you deployed via --ssh: resize the VPS through your provider console.');
|
|
121
|
+
p.log.info('If you used automated provisioning: re-run `vibecarbon deploy` to refresh.');
|
|
122
|
+
p.outro('');
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Get API token
|
|
127
|
+
const apiToken = await getApiToken(projectName);
|
|
128
|
+
if (!apiToken) {
|
|
129
|
+
p.log.error('API token required for server scaling');
|
|
130
|
+
process.exit(1);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Ensure operator IP is in the firewall allowlist before any SSH work.
|
|
134
|
+
try {
|
|
135
|
+
const isHA = envConfig.deployMode === 'compose-ha';
|
|
136
|
+
const result = await ensureOperatorIpAccess({
|
|
137
|
+
projectConfig,
|
|
138
|
+
environment,
|
|
139
|
+
isHA,
|
|
140
|
+
apiToken,
|
|
141
|
+
yes: !!options.yes,
|
|
142
|
+
onMessage: (msg) => p.log.info(msg),
|
|
143
|
+
});
|
|
144
|
+
if (result.added) {
|
|
145
|
+
p.log.success(`Firewall updated: SSH now allows ${result.cidr}`);
|
|
146
|
+
}
|
|
147
|
+
} catch (err) {
|
|
148
|
+
p.log.error(`Operator-IP check failed: ${err.message}`);
|
|
149
|
+
process.exit(1);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Fetch live server types for accurate pricing
|
|
153
|
+
await HetznerProvider.fetchServerTypes(apiToken);
|
|
154
|
+
|
|
155
|
+
const region = envConfig.region || 'fsn1';
|
|
156
|
+
const currentType = envConfig.serverType || 'cpx21';
|
|
157
|
+
|
|
158
|
+
// Display current configuration
|
|
159
|
+
const pad = 18;
|
|
160
|
+
const configLines = [
|
|
161
|
+
`${'Environment'.padEnd(pad)} ${c.bold(environment)}`,
|
|
162
|
+
`${'Deploy mode'.padEnd(pad)} ${c.bold(envConfig.deployMode)}`,
|
|
163
|
+
`${'Region'.padEnd(pad)} ${c.bold(region)}`,
|
|
164
|
+
`${'Server type'.padEnd(pad)} ${c.bold(currentType)}`,
|
|
165
|
+
];
|
|
166
|
+
for (const server of servers) {
|
|
167
|
+
const role = server.role ? ` (${server.role})` : '';
|
|
168
|
+
const id = server.id ? ` [id: ${server.id}]` : ' [--ssh, not scalable]';
|
|
169
|
+
configLines.push(`${'Server'.padEnd(pad)} ${server.ip}${role}${id}`);
|
|
170
|
+
}
|
|
171
|
+
p.note(configLines.join('\n'), 'Current Configuration');
|
|
172
|
+
|
|
173
|
+
// For compose-ha: select which server(s) to scale
|
|
174
|
+
let targetServers = resizableServers;
|
|
175
|
+
if (envConfig.deployMode === 'compose-ha' && resizableServers.length > 1) {
|
|
176
|
+
const primaryServer = resizableServers.find((s) => s.role === 'primary');
|
|
177
|
+
const standbyServer = resizableServers.find((s) => s.role === 'standby');
|
|
178
|
+
|
|
179
|
+
// Non-interactive: default to both servers when --yes is set
|
|
180
|
+
let target;
|
|
181
|
+
if (options.yes) {
|
|
182
|
+
target = 'both';
|
|
183
|
+
p.log.info('Scaling both primary and standby servers');
|
|
184
|
+
} else {
|
|
185
|
+
target = await p.select({
|
|
186
|
+
message: 'Which server(s) to scale?',
|
|
187
|
+
options: [
|
|
188
|
+
{ value: 'both', label: 'Both primary and standby', hint: 'recommended' },
|
|
189
|
+
...(primaryServer
|
|
190
|
+
? [{ value: 'primary', label: `Primary only (${primaryServer.ip})` }]
|
|
191
|
+
: []),
|
|
192
|
+
...(standbyServer
|
|
193
|
+
? [{ value: 'standby', label: `Standby only (${standbyServer.ip})` }]
|
|
194
|
+
: []),
|
|
195
|
+
],
|
|
196
|
+
});
|
|
197
|
+
if (p.isCancel(target)) {
|
|
198
|
+
p.cancel('Operation cancelled.');
|
|
199
|
+
process.exit(0);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
if (target === 'primary') targetServers = primaryServer ? [primaryServer] : [];
|
|
204
|
+
else if (target === 'standby') targetServers = standbyServer ? [standbyServer] : [];
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// Select new server type
|
|
208
|
+
const regionTypes = HetznerProvider.getServerTypesForRegion(region);
|
|
209
|
+
const typeOptions = buildComposeTypeOptions(regionTypes, currentType);
|
|
210
|
+
|
|
211
|
+
let newType;
|
|
212
|
+
if (options.type) {
|
|
213
|
+
newType = options.type;
|
|
214
|
+
p.log.info(`Using target server type: ${c.bold(newType)}`);
|
|
215
|
+
} else {
|
|
216
|
+
newType = await p.select({
|
|
217
|
+
message: 'New server type:',
|
|
218
|
+
options: typeOptions,
|
|
219
|
+
initialValue: currentType,
|
|
220
|
+
});
|
|
221
|
+
if (p.isCancel(newType)) {
|
|
222
|
+
p.cancel('Operation cancelled.');
|
|
223
|
+
process.exit(0);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
if (newType === currentType) {
|
|
228
|
+
p.log.info(`Already using ${c.bold(currentType)} — nothing to change.`);
|
|
229
|
+
p.outro('');
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// Cost comparison
|
|
234
|
+
const ha = envConfig.deployMode === 'compose-ha';
|
|
235
|
+
const services = projectConfig.services || {};
|
|
236
|
+
const costBreakdown = buildCostBreakdown({
|
|
237
|
+
isComposeDeploy: true,
|
|
238
|
+
ha,
|
|
239
|
+
serverType: newType,
|
|
240
|
+
region,
|
|
241
|
+
services,
|
|
242
|
+
});
|
|
243
|
+
p.note(formatCostLines(costBreakdown), 'New Estimated Cost');
|
|
244
|
+
|
|
245
|
+
// Confirmation
|
|
246
|
+
const serverLabel =
|
|
247
|
+
targetServers.length > 1 ? `${targetServers.length} servers (one at a time)` : '1 server';
|
|
248
|
+
p.log.info(
|
|
249
|
+
`This will create a new ${c.bold(newType)} server, migrate data, and destroy the old one.`,
|
|
250
|
+
);
|
|
251
|
+
p.log.info('Your application stays online during the migration (zero downtime).');
|
|
252
|
+
|
|
253
|
+
const confirmed =
|
|
254
|
+
options.yes ||
|
|
255
|
+
(await p.confirm({
|
|
256
|
+
message: `Scale ${serverLabel} from ${currentType} to ${newType}?`,
|
|
257
|
+
}));
|
|
258
|
+
if (p.isCancel(confirmed) || !confirmed) {
|
|
259
|
+
p.cancel('Operation cancelled.');
|
|
260
|
+
process.exit(0);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Prepare shared resources
|
|
264
|
+
const provider = new HetznerProvider(apiToken);
|
|
265
|
+
const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}`);
|
|
266
|
+
const sshPubKeyPath = `${sshKeyPath}.pub`;
|
|
267
|
+
const domain = envConfig.domain || null;
|
|
268
|
+
|
|
269
|
+
// Read SSH public key and register/reuse in Hetzner
|
|
270
|
+
const sshPubKey = readFileSync(sshPubKeyPath, 'utf-8').trim();
|
|
271
|
+
// Project + environment is already a unique key in Hetzner; no need for
|
|
272
|
+
// an extra `vibecarbon-` prefix. The SSH key here is created by the
|
|
273
|
+
// initial deploy under a Pulumi-named key (`${projectName}-${env}-${region}-key`)
|
|
274
|
+
// and createSSHKey dedups by public-key bytes regardless of the requested
|
|
275
|
+
// name, so the `name` we pass is mostly cosmetic — keep it aligned with
|
|
276
|
+
// the rest of the codebase.
|
|
277
|
+
const sshKeyName = `${projectName}-${environment}-key`;
|
|
278
|
+
const sshKeyId = await provider.createSSHKey(sshKeyName, sshPubKey);
|
|
279
|
+
|
|
280
|
+
// Blue-green replacement for each target server. For compose-ha we run
|
|
281
|
+
// primary + standby in parallel — every per-server step (backup, create,
|
|
282
|
+
// build, restore, recreate) is independent except the apex DNS update,
|
|
283
|
+
// which races to whichever server finishes step 9a last (same final-state
|
|
284
|
+
// behavior the prior sequential code had — the second iteration's
|
|
285
|
+
// updateDNS overwrote the first). perf data showed compose-ha scale was
|
|
286
|
+
// exactly 2× single-server scale (711s vs ~365s); parallelizing makes it
|
|
287
|
+
// ~max(primary, standby) ≈ 6m off the critical path.
|
|
288
|
+
const scaleSingleServer = async (server) => {
|
|
289
|
+
const label =
|
|
290
|
+
envConfig.deployMode === 'compose-ha' ? `${server.role} (${server.ip})` : server.ip;
|
|
291
|
+
const s = p.spinner();
|
|
292
|
+
|
|
293
|
+
// 1. Backup database on old server (--clean so restore can run without DROP DATABASE)
|
|
294
|
+
s.start(`Backing up database on ${label}...`);
|
|
295
|
+
const archiveName = await perfAsync('scale.backupOldServer', async () =>
|
|
296
|
+
backupCompose(server.ip, sshKeyPath, projectName, { clean: true }),
|
|
297
|
+
);
|
|
298
|
+
s.stop(`Database backed up: ${archiveName}`);
|
|
299
|
+
|
|
300
|
+
// 2. Create new server. user_data front-loads ufw + unattended-upgrades
|
|
301
|
+
// during boot so setupServer is a ~2s marker-file probe below.
|
|
302
|
+
const { loadCloudInitScript } = await import('./lib/deploy/compose/index.js');
|
|
303
|
+
const newServerName = `${projectName}-${environment}${server.role ? `-${server.role}` : ''}-new`;
|
|
304
|
+
s.start(`Creating new ${c.bold(newType)} server...`);
|
|
305
|
+
const { id: newServerId, server: newServer } = await perfAsync(
|
|
306
|
+
'scale.createNewServer',
|
|
307
|
+
async () =>
|
|
308
|
+
provider.createServer({
|
|
309
|
+
name: newServerName,
|
|
310
|
+
serverType: newType,
|
|
311
|
+
region: server.region || region,
|
|
312
|
+
image: 'docker-ce',
|
|
313
|
+
sshKeys: [sshKeyId],
|
|
314
|
+
firewalls: envConfig.firewallId ? [envConfig.firewallId] : [],
|
|
315
|
+
userData: loadCloudInitScript(),
|
|
316
|
+
labels: {
|
|
317
|
+
'managed-by': 'vibecarbon',
|
|
318
|
+
project: projectName,
|
|
319
|
+
environment,
|
|
320
|
+
},
|
|
321
|
+
}),
|
|
322
|
+
);
|
|
323
|
+
const newIp = HetznerProvider.getPublicIP(newServer);
|
|
324
|
+
s.stop(`New server created: ${newIp} (id: ${newServerId})`);
|
|
325
|
+
|
|
326
|
+
// Wrap remaining steps in try/catch — if anything fails after creating the
|
|
327
|
+
// new server, delete it so we don't leave orphaned infrastructure behind.
|
|
328
|
+
try {
|
|
329
|
+
// 3. Wait for SSH
|
|
330
|
+
s.start('Waiting for server to come online...');
|
|
331
|
+
await perfAsync('scale.waitForServer', () => provider.waitForServer(newServerId));
|
|
332
|
+
const sshReady = await perfAsync('scale.waitForSSH', () => waitForSSH(newIp, sshKeyPath, 40));
|
|
333
|
+
if (!sshReady) {
|
|
334
|
+
throw new Error(`SSH did not become available on new server ${newIp}`);
|
|
335
|
+
}
|
|
336
|
+
s.stop('Server online');
|
|
337
|
+
|
|
338
|
+
// 4. Provision new server
|
|
339
|
+
s.start('Configuring server...');
|
|
340
|
+
await perfAsync('scale.cloudInitReady', () => setupServer(newIp, sshKeyPath));
|
|
341
|
+
s.stop('Docker installed');
|
|
342
|
+
|
|
343
|
+
// Pull the OLD server's full /opt/<project>/.env so every variable the
|
|
344
|
+
// original deploy seeded (Supabase keys, S3 creds, SMTP, billing,
|
|
345
|
+
// Grafana, n8n encryption key, etc.) propagates to the new server.
|
|
346
|
+
//
|
|
347
|
+
// Why pull the whole file vs. re-deriving from project state:
|
|
348
|
+
// - The original deploy merges values from many sources — project
|
|
349
|
+
// `.env` + `~/.vibecarbon/credentials.json` (S3 secret key) +
|
|
350
|
+
// orchestrator-derived overrides (DOMAIN, ACME_EMAIL, SITE_URL,
|
|
351
|
+
// SUPABASE_URL, S3_*, S3_BACKUP_BUCKET, PROJECT_NAME, APP_IMAGE).
|
|
352
|
+
// Reconstructing all of those at scale time would mean re-prompting
|
|
353
|
+
// for credentials and re-running the same envOverrides logic.
|
|
354
|
+
// - The old server's `.env` already contains the resolved superset.
|
|
355
|
+
// Replaying it onto the new server is the smallest patch that
|
|
356
|
+
// preserves every value, including ones the project's local
|
|
357
|
+
// `.env`/`.env.local` doesn't even know about.
|
|
358
|
+
//
|
|
359
|
+
// The on-disk form is `KEY='POSIX-single-quoted-value'` (per
|
|
360
|
+
// escapeDotenv); parseDotenv decodes the wrapping quotes so the values
|
|
361
|
+
// we pass to renderBundle as envOverrides are raw strings ready to be
|
|
362
|
+
// re-escaped on write. We then pin APP_IMAGE explicitly because that's
|
|
363
|
+
// the one value scale needs to keep stable across the migration; all
|
|
364
|
+
// other keys flow through transparently.
|
|
365
|
+
let oldEnv = {};
|
|
366
|
+
try {
|
|
367
|
+
const envText = sshRun(server.ip, sshKeyPath, ['cat', `/opt/${projectName}/.env`]);
|
|
368
|
+
oldEnv = parseDotenv(envText);
|
|
369
|
+
} catch {
|
|
370
|
+
// Old server may be unreachable; renderBundle will fall back to the
|
|
371
|
+
// project's local `.env`. That path is degraded (missing S3 secrets)
|
|
372
|
+
// but at least surfaces a meaningful error during reconcile rather
|
|
373
|
+
// than crashing scale silently.
|
|
374
|
+
}
|
|
375
|
+
const oldAppImage = oldEnv.APP_IMAGE || '';
|
|
376
|
+
|
|
377
|
+
s.start('Copying project files...');
|
|
378
|
+
await perfAsync('scale.setupFiles', () =>
|
|
379
|
+
setupServerFiles(newIp, sshKeyPath, projectName, {
|
|
380
|
+
domain,
|
|
381
|
+
image: oldAppImage || `${projectName}:latest`,
|
|
382
|
+
observability: services.observability,
|
|
383
|
+
n8n: services.n8n,
|
|
384
|
+
metabase: services.metabase,
|
|
385
|
+
redis: services.redis,
|
|
386
|
+
// Replay every env var the old server had. renderBundle's existing
|
|
387
|
+
// envOverrides path overlays these onto the project-local `.env`
|
|
388
|
+
// baseline, then re-applies the deploy-time overrides (DOMAIN,
|
|
389
|
+
// SITE_URL, APP_IMAGE, etc.) on top — so the new bundle is a
|
|
390
|
+
// superset of both sources.
|
|
391
|
+
envOverrides: oldEnv,
|
|
392
|
+
}),
|
|
393
|
+
);
|
|
394
|
+
s.stop('Project files copied');
|
|
395
|
+
|
|
396
|
+
// 4b. Registry auth (optional — avoids rate limits on shared Hetzner IPs).
|
|
397
|
+
// Scale pulls the same ghcr.io/<owner>/<repo>:<tag> the original
|
|
398
|
+
// deploy pushed, so ghcr credentials are required alongside Docker Hub.
|
|
399
|
+
const creds = loadCredentials();
|
|
400
|
+
const dockerHubCreds = creds.dockerHub || null;
|
|
401
|
+
if (dockerHubCreds) {
|
|
402
|
+
s.start('Authenticating with Docker Hub...');
|
|
403
|
+
await perfAsync('scale.dockerLogin', () =>
|
|
404
|
+
dockerLoginOnServer(newIp, sshKeyPath, dockerHubCreds),
|
|
405
|
+
);
|
|
406
|
+
s.stop('Docker Hub authenticated');
|
|
407
|
+
}
|
|
408
|
+
try {
|
|
409
|
+
const ghcrCreds = await getGHCRCredentials();
|
|
410
|
+
if (ghcrCreds?.username && ghcrCreds?.token) {
|
|
411
|
+
s.start('Authenticating with ghcr.io...');
|
|
412
|
+
await perfAsync('scale.ghcrLogin', () =>
|
|
413
|
+
dockerLoginOnServer(newIp, sshKeyPath, {
|
|
414
|
+
username: ghcrCreds.username,
|
|
415
|
+
token: ghcrCreds.token,
|
|
416
|
+
registry: 'ghcr.io',
|
|
417
|
+
}),
|
|
418
|
+
);
|
|
419
|
+
s.stop('ghcr.io authenticated');
|
|
420
|
+
}
|
|
421
|
+
} catch {
|
|
422
|
+
/* non-fatal: image may be public */
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
// 4c. Direct-mode image: build on the new server instead of sideloading.
|
|
426
|
+
//
|
|
427
|
+
// Original implementation piped `ssh old docker save | ssh new docker load`
|
|
428
|
+
// from the local shell — every image byte transited the local machine
|
|
429
|
+
// twice across the WAN, causing 16+ min hangs / timeouts on a ~500MB
|
|
430
|
+
// image during matrix runs. compose's `prod.yml` resets
|
|
431
|
+
// `build: !reset null`, so `docker compose up --build` can NOT rebuild
|
|
432
|
+
// it during the reconcile step — the image must already exist in the
|
|
433
|
+
// new server's daemon by the time compose pulls.
|
|
434
|
+
//
|
|
435
|
+
// buildRemote uses DOCKER_HOST=ssh://new and runs the build directly on
|
|
436
|
+
// the new server. BuildKit only ships the build context (~tens of MB
|
|
437
|
+
// of source), not the 500MB built image, so the data path is local →
|
|
438
|
+
// new (one short hop) instead of old → local → new (two long hops).
|
|
439
|
+
// Reuses the same code that does direct-mode deploy builds.
|
|
440
|
+
if (oldAppImage && isLocalOnlyImageTag(oldAppImage)) {
|
|
441
|
+
s.start(`Building image ${oldAppImage} on new server...`);
|
|
442
|
+
await perfAsync('scale.sideloadImage', () =>
|
|
443
|
+
buildRemote(newIp, sshKeyPath, oldAppImage, process.cwd()),
|
|
444
|
+
);
|
|
445
|
+
s.stop('Image built on new server');
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// 5. Pull app + base images from ghcr.io / Docker Hub
|
|
449
|
+
// No local build — image was built by CI (see Lever 1).
|
|
450
|
+
s.start('Pulling images on server...');
|
|
451
|
+
await perfAsync('scale.pullImages', () =>
|
|
452
|
+
pullComposeImages(newIp, sshKeyPath, projectName, services),
|
|
453
|
+
);
|
|
454
|
+
s.stop('Docker images ready');
|
|
455
|
+
|
|
456
|
+
// 6. Start compose stack on new server
|
|
457
|
+
s.start('Starting services on new server...');
|
|
458
|
+
await perfAsync('scale.composeUp', () =>
|
|
459
|
+
startComposeStack(newIp, sshKeyPath, projectName, services),
|
|
460
|
+
);
|
|
461
|
+
s.stop('Services started');
|
|
462
|
+
|
|
463
|
+
// 7. Wait for DB to be ready (backup uses --clean so no DROP DATABASE needed)
|
|
464
|
+
s.start('Waiting for database to be ready...');
|
|
465
|
+
const remoteDir = `/opt/${projectName}`;
|
|
466
|
+
// Shell loop + `$(...)` needs a real shell; projectName is basename-validated.
|
|
467
|
+
sshRunScript(
|
|
468
|
+
newIp,
|
|
469
|
+
sshKeyPath,
|
|
470
|
+
`cd ${remoteDir} && for i in $(seq 1 30); do docker compose exec -T db pg_isready -U postgres >/dev/null 2>&1 && break || sleep 2; done`,
|
|
471
|
+
{ timeout: 120_000 },
|
|
472
|
+
);
|
|
473
|
+
s.stop('Database ready for restore');
|
|
474
|
+
|
|
475
|
+
// 8. Transfer backup from old server to new server
|
|
476
|
+
s.start('Transferring backup to new server...');
|
|
477
|
+
const localTmpBackup = join(os.tmpdir(), archiveName);
|
|
478
|
+
try {
|
|
479
|
+
scpDownload(server.ip, sshKeyPath, `${remoteDir}/backups/${archiveName}`, localTmpBackup);
|
|
480
|
+
sshRun(newIp, sshKeyPath, ['mkdir', '-p', `${remoteDir}/backups`]);
|
|
481
|
+
scpUpload(newIp, sshKeyPath, localTmpBackup, `${remoteDir}/backups/${archiveName}`);
|
|
482
|
+
} finally {
|
|
483
|
+
try {
|
|
484
|
+
unlinkSync(localTmpBackup);
|
|
485
|
+
} catch {
|
|
486
|
+
// ignore cleanup errors
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
s.stop('Backup transferred');
|
|
490
|
+
|
|
491
|
+
// 9. Restore database on new server
|
|
492
|
+
s.start('Restoring database...');
|
|
493
|
+
await perfAsync('scale.restoreDb', async () =>
|
|
494
|
+
restoreCompose(newIp, sshKeyPath, projectName, archiveName),
|
|
495
|
+
);
|
|
496
|
+
s.stop('Database restored');
|
|
497
|
+
|
|
498
|
+
// 9a. Update DNS BEFORE recreating services so Traefik can get ACME certs.
|
|
499
|
+
// Traefik attempts HTTP-01 challenges immediately on startup; if DNS still
|
|
500
|
+
// points to the old server, all challenges fail and the next retry is 30+ min.
|
|
501
|
+
if (domain) {
|
|
502
|
+
s.start('Updating DNS to new server...');
|
|
503
|
+
await perfAsync('scale.updateDNS', () => updateDNS(envConfig, apiToken, domain, newIp));
|
|
504
|
+
s.stop('DNS updated');
|
|
505
|
+
|
|
506
|
+
// Poll until Cloudflare/Google DNS returns the new IP (max 120s).
|
|
507
|
+
// Fixed waits are unreliable — propagation speed varies per resolver.
|
|
508
|
+
s.start('Waiting for DNS propagation...');
|
|
509
|
+
const dnsOk = await perfAsync('scale.waitForDNS', () =>
|
|
510
|
+
waitForDNSPropagation(domain, newIp, 120_000),
|
|
511
|
+
);
|
|
512
|
+
s.stop(dnsOk ? 'DNS propagated' : 'DNS propagation timed out (proceeding anyway)');
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
// 9b. Reset ACME state so Traefik retries cert challenges with the correct DNS.
|
|
516
|
+
// startComposeStack (step 6) ran before DNS was updated — all challenges failed
|
|
517
|
+
// and are cached as errors in the letsencrypt_data volume.
|
|
518
|
+
// Use docker compose exec (Traefik is already running) instead of pulling alpine.
|
|
519
|
+
// Uses an SCP'd script so quoting/redirection stay intact without local shell.
|
|
520
|
+
sshRunScript(
|
|
521
|
+
newIp,
|
|
522
|
+
sshKeyPath,
|
|
523
|
+
`cd ${remoteDir} && docker compose exec -T traefik sh -c 'echo "{}" > /letsencrypt/acme.json && chmod 600 /letsencrypt/acme.json'`,
|
|
524
|
+
{ timeout: 30_000 },
|
|
525
|
+
);
|
|
526
|
+
|
|
527
|
+
// 9c. Recreate all services after restore so they reconnect to the restored DB.
|
|
528
|
+
// Include all feature compose file flags so feature containers (redis, n8n,
|
|
529
|
+
// observability) are recreated too. Using --force-recreate (not down+up) avoids
|
|
530
|
+
// a Docker network removal error when feature containers on separate compose files
|
|
531
|
+
// are still holding the network open. depends_on ordering is still respected.
|
|
532
|
+
const composeFlags = [
|
|
533
|
+
'-f docker-compose.yml',
|
|
534
|
+
'-f docker-compose.prod.yml',
|
|
535
|
+
...(services.observability
|
|
536
|
+
? ['-f docker-compose.observability.yml', '-f docker-compose.observability.prod.yml']
|
|
537
|
+
: []),
|
|
538
|
+
...(services.n8n ? ['-f docker-compose.n8n.yml', '-f docker-compose.n8n.prod.yml'] : []),
|
|
539
|
+
...(services.metabase
|
|
540
|
+
? ['-f docker-compose.metabase.yml', '-f docker-compose.metabase.prod.yml']
|
|
541
|
+
: []),
|
|
542
|
+
...(services.redis ? ['-f docker-compose.redis.yml'] : []),
|
|
543
|
+
].join(' ');
|
|
544
|
+
s.start('Recreating all services after restore...');
|
|
545
|
+
sshRunScript(
|
|
546
|
+
newIp,
|
|
547
|
+
sshKeyPath,
|
|
548
|
+
`cd ${remoteDir} && docker compose ${composeFlags} up -d --force-recreate 2>&1`,
|
|
549
|
+
{ timeout: 600_000 },
|
|
550
|
+
);
|
|
551
|
+
s.stop('All services running');
|
|
552
|
+
|
|
553
|
+
// 11. Re-setup backup cron if configured. envConfig only persists
|
|
554
|
+
// bucket/region/endpoint under `s3` — accessKey/secretKey live in
|
|
555
|
+
// ~/.vibecarbon/credentials.json (loaded into `creds` at step 4b).
|
|
556
|
+
// Without merging them in, setupComposeBackupCron would write a
|
|
557
|
+
// /etc/vibecarbon/backup.env with empty AWS_ACCESS_KEY_ID /
|
|
558
|
+
// AWS_SECRET_ACCESS_KEY and every cron-triggered backup would 403
|
|
559
|
+
// against S3.
|
|
560
|
+
if (envConfig.backup) {
|
|
561
|
+
const backupS3Config =
|
|
562
|
+
envConfig.s3 && creds.s3
|
|
563
|
+
? {
|
|
564
|
+
bucket: envConfig.s3.bucket,
|
|
565
|
+
accessKey: creds.s3.accessKey,
|
|
566
|
+
secretKey: creds.s3.secretKey,
|
|
567
|
+
endpoint: envConfig.s3.endpoint,
|
|
568
|
+
region: envConfig.s3.region,
|
|
569
|
+
}
|
|
570
|
+
: null;
|
|
571
|
+
setupComposeBackupCron(newIp, sshKeyPath, projectName, backupS3Config, envConfig.backup);
|
|
572
|
+
p.log.info('Backup cron restored on new server');
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// 12. Destroy old server
|
|
576
|
+
s.start(`Destroying old server ${server.ip}...`);
|
|
577
|
+
await perfAsync('scale.destroyOldServer', () => provider.deleteServer(server.id));
|
|
578
|
+
s.stop(`Old server destroyed: ${server.ip}`);
|
|
579
|
+
|
|
580
|
+
// Rename new server to permanent name
|
|
581
|
+
const permanentName = `${projectName}-${environment}${server.role ? `-${server.role}` : ''}`;
|
|
582
|
+
try {
|
|
583
|
+
await provider.apiRequest(`/servers/${newServerId}`, {
|
|
584
|
+
method: 'PUT',
|
|
585
|
+
body: JSON.stringify({ name: permanentName }),
|
|
586
|
+
});
|
|
587
|
+
} catch {
|
|
588
|
+
// Non-critical — server works fine with the temporary name
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
// Update server entry in the config. We spread the original `server`
|
|
592
|
+
// entry first so any field we don't explicitly touch (e.g.
|
|
593
|
+
// `supabaseIp` on shapes that have it) carries over verbatim — then
|
|
594
|
+
// override the fields that actually changed during the migration.
|
|
595
|
+
// We deliberately keep `server.name` (the role label —
|
|
596
|
+
// 'master' / 'primary' / 'standby') because failover.js, diagnose.js,
|
|
597
|
+
// and destroy.js all filter on it (e.g. `s.name === 'master'`). The
|
|
598
|
+
// Hetzner-API name goes into `hetznerName` so destroy's name-fallback
|
|
599
|
+
// can locate the VPS.
|
|
600
|
+
server._replacement = {
|
|
601
|
+
...server,
|
|
602
|
+
id: newServerId,
|
|
603
|
+
hetznerName: permanentName,
|
|
604
|
+
ip: newIp,
|
|
605
|
+
region: server.region || region,
|
|
606
|
+
serverType: newType,
|
|
607
|
+
};
|
|
608
|
+
} catch (scaleErr) {
|
|
609
|
+
// Clean up the new server so it doesn't become orphaned infrastructure.
|
|
610
|
+
p.log.error(`Scale failed: ${scaleErr.message}`);
|
|
611
|
+
p.log.info(`Cleaning up new server ${newIp} (id: ${newServerId})...`);
|
|
612
|
+
try {
|
|
613
|
+
await provider.deleteServer(newServerId);
|
|
614
|
+
p.log.info('New server deleted. Old server is still running.');
|
|
615
|
+
} catch (cleanupErr) {
|
|
616
|
+
p.log.error(
|
|
617
|
+
`Failed to delete new server ${newServerId}: ${cleanupErr.message}. Delete it manually.`,
|
|
618
|
+
);
|
|
619
|
+
}
|
|
620
|
+
throw scaleErr;
|
|
621
|
+
}
|
|
622
|
+
};
|
|
623
|
+
|
|
624
|
+
if (ha && targetServers.length > 1) {
|
|
625
|
+
// Wrap parallel scale in a perfTimer so the wall-clock win is visible
|
|
626
|
+
// in perf_substep alongside the per-server scale.* timings.
|
|
627
|
+
await perfAsync('scale.ha.fan', () => Promise.all(targetServers.map(scaleSingleServer)));
|
|
628
|
+
} else {
|
|
629
|
+
for (const server of targetServers) {
|
|
630
|
+
await scaleSingleServer(server);
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
// Update .vibecarbon.json with new server details
|
|
635
|
+
const updatedServers = servers.map((server) => {
|
|
636
|
+
if (server._replacement) {
|
|
637
|
+
const replacement = { ...server._replacement };
|
|
638
|
+
delete server._replacement;
|
|
639
|
+
return replacement;
|
|
640
|
+
}
|
|
641
|
+
return server;
|
|
642
|
+
});
|
|
643
|
+
|
|
644
|
+
const allScaled = servers.every((s) => !s.id || targetServers.some((t) => t.id === s.id));
|
|
645
|
+
const updatedConfig = {
|
|
646
|
+
...projectConfig,
|
|
647
|
+
environments: {
|
|
648
|
+
...projectConfig.environments,
|
|
649
|
+
[environment]: {
|
|
650
|
+
...envConfig,
|
|
651
|
+
...(allScaled && { serverType: newType }),
|
|
652
|
+
servers: updatedServers,
|
|
653
|
+
},
|
|
654
|
+
},
|
|
655
|
+
};
|
|
656
|
+
saveProjectConfig(updatedConfig);
|
|
657
|
+
|
|
658
|
+
p.outro(
|
|
659
|
+
`${c.success('Scale complete!')} ${c.bold(currentType)} → ${c.bold(newType)} (zero downtime)`,
|
|
660
|
+
);
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
/**
|
|
664
|
+
* Update DNS A records to point to the new server IP.
|
|
665
|
+
*/
|
|
666
|
+
async function updateDNS(envConfig, apiToken, domain, newIp) {
|
|
667
|
+
const dnsProvider = envConfig.dnsProvider || envConfig.dns?.provider;
|
|
668
|
+
|
|
669
|
+
if (dnsProvider === 'cloudflare') {
|
|
670
|
+
const cfToken = process.env.CLOUDFLARE_API_TOKEN;
|
|
671
|
+
const zoneId = envConfig.dns?.cloudflareZoneId;
|
|
672
|
+
if (cfToken && zoneId) {
|
|
673
|
+
const { createDNSRecord } = await import('./lib/cloudflare.js');
|
|
674
|
+
// proxied:false matches deploy-time setup (Cloudflare LBs removed
|
|
675
|
+
// 2026-03-26 — orange cloud no longer needed).
|
|
676
|
+
await createDNSRecord(cfToken, zoneId, {
|
|
677
|
+
type: 'A',
|
|
678
|
+
name: domain,
|
|
679
|
+
content: newIp,
|
|
680
|
+
proxied: false,
|
|
681
|
+
});
|
|
682
|
+
await createDNSRecord(cfToken, zoneId, {
|
|
683
|
+
type: 'A',
|
|
684
|
+
name: `*.${domain}`,
|
|
685
|
+
content: newIp,
|
|
686
|
+
proxied: false,
|
|
687
|
+
});
|
|
688
|
+
p.log.info(`DNS updated via Cloudflare: ${domain} → ${newIp}`);
|
|
689
|
+
} else {
|
|
690
|
+
p.log.warn(`Update DNS manually: ${domain} → ${newIp}`);
|
|
691
|
+
}
|
|
692
|
+
} else if (dnsProvider === 'hetzner') {
|
|
693
|
+
const zoneId = envConfig.dns?.hetznerZoneId;
|
|
694
|
+
if (zoneId) {
|
|
695
|
+
const { createDNSRecord, getZone } = await import('./lib/hetzner-dns.js');
|
|
696
|
+
const zone = await getZone(apiToken, zoneId);
|
|
697
|
+
await createDNSRecord(apiToken, zoneId, zone.name, {
|
|
698
|
+
type: 'A',
|
|
699
|
+
name: domain,
|
|
700
|
+
value: newIp,
|
|
701
|
+
ttl: 60,
|
|
702
|
+
});
|
|
703
|
+
await createDNSRecord(apiToken, zoneId, zone.name, {
|
|
704
|
+
type: 'A',
|
|
705
|
+
name: `*.${domain}`,
|
|
706
|
+
value: newIp,
|
|
707
|
+
ttl: 60,
|
|
708
|
+
});
|
|
709
|
+
p.log.info(`DNS updated via Hetzner DNS: ${domain} → ${newIp}`);
|
|
710
|
+
} else {
|
|
711
|
+
p.log.warn(`Update DNS manually: ${domain} → ${newIp}`);
|
|
712
|
+
}
|
|
713
|
+
} else if (domain) {
|
|
714
|
+
p.log.warn(`Update your DNS A record: ${domain} → ${newIp}`);
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
// waitForDNSPropagation now lives in src/lib/dns-propagation.js (imported
|
|
719
|
+
// at the top of this file). The compose deploy path needs the same
|
|
720
|
+
// utility for the cold-deploy ACME race fix; extracting it lets both
|
|
721
|
+
// paths share the implementation.
|
|
722
|
+
|
|
723
|
+
// ============================================================================
|
|
724
|
+
// MAIN
|
|
725
|
+
// ============================================================================
|
|
726
|
+
|
|
727
|
+
export async function run(args) {
|
|
728
|
+
const { values, positional, errors } = parseFlags(args, SPEC);
|
|
729
|
+
|
|
730
|
+
if (errors.length > 0) {
|
|
731
|
+
for (const e of errors) {
|
|
732
|
+
process.stderr.write(`${c.error('✗')} ${e}\n`);
|
|
733
|
+
}
|
|
734
|
+
process.stderr.write(`Run ${c.info('vibecarbon scale -h')} for usage.\n`);
|
|
735
|
+
process.exit(1);
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
if (values.h) {
|
|
739
|
+
process.stdout.write(renderHelp(SPEC));
|
|
740
|
+
return;
|
|
741
|
+
}
|
|
742
|
+
if (values.v) {
|
|
743
|
+
console.log(`vibecarbon v${VERSION}`);
|
|
744
|
+
return;
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
// Project guard runs before banner so an accidental `vibecarbon
|
|
748
|
+
// scale` from a parent directory emits the canonical message.
|
|
749
|
+
const projectConfig = assertInProjectDir();
|
|
750
|
+
|
|
751
|
+
requireLicense('scale');
|
|
752
|
+
printBanner();
|
|
753
|
+
p.intro(`${c.bold('vibecarbon scale')} ${c.dim(`v${VERSION}`)}`);
|
|
754
|
+
|
|
755
|
+
const envs = projectConfig.environments || {};
|
|
756
|
+
const deployedEnvs = Object.entries(envs)
|
|
757
|
+
.filter(([, cfg]) => cfg.status === 'deployed')
|
|
758
|
+
.map(([name]) => name);
|
|
759
|
+
|
|
760
|
+
if (deployedEnvs.length === 0) {
|
|
761
|
+
p.log.error('No deployed environments found. Run vibecarbon deploy first.');
|
|
762
|
+
process.exit(1);
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
// Build a legacy-shaped `parsed` struct for the orchestration code below.
|
|
766
|
+
// Worker-bound flags are gone in the new flag set — the prompt covers
|
|
767
|
+
// them — so they're nulls here.
|
|
768
|
+
const envSeed =
|
|
769
|
+
/** @type {string|undefined} */ (positional.env) ||
|
|
770
|
+
/** @type {string|null} */ (values.env) ||
|
|
771
|
+
null;
|
|
772
|
+
const parsed = {
|
|
773
|
+
env: envSeed,
|
|
774
|
+
type: /** @type {string|null} */ (values.type),
|
|
775
|
+
yes: !!values.y,
|
|
776
|
+
minWorkers: null,
|
|
777
|
+
maxWorkers: null,
|
|
778
|
+
};
|
|
779
|
+
|
|
780
|
+
// TTY guard: env prompt fires if multi-deployed and no seed.
|
|
781
|
+
requireTTYOrFlags({
|
|
782
|
+
requirements: [
|
|
783
|
+
{
|
|
784
|
+
flag: 'env',
|
|
785
|
+
description: 'name an environment to scale',
|
|
786
|
+
satisfied: !!envSeed || deployedEnvs.length <= 1,
|
|
787
|
+
},
|
|
788
|
+
],
|
|
789
|
+
});
|
|
790
|
+
|
|
791
|
+
// Select environment (preserves the old behavior of filtering to
|
|
792
|
+
// deployed envs only — different from selectEnvironment which lists
|
|
793
|
+
// all envs in projectConfig regardless of status).
|
|
794
|
+
let environment = parsed.env;
|
|
795
|
+
if (!environment) {
|
|
796
|
+
if (deployedEnvs.length === 1) {
|
|
797
|
+
environment = deployedEnvs[0];
|
|
798
|
+
p.log.info(`Using environment: ${c.bold(environment)}`);
|
|
799
|
+
} else {
|
|
800
|
+
environment = await p.select({
|
|
801
|
+
message: 'Which environment to scale?',
|
|
802
|
+
options: deployedEnvs.map((e) => ({ value: e, label: e })),
|
|
803
|
+
});
|
|
804
|
+
if (p.isCancel(environment)) {
|
|
805
|
+
p.cancel('Operation cancelled.');
|
|
806
|
+
process.exit(0);
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
parsed.env = environment;
|
|
811
|
+
|
|
812
|
+
const envConfig = envs[environment];
|
|
813
|
+
if (!envConfig) {
|
|
814
|
+
p.log.error(`Environment "${environment}" not found in .vibecarbon.json`);
|
|
815
|
+
process.exit(1);
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
// Compose deployments: vertical scaling via Hetzner API
|
|
819
|
+
if (envConfig.deployMode === 'compose' || envConfig.deployMode === 'compose-ha') {
|
|
820
|
+
await scaleCompose(environment, envConfig, projectConfig, {
|
|
821
|
+
type: parsed.type,
|
|
822
|
+
yes: parsed.yes,
|
|
823
|
+
});
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
const region = envConfig.region || 'fsn1';
|
|
828
|
+
|
|
829
|
+
// Pre-fetch live server types (non-blocking, uses saved credentials if available)
|
|
830
|
+
{
|
|
831
|
+
const token = process.env.HETZNER_API_TOKEN || loadCredentials()?.hetzner?.apiToken;
|
|
832
|
+
if (token) await HetznerProvider.fetchServerTypes(token);
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
// Per-role types are optional in the deploy. The orchestrator saves only
|
|
836
|
+
// `serverType` (the unified default) to .vibecarbon.json — the master/
|
|
837
|
+
// supabase/worker types only appear here after a previous `scale` set
|
|
838
|
+
// them. Fall back to envConfig.serverType so we don't send the em-dash
|
|
839
|
+
// placeholder to Pulumi: doing so triggered ~serverType against the
|
|
840
|
+
// already-provisioned `cx23` master/supabase servers and Hetzner replied
|
|
841
|
+
// 'server type — not found'.
|
|
842
|
+
const fallbackType = envConfig.serverType || 'cx23';
|
|
843
|
+
const currentMasterType = envConfig.masterServerType || fallbackType;
|
|
844
|
+
const currentSupabaseType = envConfig.supabaseServerType || fallbackType;
|
|
845
|
+
const currentWorkerType = envConfig.workerServerType || fallbackType;
|
|
846
|
+
|
|
847
|
+
// 3. Display current config
|
|
848
|
+
const pad = 18;
|
|
849
|
+
const configLines = [`${'Environment'.padEnd(pad)} ${c.bold(environment)}`];
|
|
850
|
+
if (envConfig.ha?.enabled && envConfig.secondaryRegion) {
|
|
851
|
+
configLines.push(`${'Primary region'.padEnd(pad)} ${c.bold(region)}`);
|
|
852
|
+
configLines.push(
|
|
853
|
+
`${'Standby region'.padEnd(pad)} ${c.bold(envConfig.secondaryRegion)} ${c.dim('(will be scaled identically)')}`,
|
|
854
|
+
);
|
|
855
|
+
} else {
|
|
856
|
+
configLines.push(`${'Region'.padEnd(pad)} ${c.bold(region)}`);
|
|
857
|
+
}
|
|
858
|
+
configLines.push(
|
|
859
|
+
`${'Master type'.padEnd(pad)} ${c.bold(currentMasterType)}`,
|
|
860
|
+
`${'Supabase type'.padEnd(pad)} ${c.bold(currentSupabaseType)}`,
|
|
861
|
+
`${'Worker type'.padEnd(pad)} ${c.bold(currentWorkerType)}`,
|
|
862
|
+
);
|
|
863
|
+
p.note(configLines.join('\n'), 'Current Configuration');
|
|
864
|
+
|
|
865
|
+
// 4. Select what to change
|
|
866
|
+
// 5. Collect new values
|
|
867
|
+
const newValues = {};
|
|
868
|
+
const regionTypes = HetznerProvider.getServerTypesForRegion(region);
|
|
869
|
+
const serverTypeOptions = buildSimpleTypeOptions(regionTypes, { filterSharedCpu: false });
|
|
870
|
+
|
|
871
|
+
let changes;
|
|
872
|
+
if (parsed.yes && parsed.type) {
|
|
873
|
+
// Non-interactive: `--type X` means "scale every node role to X" for
|
|
874
|
+
// parity with compose's `--type X` (which resizes the one VPS).
|
|
875
|
+
// Without this, a k8s `--type cx33` only changed worker pools and
|
|
876
|
+
// left master + supabase on the previous type — a UX inconsistency
|
|
877
|
+
// the e2e verify-scale assertion caught (matrix run #10 k8s
|
|
878
|
+
// scale: only the worker IP showed cx33 post-scale, supabase stayed
|
|
879
|
+
// cx23).
|
|
880
|
+
//
|
|
881
|
+
// Hetzner server_type changes are in-place resizes (~2 min reboot
|
|
882
|
+
// per node), so the master-replace guard below does NOT trip on
|
|
883
|
+
// legitimate type bumps — that guard fires only on userData drift
|
|
884
|
+
// (k3sToken, labels, cloud-init template), which the probe-and-replay
|
|
885
|
+
// earlier in this function already prevents. Master upgrades cause
|
|
886
|
+
// ~2 min of control-plane downtime; running under -y means the
|
|
887
|
+
// operator accepted the cost.
|
|
888
|
+
//
|
|
889
|
+
// Per-role overrides remain available through the interactive
|
|
890
|
+
// multiselect when finer control is needed.
|
|
891
|
+
changes = ['masterType', 'supabaseType', 'workerType'];
|
|
892
|
+
newValues.masterType = parsed.type;
|
|
893
|
+
newValues.supabaseType = parsed.type;
|
|
894
|
+
newValues.workerType = parsed.type;
|
|
895
|
+
// Allow --type and bounds to be combined in one invocation. CA
|
|
896
|
+
// re-patch happens regardless once changes include 'workerBounds'.
|
|
897
|
+
if (parsed.minWorkers != null || parsed.maxWorkers != null) {
|
|
898
|
+
changes.push('workerBounds');
|
|
899
|
+
if (parsed.minWorkers != null) newValues.minWorkers = parsed.minWorkers;
|
|
900
|
+
if (parsed.maxWorkers != null) newValues.maxWorkers = parsed.maxWorkers;
|
|
901
|
+
}
|
|
902
|
+
p.log.info(
|
|
903
|
+
`Scaling all node roles (master, supabase, worker) to ${c.bold(parsed.type)} ${c.dim('(in-place resize, ~2 min downtime per node)')}`,
|
|
904
|
+
);
|
|
905
|
+
} else if (parsed.yes && (parsed.minWorkers != null || parsed.maxWorkers != null)) {
|
|
906
|
+
// Non-interactive bounds-only path: --min-workers and/or --max-workers
|
|
907
|
+
// without --type. We do not run Pulumi here unless minWorkers rose above
|
|
908
|
+
// the previously persisted floor (which would require new static workers
|
|
909
|
+
// to be provisioned).
|
|
910
|
+
//
|
|
911
|
+
// TODO(phase 8 follow-up): bounds-driven Pulumi changes — when
|
|
912
|
+
// minWorkers rises above the persisted floor, run Pulumi to add static
|
|
913
|
+
// workers; when minWorkers drops, leave Pulumi alone (don't reap static
|
|
914
|
+
// workers on bounds relax). For now we patch CA only; bumping min above
|
|
915
|
+
// the persisted floor will not provision new static workers until the
|
|
916
|
+
// next deploy.
|
|
917
|
+
changes = ['workerBounds'];
|
|
918
|
+
if (parsed.minWorkers != null) newValues.minWorkers = parsed.minWorkers;
|
|
919
|
+
if (parsed.maxWorkers != null) newValues.maxWorkers = parsed.maxWorkers;
|
|
920
|
+
p.log.info(
|
|
921
|
+
`Updating cluster-autoscaler bounds: ${c.bold(
|
|
922
|
+
`min=${newValues.minWorkers ?? envConfig.minWorkers ?? DEFAULT_WORKER_MIN}`,
|
|
923
|
+
)} ${c.bold(`max=${newValues.maxWorkers ?? envConfig.maxWorkers ?? DEFAULT_WORKER_MAX}`)}`,
|
|
924
|
+
);
|
|
925
|
+
} else {
|
|
926
|
+
changes = await p.multiselect({
|
|
927
|
+
message: 'What would you like to change?',
|
|
928
|
+
options: [
|
|
929
|
+
{ value: 'workerType', label: 'Worker server type (replace)' },
|
|
930
|
+
{
|
|
931
|
+
value: 'masterType',
|
|
932
|
+
label: 'Master server type (~2 min control-plane downtime)',
|
|
933
|
+
},
|
|
934
|
+
{
|
|
935
|
+
value: 'supabaseType',
|
|
936
|
+
label: 'Supabase server type (database restart — back up first)',
|
|
937
|
+
},
|
|
938
|
+
],
|
|
939
|
+
required: true,
|
|
940
|
+
});
|
|
941
|
+
if (p.isCancel(changes)) {
|
|
942
|
+
p.cancel('Operation cancelled.');
|
|
943
|
+
process.exit(0);
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
for (const change of changes) {
|
|
947
|
+
if (change === 'workerType') {
|
|
948
|
+
const val = await p.select({
|
|
949
|
+
message: 'New worker server type:',
|
|
950
|
+
options: serverTypeOptions,
|
|
951
|
+
initialValue: serverTypeOptions.find((t) => t.value === currentWorkerType)?.value,
|
|
952
|
+
});
|
|
953
|
+
if (p.isCancel(val)) {
|
|
954
|
+
p.cancel('Operation cancelled.');
|
|
955
|
+
process.exit(0);
|
|
956
|
+
}
|
|
957
|
+
newValues.workerType = val;
|
|
958
|
+
} else if (change === 'masterType') {
|
|
959
|
+
p.log.warn('Changing master type will cause ~2 minutes of control-plane downtime.');
|
|
960
|
+
const val = await p.select({
|
|
961
|
+
message: 'New master server type:',
|
|
962
|
+
options: serverTypeOptions,
|
|
963
|
+
initialValue: serverTypeOptions.find((t) => t.value === currentMasterType)?.value,
|
|
964
|
+
});
|
|
965
|
+
if (p.isCancel(val)) {
|
|
966
|
+
p.cancel('Operation cancelled.');
|
|
967
|
+
process.exit(0);
|
|
968
|
+
}
|
|
969
|
+
newValues.masterType = val;
|
|
970
|
+
} else if (change === 'supabaseType') {
|
|
971
|
+
p.log.warn('Changing Supabase type will restart the database. Back up first!');
|
|
972
|
+
const val = await p.select({
|
|
973
|
+
message: 'New Supabase server type:',
|
|
974
|
+
options: serverTypeOptions,
|
|
975
|
+
initialValue: serverTypeOptions.find((t) => t.value === currentSupabaseType)?.value,
|
|
976
|
+
});
|
|
977
|
+
if (p.isCancel(val)) {
|
|
978
|
+
p.cancel('Operation cancelled.');
|
|
979
|
+
process.exit(0);
|
|
980
|
+
}
|
|
981
|
+
newValues.supabaseType = val;
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
// Server type changes apply via Pulumi up → Hetzner in-place resize per
|
|
987
|
+
// node (~2 min reboot each).
|
|
988
|
+
const infraChanges = changes.filter(
|
|
989
|
+
(c) => c === 'workerType' || c === 'masterType' || c === 'supabaseType',
|
|
990
|
+
);
|
|
991
|
+
|
|
992
|
+
// Cost breakdown with new values applied
|
|
993
|
+
const services = projectConfig.services || {};
|
|
994
|
+
const isHA = envConfig.ha?.enabled && envConfig.secondaryRegion;
|
|
995
|
+
const costBreakdown = buildCostBreakdown({
|
|
996
|
+
isComposeDeploy: false,
|
|
997
|
+
ha: !!isHA,
|
|
998
|
+
masterServerType: newValues.masterType || currentMasterType,
|
|
999
|
+
supabaseServerType: newValues.supabaseType || currentSupabaseType,
|
|
1000
|
+
workerServerType: newValues.workerType || currentWorkerType,
|
|
1001
|
+
region,
|
|
1002
|
+
services,
|
|
1003
|
+
});
|
|
1004
|
+
p.note(formatCostLines(costBreakdown), 'New Estimated Cost');
|
|
1005
|
+
|
|
1006
|
+
// 6. Require API token if infra changes (server type swap) needed
|
|
1007
|
+
let apiToken = null;
|
|
1008
|
+
if (infraChanges.length > 0) {
|
|
1009
|
+
apiToken = await getApiToken(projectConfig.projectName);
|
|
1010
|
+
if (!apiToken) {
|
|
1011
|
+
p.log.error('API token required for server type changes');
|
|
1012
|
+
process.exit(1);
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
// Ensure operator IP is in the firewall allowlist before any SSH or k8s
|
|
1017
|
+
// API work. Skip when there's no apiToken (no infra changes and no SSH —
|
|
1018
|
+
// a no-op scale that just returns) since both paths below require token.
|
|
1019
|
+
if (apiToken) {
|
|
1020
|
+
try {
|
|
1021
|
+
const isHA = !!envConfig.ha?.enabled;
|
|
1022
|
+
const result = await ensureOperatorIpAccess({
|
|
1023
|
+
projectConfig,
|
|
1024
|
+
environment,
|
|
1025
|
+
isHA,
|
|
1026
|
+
apiToken,
|
|
1027
|
+
yes: !!parsed.yes,
|
|
1028
|
+
onMessage: (msg) => p.log.info(msg),
|
|
1029
|
+
});
|
|
1030
|
+
if (result.added) {
|
|
1031
|
+
p.log.success(`Firewall updated: SSH/k8s API now allow ${result.cidr}`);
|
|
1032
|
+
}
|
|
1033
|
+
} catch (err) {
|
|
1034
|
+
p.log.error(`Operator-IP check failed: ${err.message}`);
|
|
1035
|
+
process.exit(1);
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
// 7+8. Apply changes — for HA environments, apply to both clusters
|
|
1040
|
+
const secondaryRegion = envConfig.secondaryRegion;
|
|
1041
|
+
|
|
1042
|
+
async function applyScaleChanges({ clusterEnv, clusterRegion, label }) {
|
|
1043
|
+
// Apply server type changes via Pulumi. Hetzner server_type is an
|
|
1044
|
+
// in-place resize via the provider's update path (~2 min reboot), so a
|
|
1045
|
+
// legitimate type-bump shows up as `update`, not `replace`. Any planned
|
|
1046
|
+
// `replace` of master is the destructive userData-drift bug — see the
|
|
1047
|
+
// master-replace defense block below.
|
|
1048
|
+
if (infraChanges.length > 0) {
|
|
1049
|
+
const { upStack, getStackOutputs, getOrCreateStack, classifyK3sTokenProbe } = await import(
|
|
1050
|
+
'./lib/iac/index.js'
|
|
1051
|
+
);
|
|
1052
|
+
const { buildHetznerK8sProgram } = await import('./lib/iac/programs/hetzner-k8s.js');
|
|
1053
|
+
const { K3S_VERSION } = await import('./lib/deploy/k8s/k3s.js');
|
|
1054
|
+
const REGION_TO_NETWORK_ZONE = {
|
|
1055
|
+
nbg1: 'eu-central',
|
|
1056
|
+
hel1: 'eu-central',
|
|
1057
|
+
fsn1: 'eu-central',
|
|
1058
|
+
hil: 'us-west',
|
|
1059
|
+
ash: 'us-east',
|
|
1060
|
+
sin: 'ap-southeast',
|
|
1061
|
+
};
|
|
1062
|
+
// SSH key location depends on deploy topology:
|
|
1063
|
+
// - Single-cluster k8s: deployK3s' ensureSshKey writes
|
|
1064
|
+
// `.vibecarbon/ssh-<env>` and lets Pulumi manage the SshKey resource.
|
|
1065
|
+
// - HA k8s: deployK8sHA generates a SHARED key at
|
|
1066
|
+
// `.vibecarbon/deploy_key_<environment>` (one file, used by both
|
|
1067
|
+
// primary and standby), uploads it to Hetzner ONCE via
|
|
1068
|
+
// uploadSshKeyToHetzner, and passes the resulting `existingSshKeyId`
|
|
1069
|
+
// into buildHetznerK8sProgram so neither stack manages the SshKey
|
|
1070
|
+
// resource directly.
|
|
1071
|
+
//
|
|
1072
|
+
// If we read the wrong path, sshPublicKey resolves to `''` and Pulumi
|
|
1073
|
+
// would either replace the managed SshKey (single-cluster) or — for
|
|
1074
|
+
// HA — try to *create* a brand-new SshKey resource that the original
|
|
1075
|
+
// deploy never managed. The new resource has a different id,
|
|
1076
|
+
// master/supabase/worker `sshKeys: [sshKeyId]` diffs against state,
|
|
1077
|
+
// and Pulumi plans a destructive replace of every node (etcd loss).
|
|
1078
|
+
// The k3sToken probe (further down) is necessary but insufficient
|
|
1079
|
+
// for HA — we ALSO need to mirror the deploy's `existingSshKeyId`
|
|
1080
|
+
// decision, which means reading the shared key file and re-resolving
|
|
1081
|
+
// the Hetzner SshKey id by name.
|
|
1082
|
+
let sshPublicKey = '';
|
|
1083
|
+
let existingSshKeyId;
|
|
1084
|
+
if (isHA) {
|
|
1085
|
+
const sharedKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}.pub`);
|
|
1086
|
+
if (existsSync(sharedKeyPath)) {
|
|
1087
|
+
sshPublicKey = readFileSync(sharedKeyPath, 'utf-8').trim();
|
|
1088
|
+
} else {
|
|
1089
|
+
p.log.error(
|
|
1090
|
+
`HA shared SSH key not found at ${sharedKeyPath}. ` +
|
|
1091
|
+
`Was this environment deployed with the current CLI? Refusing to ` +
|
|
1092
|
+
`proceed — a missing key would force Pulumi to recreate every node.`,
|
|
1093
|
+
);
|
|
1094
|
+
process.exit(1);
|
|
1095
|
+
}
|
|
1096
|
+
// Resolve the existing Hetzner SshKey id by the deploy-time name.
|
|
1097
|
+
// HetznerProvider.createSSHKey is idempotent: matches first by
|
|
1098
|
+
// <name>, then by public-key bytes, and only POSTs when neither
|
|
1099
|
+
// exists. Same call deploy made via uploadSshKeyToHetzner — must
|
|
1100
|
+
// match so Pulumi sees the same `existingSshKeyId` input.
|
|
1101
|
+
const provider = new HetznerProvider(apiToken);
|
|
1102
|
+
const haSshKeyName = `${projectConfig.projectName}-${environment}-ha-key`;
|
|
1103
|
+
try {
|
|
1104
|
+
const id = await provider.createSSHKey(haSshKeyName, sshPublicKey);
|
|
1105
|
+
existingSshKeyId = String(id);
|
|
1106
|
+
} catch (err) {
|
|
1107
|
+
p.log.error(
|
|
1108
|
+
`Could not resolve HA SSH key '${haSshKeyName}' in Hetzner: ${err.message?.split('\n')[0] || err}`,
|
|
1109
|
+
);
|
|
1110
|
+
process.exit(1);
|
|
1111
|
+
}
|
|
1112
|
+
} else {
|
|
1113
|
+
const sshKeyPath = join(process.cwd(), '.vibecarbon', `ssh-${clusterEnv}`);
|
|
1114
|
+
sshPublicKey = existsSync(`${sshKeyPath}.pub`)
|
|
1115
|
+
? readFileSync(`${sshKeyPath}.pub`, 'utf-8').trim()
|
|
1116
|
+
: '';
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
// Reconstruct the s3Config that the original deploy used so pulumi
|
|
1120
|
+
// hits the same backend bucket. envConfig persists bucket + region +
|
|
1121
|
+
// endpoint under `s3` (not `s3Config` — the typo on this line was
|
|
1122
|
+
// silently undefined-ing the config, falling pulumi back to file://
|
|
1123
|
+
// backend with empty state, then trying to CREATE every Hetzner
|
|
1124
|
+
// resource fresh and 409-ing on "name is already used"). Keys aren't
|
|
1125
|
+
// persisted (they live in ~/.vibecarbon/credentials.json) so we
|
|
1126
|
+
// re-fetch them via getS3Credentials.
|
|
1127
|
+
let s3Config;
|
|
1128
|
+
if (envConfig.s3?.bucket) {
|
|
1129
|
+
const s3Creds = await getS3Credentials(projectConfig.projectName, { save: false });
|
|
1130
|
+
if (!s3Creds) {
|
|
1131
|
+
p.log.error('S3 credentials required to access pulumi backend');
|
|
1132
|
+
process.exit(1);
|
|
1133
|
+
}
|
|
1134
|
+
s3Config = {
|
|
1135
|
+
bucket: envConfig.s3.bucket,
|
|
1136
|
+
region: envConfig.s3.region,
|
|
1137
|
+
endpoint: envConfig.s3.endpoint,
|
|
1138
|
+
accessKey: s3Creds.accessKey,
|
|
1139
|
+
secretKey: s3Creds.secretKey,
|
|
1140
|
+
};
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
// Common program inputs shared by both the probe (read-only outputs())
|
|
1144
|
+
// and the real `up` call. They MUST be byte-identical to what deployK3s
|
|
1145
|
+
// passed at deploy time — every input that flows into a Server's
|
|
1146
|
+
// `sshKeys`, `firewallIds`, `labels`, or `userData` is a tripwire for
|
|
1147
|
+
// destructive replace. The k3sToken probe-and-replay (further below)
|
|
1148
|
+
// keeps userData stable; HA needed `existingSshKeyId` to also be
|
|
1149
|
+
// mirrored or master.sshKeys would still drift and replace etcd.
|
|
1150
|
+
const allowedCidrs = (projectConfig.operatorCidrs ?? []).map((e) => e.cidr);
|
|
1151
|
+
const programConfig = {
|
|
1152
|
+
projectName: projectConfig.projectName,
|
|
1153
|
+
environment: clusterEnv,
|
|
1154
|
+
sshPublicKey,
|
|
1155
|
+
allowedSshIps: allowedCidrs,
|
|
1156
|
+
allowedK8sApiIps: allowedCidrs,
|
|
1157
|
+
// HA deploys share one Hetzner SshKey across primary + standby and
|
|
1158
|
+
// pass `existingSshKeyId` so neither stack manages the resource.
|
|
1159
|
+
// Single-cluster lets Pulumi own the SshKey (existingSshKeyId =
|
|
1160
|
+
// undefined). Mirror whichever the deploy did or master.sshKeys
|
|
1161
|
+
// changes and Pulumi replaces every node.
|
|
1162
|
+
existingSshKeyId,
|
|
1163
|
+
location: clusterRegion,
|
|
1164
|
+
networkZone: REGION_TO_NETWORK_ZONE[clusterRegion] || 'eu-central',
|
|
1165
|
+
masterServerType: newValues.masterType ?? currentMasterType,
|
|
1166
|
+
supabaseServerType: newValues.supabaseType ?? currentSupabaseType,
|
|
1167
|
+
workerServerType: newValues.workerType ?? currentWorkerType,
|
|
1168
|
+
// Phase 8: minWorkers/maxWorkers replace workerCount. minWorkers is
|
|
1169
|
+
// what Pulumi consumes (static floor of worker servers it owns).
|
|
1170
|
+
// maxWorkers is not consumed by Pulumi; it flows to the CA Deployment
|
|
1171
|
+
// patch below. If the operator passed --min-workers/--max-workers
|
|
1172
|
+
// we honor them; otherwise replay the previously persisted values
|
|
1173
|
+
// (which are needed verbatim to keep Pulumi's plan a no-op for the
|
|
1174
|
+
// worker pool when the operator is just resizing types).
|
|
1175
|
+
minWorkers: newValues.minWorkers ?? envConfig.minWorkers ?? DEFAULT_WORKER_MIN,
|
|
1176
|
+
maxWorkers: newValues.maxWorkers ?? envConfig.maxWorkers ?? DEFAULT_WORKER_MAX,
|
|
1177
|
+
// MUST match the K3S_VERSION used by deployK3s — k3sVersion is
|
|
1178
|
+
// interpolated into each server's userData (cloud-init script), so a
|
|
1179
|
+
// mismatch makes Pulumi see ~userData on master/supabase/workers and
|
|
1180
|
+
// tries to replace them. Replacing master = etcd wipe.
|
|
1181
|
+
k3sVersion: K3S_VERSION,
|
|
1182
|
+
// MUST match the labels passed by deployK3s (src/lib/deploy/k8s/k3s.js).
|
|
1183
|
+
// Mismatched labels make Pulumi mark every resource as ~labels and
|
|
1184
|
+
// attempt a replace — for hcloud.SshKey, replace = delete+create with
|
|
1185
|
+
// the same public-key bytes, which Hetzner rejects with HTTP 409.
|
|
1186
|
+
labels: { 'managed-by': 'vibecarbon', 'os-flavor': 'k3s' },
|
|
1187
|
+
apiToken,
|
|
1188
|
+
};
|
|
1189
|
+
|
|
1190
|
+
// Probe prior stack outputs for the k3sToken so we can replay it into
|
|
1191
|
+
// the real program. classifyK3sTokenProbe normalizes recovered/empty/
|
|
1192
|
+
// errored into a tagged result so the visible logging path matches the
|
|
1193
|
+
// unit tests for this seam (see tests/unit/iac/probe-classify.test.ts).
|
|
1194
|
+
let probeOutputs = null;
|
|
1195
|
+
let probeError = null;
|
|
1196
|
+
try {
|
|
1197
|
+
const probeProgram = buildHetznerK8sProgram(programConfig);
|
|
1198
|
+
probeOutputs = await perfAsync(`scale.k8s.${clusterEnv}.probeOutputs`, () =>
|
|
1199
|
+
getStackOutputs(clusterEnv, probeProgram, {
|
|
1200
|
+
hcloudToken: apiToken,
|
|
1201
|
+
s3Config,
|
|
1202
|
+
}),
|
|
1203
|
+
);
|
|
1204
|
+
} catch (err) {
|
|
1205
|
+
probeError = err instanceof Error ? err : new Error(String(err));
|
|
1206
|
+
}
|
|
1207
|
+
const probe = classifyK3sTokenProbe({ outputs: probeOutputs, error: probeError });
|
|
1208
|
+
const priorK3sToken = probe.priorK3sToken;
|
|
1209
|
+
switch (probe.status) {
|
|
1210
|
+
case 'recovered':
|
|
1211
|
+
break;
|
|
1212
|
+
case 'empty':
|
|
1213
|
+
// Legit on a fresh stack; suspicious on an established one. Either
|
|
1214
|
+
// way, the master-replace defense below is what protects etcd —
|
|
1215
|
+
// we just log loudly so a misbehaving probe is visible in run logs.
|
|
1216
|
+
p.log.warn(
|
|
1217
|
+
`${label}: ${probe.reason}. Proceeding with newly-minted k3sToken — master-replace defense will block any destructive plan.`,
|
|
1218
|
+
);
|
|
1219
|
+
break;
|
|
1220
|
+
case 'errored':
|
|
1221
|
+
// Genuine backend failure (S3 outage, expired creds, corrupt
|
|
1222
|
+
// state). Surface as an error so it doesn't blend into routine
|
|
1223
|
+
// info noise. The real `up` below will resurface the same problem
|
|
1224
|
+
// with a cleaner message.
|
|
1225
|
+
p.log.error(
|
|
1226
|
+
`${label}: ${probe.reason}. Master-replace defense remains armed; if it doesn't fire, fix the backend before scaling.`,
|
|
1227
|
+
);
|
|
1228
|
+
break;
|
|
1229
|
+
default:
|
|
1230
|
+
// Future-proof: classifyK3sTokenProbe is the single source of
|
|
1231
|
+
// truth for status values; surface any unhandled status so a
|
|
1232
|
+
// silent regression can't flow through.
|
|
1233
|
+
p.log.error(`${label}: unexpected probe status '${probe.status}': ${probe.reason}`);
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
const program = buildHetznerK8sProgram({
|
|
1237
|
+
...programConfig,
|
|
1238
|
+
// Replay the prior token (when available) so userData hashes match
|
|
1239
|
+
// and Pulumi plans an in-place server-type change rather than a
|
|
1240
|
+
// destructive replace of every node.
|
|
1241
|
+
k3sToken: priorK3sToken,
|
|
1242
|
+
});
|
|
1243
|
+
|
|
1244
|
+
// Defense in depth: run a Pulumi preview and refuse to proceed if any
|
|
1245
|
+
// resource with role=master is scheduled for replacement. The k3sToken
|
|
1246
|
+
// probe-and-replay above should make this unreachable, but if any
|
|
1247
|
+
// future input drift (cloud-init template change, label rename, etc.)
|
|
1248
|
+
// re-introduces userData hash drift, this fails fast instead of
|
|
1249
|
+
// silently wiping etcd. Replacements show up under three op codes:
|
|
1250
|
+
// "replace", "create-replacement", and "delete-replaced" — Pulumi
|
|
1251
|
+
// emits all three for a single replaced resource. Catching any one
|
|
1252
|
+
// is sufficient.
|
|
1253
|
+
const s = p.spinner();
|
|
1254
|
+
s.start(`Previewing infra changes (${label})...`);
|
|
1255
|
+
// perf coverage: scale on k8s/k8s-ha was a black box — cli.scale.total
|
|
1256
|
+
// landed at ~470s with only ~9s instrumented (waitForCsiNode +
|
|
1257
|
+
// waitForPodsReady at the tail). The dominant cost is the Pulumi
|
|
1258
|
+
// preview + up cycle below; wrap each so perf_substep can finally
|
|
1259
|
+
// attribute the gap. Tag with clusterEnv (e.g. e4-primary vs
|
|
1260
|
+
// e4-standby) so HA primary/standby are distinguishable.
|
|
1261
|
+
try {
|
|
1262
|
+
const previewStack = await perfAsync(`scale.k8s.${clusterEnv}.previewStack`, () =>
|
|
1263
|
+
getOrCreateStack(clusterEnv, program, {
|
|
1264
|
+
hcloudToken: apiToken,
|
|
1265
|
+
s3Config,
|
|
1266
|
+
}),
|
|
1267
|
+
);
|
|
1268
|
+
const masterReplacements = [];
|
|
1269
|
+
await perfAsync(`scale.k8s.${clusterEnv}.preview`, () =>
|
|
1270
|
+
previewStack.preview({
|
|
1271
|
+
color: 'never',
|
|
1272
|
+
onEvent: (event) => {
|
|
1273
|
+
const meta = event?.resourcePreEvent?.metadata;
|
|
1274
|
+
if (!meta) return;
|
|
1275
|
+
const isReplaceOp =
|
|
1276
|
+
meta.op === 'replace' ||
|
|
1277
|
+
meta.op === 'create-replacement' ||
|
|
1278
|
+
meta.op === 'delete-replaced';
|
|
1279
|
+
if (!isReplaceOp) return;
|
|
1280
|
+
// Resource was created with `new hcloud.Server('master', ...)`
|
|
1281
|
+
// and labelled `role: 'master'`. Either signal flags it.
|
|
1282
|
+
const role = meta.new?.inputs?.labels?.role || meta.old?.inputs?.labels?.role || null;
|
|
1283
|
+
const urn = meta.urn || '';
|
|
1284
|
+
if (role === 'master' || /::master$/.test(urn)) {
|
|
1285
|
+
masterReplacements.push({ op: meta.op, urn });
|
|
1286
|
+
}
|
|
1287
|
+
},
|
|
1288
|
+
}),
|
|
1289
|
+
);
|
|
1290
|
+
s.stop(`Preview complete (${label})`);
|
|
1291
|
+
if (masterReplacements.length > 0) {
|
|
1292
|
+
p.log.error(
|
|
1293
|
+
`Refusing to scale: would replace master node (etcd loss). ` +
|
|
1294
|
+
`Inspect with: pulumi preview --stack ${clusterEnv}`,
|
|
1295
|
+
);
|
|
1296
|
+
for (const r of masterReplacements) {
|
|
1297
|
+
p.log.error(` ${r.op}: ${r.urn}`);
|
|
1298
|
+
}
|
|
1299
|
+
process.exit(1);
|
|
1300
|
+
}
|
|
1301
|
+
} catch (err) {
|
|
1302
|
+
s.stop(`Pulumi preview failed (${label})`);
|
|
1303
|
+
p.log.error(err.message);
|
|
1304
|
+
process.exit(1);
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
s.start(`Applying infra changes via Pulumi (${label})...`);
|
|
1308
|
+
try {
|
|
1309
|
+
await perfAsync(`scale.k8s.${clusterEnv}.upStack`, () =>
|
|
1310
|
+
upStack(clusterEnv, program, {
|
|
1311
|
+
hcloudToken: apiToken,
|
|
1312
|
+
s3Config,
|
|
1313
|
+
}),
|
|
1314
|
+
);
|
|
1315
|
+
s.stop(`Infra updated (${label})`);
|
|
1316
|
+
} catch (err) {
|
|
1317
|
+
s.stop(`Pulumi up failed (${label})`);
|
|
1318
|
+
p.log.error(err.message);
|
|
1319
|
+
process.exit(1);
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
// Phase 8: re-patch the cluster-autoscaler Deployment's `--nodes` argv
|
|
1324
|
+
// slot whenever bounds or worker type changed. The CA pool spec is
|
|
1325
|
+
// `<min>:<max>:<workerServerType>:<region>:<poolName>` per
|
|
1326
|
+
// formatCaNodesSpec — bumping the type without re-patching CA leaves
|
|
1327
|
+
// the autoscaler spawning the OLD type for any new nodes. Bumping
|
|
1328
|
+
// bounds without a type change still requires a re-patch.
|
|
1329
|
+
//
|
|
1330
|
+
// When --type is passed alongside bounds (or vs Pulumi already
|
|
1331
|
+
// resized workers), CA-spawned workers should also be drained so CA
|
|
1332
|
+
// re-spawns them at the new type. That drain step is deferred to a
|
|
1333
|
+
// follow-up — for now we surface a 1-line warning.
|
|
1334
|
+
const needsCaRepatch = changes.includes('workerType') || changes.includes('workerBounds');
|
|
1335
|
+
if (needsCaRepatch) {
|
|
1336
|
+
const kubeconfigPath = join(process.cwd(), '.vibecarbon', `kubeconfig-${clusterEnv}`);
|
|
1337
|
+
if (!existsSync(kubeconfigPath)) {
|
|
1338
|
+
p.log.warn(`kubeconfig missing at ${kubeconfigPath} (${label}); skipping CA re-patch`);
|
|
1339
|
+
} else {
|
|
1340
|
+
const { formatCaNodesSpec } = await import('./lib/deploy/k8s/k3s.js');
|
|
1341
|
+
const newMin = newValues.minWorkers ?? envConfig.minWorkers ?? DEFAULT_WORKER_MIN;
|
|
1342
|
+
const newMax = newValues.maxWorkers ?? envConfig.maxWorkers ?? DEFAULT_WORKER_MAX;
|
|
1343
|
+
const newType = newValues.workerType ?? currentWorkerType;
|
|
1344
|
+
const caNodesSpec = formatCaNodesSpec({
|
|
1345
|
+
minWorkers: newMin,
|
|
1346
|
+
maxWorkers: newMax,
|
|
1347
|
+
workerServerType: newType,
|
|
1348
|
+
region: clusterRegion,
|
|
1349
|
+
});
|
|
1350
|
+
// Patch the CA Deployment's container args via JSON-pointer so
|
|
1351
|
+
// a kubectl rollout restart picks up the new --nodes flag. The
|
|
1352
|
+
// `--nodes=` arg lives at .spec.template.spec.containers[0].args
|
|
1353
|
+
// and is the only arg we need to update.
|
|
1354
|
+
const caSpinner = p.spinner();
|
|
1355
|
+
caSpinner.start(`Re-patching cluster-autoscaler --nodes (${label})`);
|
|
1356
|
+
try {
|
|
1357
|
+
// Read current args so we can replace just the --nodes one.
|
|
1358
|
+
// runCommand returns the captured stdout string when `silent: true`.
|
|
1359
|
+
const argsJsonRaw = runCommand(
|
|
1360
|
+
[
|
|
1361
|
+
'kubectl',
|
|
1362
|
+
'--kubeconfig',
|
|
1363
|
+
kubeconfigPath,
|
|
1364
|
+
'-n',
|
|
1365
|
+
'kube-system',
|
|
1366
|
+
'get',
|
|
1367
|
+
'deploy',
|
|
1368
|
+
'cluster-autoscaler',
|
|
1369
|
+
'-o',
|
|
1370
|
+
'jsonpath={.spec.template.spec.containers[0].args}',
|
|
1371
|
+
],
|
|
1372
|
+
{ silent: true },
|
|
1373
|
+
);
|
|
1374
|
+
const argsJson = typeof argsJsonRaw === 'string' ? argsJsonRaw.trim() : '';
|
|
1375
|
+
let argList = [];
|
|
1376
|
+
try {
|
|
1377
|
+
argList = JSON.parse(argsJson);
|
|
1378
|
+
} catch {
|
|
1379
|
+
argList = [];
|
|
1380
|
+
}
|
|
1381
|
+
const updated = argList.map((a) =>
|
|
1382
|
+
typeof a === 'string' && a.startsWith('--nodes=') ? `--nodes=${caNodesSpec}` : a,
|
|
1383
|
+
);
|
|
1384
|
+
// If for some reason --nodes wasn't there, append it.
|
|
1385
|
+
if (!updated.some((a) => typeof a === 'string' && a.startsWith('--nodes='))) {
|
|
1386
|
+
updated.push(`--nodes=${caNodesSpec}`);
|
|
1387
|
+
}
|
|
1388
|
+
const patch = JSON.stringify({
|
|
1389
|
+
spec: {
|
|
1390
|
+
template: { spec: { containers: [{ name: 'cluster-autoscaler', args: updated }] } },
|
|
1391
|
+
},
|
|
1392
|
+
});
|
|
1393
|
+
runCommand(
|
|
1394
|
+
[
|
|
1395
|
+
'kubectl',
|
|
1396
|
+
'--kubeconfig',
|
|
1397
|
+
kubeconfigPath,
|
|
1398
|
+
'-n',
|
|
1399
|
+
'kube-system',
|
|
1400
|
+
'patch',
|
|
1401
|
+
'deploy',
|
|
1402
|
+
'cluster-autoscaler',
|
|
1403
|
+
'--type=strategic',
|
|
1404
|
+
'-p',
|
|
1405
|
+
patch,
|
|
1406
|
+
],
|
|
1407
|
+
{ silent: true },
|
|
1408
|
+
);
|
|
1409
|
+
runCommand(
|
|
1410
|
+
[
|
|
1411
|
+
'kubectl',
|
|
1412
|
+
'--kubeconfig',
|
|
1413
|
+
kubeconfigPath,
|
|
1414
|
+
'-n',
|
|
1415
|
+
'kube-system',
|
|
1416
|
+
'rollout',
|
|
1417
|
+
'restart',
|
|
1418
|
+
'deploy/cluster-autoscaler',
|
|
1419
|
+
],
|
|
1420
|
+
{ silent: true },
|
|
1421
|
+
);
|
|
1422
|
+
caSpinner.stop(`cluster-autoscaler re-patched (${label}): ${caNodesSpec}`);
|
|
1423
|
+
if (changes.includes('workerType')) {
|
|
1424
|
+
p.log.warn(
|
|
1425
|
+
`CA-spawned workers (if any) are still on the old type. ` +
|
|
1426
|
+
`Drain them with kubectl drain so CA respawns at the new type.`,
|
|
1427
|
+
);
|
|
1428
|
+
}
|
|
1429
|
+
} catch (err) {
|
|
1430
|
+
caSpinner.stop(`CA re-patch failed (${label})`);
|
|
1431
|
+
p.log.warn(
|
|
1432
|
+
`Could not re-patch cluster-autoscaler: ${err.message?.split('\n')[0] || err}`,
|
|
1433
|
+
);
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
// Apply to primary (or single) cluster
|
|
1440
|
+
await applyScaleChanges({
|
|
1441
|
+
clusterEnv: isHA ? `${environment}-primary` : environment,
|
|
1442
|
+
clusterRegion: region,
|
|
1443
|
+
label: isHA ? `primary — ${region}` : region,
|
|
1444
|
+
});
|
|
1445
|
+
|
|
1446
|
+
// Apply to standby cluster if HA
|
|
1447
|
+
if (isHA) {
|
|
1448
|
+
p.log.step(`Applying same changes to standby cluster (${secondaryRegion})...`);
|
|
1449
|
+
await applyScaleChanges({
|
|
1450
|
+
clusterEnv: `${environment}-standby`,
|
|
1451
|
+
clusterRegion: secondaryRegion,
|
|
1452
|
+
label: `standby — ${secondaryRegion}`,
|
|
1453
|
+
});
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
// Hetzner in-place server_type changes are ~2 min reboots per node.
|
|
1457
|
+
// Pulumi's upStack returns when the VM is back up + k3s reports
|
|
1458
|
+
// ready, but stateful workloads inside the cluster (notably
|
|
1459
|
+
// supabase-supabase-db Postgres) need additional time to finish
|
|
1460
|
+
// their own init / WAL replay before they accept connections. If
|
|
1461
|
+
// the next caller (e.g. e2e backup step) runs against a
|
|
1462
|
+
// still-recovering cluster, pg_dump gets "connection refused" and
|
|
1463
|
+
// the work fails for what looks like a backup bug but is actually
|
|
1464
|
+
// a "scale didn't wait for the cluster to settle" problem. Observed
|
|
1465
|
+
// 2026-04-28 e2e — every backup-step failure across three
|
|
1466
|
+
// reruns had this same root cause.
|
|
1467
|
+
//
|
|
1468
|
+
// Wait for all pods in the vibecarbon namespace to reach
|
|
1469
|
+
// Ready=True. The supabase chart's readiness probe on
|
|
1470
|
+
// supabase-supabase-db is TCP-based on 5432, so once that pod is
|
|
1471
|
+
// Ready the connection-refused window has closed.
|
|
1472
|
+
const clustersToVerify = isHA
|
|
1473
|
+
? [
|
|
1474
|
+
{ env: `${environment}-primary`, label: `primary — ${region}` },
|
|
1475
|
+
{ env: `${environment}-standby`, label: `standby — ${secondaryRegion}` },
|
|
1476
|
+
]
|
|
1477
|
+
: [{ env: environment, label: region }];
|
|
1478
|
+
// 10 min is the floor that holds for HA (k8s-ha rerun 2026-04-28: standby
|
|
1479
|
+
// timed out 12/12 pods at 5 min — supabase-db-0 alone needs ~60-90s to
|
|
1480
|
+
// start postgres after a node reboot, then auth/kong/realtime/etc.
|
|
1481
|
+
// dependency-chain off it, then app pulls images that may not be cached
|
|
1482
|
+
// on the freshly-rebooted node). Single-cluster k8s only used ~70s of
|
|
1483
|
+
// its old 300s budget, so the bump costs nothing on the happy path.
|
|
1484
|
+
const WAIT_TIMEOUT_SEC = 600;
|
|
1485
|
+
for (const { env: clusterEnv, label } of clustersToVerify) {
|
|
1486
|
+
const kubeconfigPath = join(process.cwd(), '.vibecarbon', `kubeconfig-${clusterEnv}`);
|
|
1487
|
+
if (!existsSync(kubeconfigPath)) {
|
|
1488
|
+
p.log.warn(
|
|
1489
|
+
`kubeconfig missing at ${kubeconfigPath} (${label}); skipping post-resize Ready wait`,
|
|
1490
|
+
);
|
|
1491
|
+
continue;
|
|
1492
|
+
}
|
|
1493
|
+
// Wait for the API server to be reachable BEFORE running any kubectl
|
|
1494
|
+
// commands. RCA from k8s scale failure 2026-05-07 (e2e run
|
|
1495
|
+
// bt5zvlw7n): Pulumi's upStack returns when the Hetzner VM is back up
|
|
1496
|
+
// and k3s reports ready inside the master, but the floating IP that
|
|
1497
|
+
// kubeconfig points at re-attaches asynchronously — there's a 30-90s
|
|
1498
|
+
// window where `kubectl --kubeconfig ...` returns
|
|
1499
|
+
// `ServiceUnavailable: the server is currently unable to handle the
|
|
1500
|
+
// request`. The next call (`kubectl rollout status` for hcloud-csi-node)
|
|
1501
|
+
// exits in <1s with that error, then `kubectl wait --for=Ready pods`
|
|
1502
|
+
// does the same, and scale fails for what looks like "pods didn't
|
|
1503
|
+
// settle" but is actually "API server wasn't reachable yet". Poll
|
|
1504
|
+
// `--raw=/healthz` until we see ok before continuing — k3s's healthz
|
|
1505
|
+
// returns "ok" only when the apiserver + scheduler + controller-manager
|
|
1506
|
+
// are all alive, so a single success here means the cluster control
|
|
1507
|
+
// plane has truly stabilized post-resize.
|
|
1508
|
+
const apiSpinner = p.spinner();
|
|
1509
|
+
apiSpinner.start(`Waiting for API server to be reachable (${label})`);
|
|
1510
|
+
const apiStart = Date.now();
|
|
1511
|
+
const API_READY_BUDGET_MS = 300_000;
|
|
1512
|
+
let apiReady = false;
|
|
1513
|
+
let lastApiErr = '';
|
|
1514
|
+
while (Date.now() - apiStart < API_READY_BUDGET_MS) {
|
|
1515
|
+
try {
|
|
1516
|
+
runCommand(['kubectl', '--kubeconfig', kubeconfigPath, 'get', '--raw=/healthz'], {
|
|
1517
|
+
silent: true,
|
|
1518
|
+
timeout: 10_000,
|
|
1519
|
+
});
|
|
1520
|
+
apiReady = true;
|
|
1521
|
+
break;
|
|
1522
|
+
} catch (err) {
|
|
1523
|
+
lastApiErr = err.message?.split('\n')[0]?.slice(0, 120) || String(err).slice(0, 120);
|
|
1524
|
+
await new Promise((r) => setTimeout(r, 5_000));
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
if (!apiReady) {
|
|
1528
|
+
apiSpinner.stop(`API server (${label}) not reachable within 5 min`);
|
|
1529
|
+
throw new Error(
|
|
1530
|
+
`Kube API server for ${label} never returned /healthz=ok within 5 min after Pulumi resize. ` +
|
|
1531
|
+
`Last error: ${lastApiErr}`,
|
|
1532
|
+
);
|
|
1533
|
+
}
|
|
1534
|
+
apiSpinner.stop(`API server reachable (${label})`);
|
|
1535
|
+
// Wait for the Hetzner CSI driver DaemonSet to register on every
|
|
1536
|
+
// node BEFORE checking pod-Ready. RCA from k8s-ha standby failure
|
|
1537
|
+
// 2026-04-28 (live cluster repro under triple-node-reboot):
|
|
1538
|
+
// kubelet can flip the supabase node Ready before hcloud-csi-node
|
|
1539
|
+
// pod registers; supabase-db-0 then tries to mount its PVC and
|
|
1540
|
+
// hits "MountVolume.MountDevice failed... driver name
|
|
1541
|
+
// csi.hetzner.cloud not found in the list of registered CSI
|
|
1542
|
+
// drivers". The pod sandbox gets killed + recreated in a tight
|
|
1543
|
+
// loop, eventually enters CrashLoopBackOff with a 5-min backoff
|
|
1544
|
+
// timer that the pod-Ready wait then can't outrun. Waiting for
|
|
1545
|
+
// the DS rollout first eliminates the race entirely.
|
|
1546
|
+
const csiSpinner = p.spinner();
|
|
1547
|
+
csiSpinner.start(`Waiting for hcloud-csi-node DaemonSet rollout (${label})`);
|
|
1548
|
+
try {
|
|
1549
|
+
await perfAsync('scale.waitForCsiNode', async () =>
|
|
1550
|
+
runCommand(
|
|
1551
|
+
[
|
|
1552
|
+
'kubectl',
|
|
1553
|
+
'--kubeconfig',
|
|
1554
|
+
kubeconfigPath,
|
|
1555
|
+
'-n',
|
|
1556
|
+
'kube-system',
|
|
1557
|
+
'rollout',
|
|
1558
|
+
'status',
|
|
1559
|
+
'daemonset/hcloud-csi-node',
|
|
1560
|
+
'--timeout=300s',
|
|
1561
|
+
],
|
|
1562
|
+
{ silent: true },
|
|
1563
|
+
),
|
|
1564
|
+
);
|
|
1565
|
+
csiSpinner.stop(`hcloud-csi-node Ready on all nodes (${label})`);
|
|
1566
|
+
} catch {
|
|
1567
|
+
csiSpinner.stop(`hcloud-csi-node rollout did not complete in 5 min (${label})`);
|
|
1568
|
+
// Continue anyway — the pod-Ready wait below will surface any
|
|
1569
|
+
// real persistent issue via the diag dump.
|
|
1570
|
+
}
|
|
1571
|
+
// (Live debug 2026-04-28 found that bouncing pods to break a
|
|
1572
|
+
// CrashLoopBackOff cycle leaves the underlying CSI VolumeAttachment
|
|
1573
|
+
// stuck in "deletionTimestamp set, finalizer external-attacher/
|
|
1574
|
+
// csi-hetzner-cloud held" state — supabase-db-0 then sits in
|
|
1575
|
+
// Init:0/1 for 10+ min waiting for the new VolumeAttachment to
|
|
1576
|
+
// bind. Solution: do not bounce. Rely on the csi-node rollout
|
|
1577
|
+
// wait above eliminating the race in the first place; a pod that
|
|
1578
|
+
// entered BackOff before CSI registered will exit BackOff cleanly
|
|
1579
|
+
// once kubelet's next retry tick fires (5-min cap).)
|
|
1580
|
+
//
|
|
1581
|
+
// HA standby exception: skip the pod-Ready wait entirely when this
|
|
1582
|
+
// is the standby cluster. The standby's DB is supposed to be a
|
|
1583
|
+
// streaming replica (read-only), and the supabase chart's
|
|
1584
|
+
// app-tier pods (app/kong/rest/auth/realtime/storage) currently
|
|
1585
|
+
// can't satisfy their readiness probes against a read-only DB —
|
|
1586
|
+
// their probes call /api/health/ready (or similar) which times out
|
|
1587
|
+
// upstream and returns 503. `kubectl wait --all` then blocks until
|
|
1588
|
+
// its 10-min budget runs out and the scale step fails for what
|
|
1589
|
+
// looks like "scale didn't settle" but is actually "standby's app
|
|
1590
|
+
// tier is not designed to Ready before failover". Surfaced
|
|
1591
|
+
// 2026-04-29 in k8s-ha e2e run 674f03c3-8b7a after the
|
|
1592
|
+
// earlier deploy-step issues were fixed — scale was the first
|
|
1593
|
+
// step doing `wait --all` against the standby kubeconfig.
|
|
1594
|
+
//
|
|
1595
|
+
// The CSI rollout above already verifies node-level health
|
|
1596
|
+
// post-resize; pod-level readiness on standby is validated end-
|
|
1597
|
+
// to-end by the failover step (which promotes the DB and re-
|
|
1598
|
+
// probes the app pods). Until HA replication is properly wired
|
|
1599
|
+
// up — see memory: project_replication_broken.md ("K8s HA
|
|
1600
|
+
// replication never worked") — pod-Ready against standby is a
|
|
1601
|
+
// known false-failure mode.
|
|
1602
|
+
const isStandbyCluster = clusterEnv.endsWith('-standby');
|
|
1603
|
+
if (isStandbyCluster) {
|
|
1604
|
+
p.log.info(
|
|
1605
|
+
`Skipping pod-Ready wait on standby (${label}) — replica-mode app tier ` +
|
|
1606
|
+
`is not expected to Ready until failover; CSI rollout above already ` +
|
|
1607
|
+
`verified node-level health.`,
|
|
1608
|
+
);
|
|
1609
|
+
continue;
|
|
1610
|
+
}
|
|
1611
|
+
const spinner = p.spinner();
|
|
1612
|
+
spinner.start(`Waiting for vibecarbon pods to be Ready (${label})`);
|
|
1613
|
+
try {
|
|
1614
|
+
await perfAsync('scale.waitForPodsReady', async () =>
|
|
1615
|
+
runCommand(
|
|
1616
|
+
[
|
|
1617
|
+
'kubectl',
|
|
1618
|
+
'--kubeconfig',
|
|
1619
|
+
kubeconfigPath,
|
|
1620
|
+
'-n',
|
|
1621
|
+
'vibecarbon',
|
|
1622
|
+
'wait',
|
|
1623
|
+
'--for=condition=Ready',
|
|
1624
|
+
'pods',
|
|
1625
|
+
'--all',
|
|
1626
|
+
`--timeout=${WAIT_TIMEOUT_SEC}s`,
|
|
1627
|
+
],
|
|
1628
|
+
{ silent: true },
|
|
1629
|
+
),
|
|
1630
|
+
);
|
|
1631
|
+
spinner.stop(`Pods Ready (${label})`);
|
|
1632
|
+
} catch (err) {
|
|
1633
|
+
spinner.stop(`Pods did not all become Ready within ${WAIT_TIMEOUT_SEC / 60} min (${label})`);
|
|
1634
|
+
// On timeout, dump pod state + describe + logs of any non-Ready
|
|
1635
|
+
// pods so the next debug session knows *which* pods didn't
|
|
1636
|
+
// recover and *why* (CrashLoopBackOff/ImagePullBackOff/post-reboot
|
|
1637
|
+
// postgres init failure). Without this the failure attribution is
|
|
1638
|
+
// just "kubectl wait timed out" with no signal as to which
|
|
1639
|
+
// controller is wedged.
|
|
1640
|
+
try {
|
|
1641
|
+
runCommand(
|
|
1642
|
+
[
|
|
1643
|
+
'kubectl',
|
|
1644
|
+
'--kubeconfig',
|
|
1645
|
+
kubeconfigPath,
|
|
1646
|
+
'-n',
|
|
1647
|
+
'vibecarbon',
|
|
1648
|
+
'get',
|
|
1649
|
+
'pods',
|
|
1650
|
+
'-o',
|
|
1651
|
+
'wide',
|
|
1652
|
+
],
|
|
1653
|
+
{ silent: false, ignoreError: true },
|
|
1654
|
+
);
|
|
1655
|
+
runCommand(
|
|
1656
|
+
[
|
|
1657
|
+
'kubectl',
|
|
1658
|
+
'--kubeconfig',
|
|
1659
|
+
kubeconfigPath,
|
|
1660
|
+
'-n',
|
|
1661
|
+
'vibecarbon',
|
|
1662
|
+
'get',
|
|
1663
|
+
'events',
|
|
1664
|
+
'--sort-by=.lastTimestamp',
|
|
1665
|
+
],
|
|
1666
|
+
{ silent: false, ignoreError: true },
|
|
1667
|
+
);
|
|
1668
|
+
// Per-non-Ready-pod describe + previous-container logs. We
|
|
1669
|
+
// shell-pipe through bash because kubectl doesn't have a native
|
|
1670
|
+
// way to "describe + logs everything that isn't Ready". Pre-Ready
|
|
1671
|
+
// pods have no `--previous` so we attempt both with-and-without.
|
|
1672
|
+
// Container name is left out so kubectl picks the main container
|
|
1673
|
+
// (works for single-container pods; multi-container pods get a
|
|
1674
|
+
// generic dump which still tells us *something*).
|
|
1675
|
+
runCommand(
|
|
1676
|
+
[
|
|
1677
|
+
'bash',
|
|
1678
|
+
'-c',
|
|
1679
|
+
`KCFG="${kubeconfigPath}"; for pod in $(kubectl --kubeconfig "$KCFG" -n vibecarbon get pods --no-headers 2>/dev/null | awk '$2 !~ /^([0-9]+)\\/\\1$/ {print $1}'); do
|
|
1680
|
+
echo "=== describe $pod ===";
|
|
1681
|
+
kubectl --kubeconfig "$KCFG" -n vibecarbon describe pod "$pod" 2>&1 | head -n 80;
|
|
1682
|
+
echo "=== logs $pod (current) ===";
|
|
1683
|
+
kubectl --kubeconfig "$KCFG" -n vibecarbon logs "$pod" --tail=80 --all-containers 2>&1 | head -n 200;
|
|
1684
|
+
echo "=== logs $pod (previous) ===";
|
|
1685
|
+
kubectl --kubeconfig "$KCFG" -n vibecarbon logs "$pod" --tail=80 --all-containers --previous 2>&1 | head -n 200;
|
|
1686
|
+
done`,
|
|
1687
|
+
],
|
|
1688
|
+
{ silent: false, ignoreError: true },
|
|
1689
|
+
);
|
|
1690
|
+
} catch {
|
|
1691
|
+
/* best-effort diag */
|
|
1692
|
+
}
|
|
1693
|
+
throw err;
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1697
|
+
// 9. Update .vibecarbon.json with new sizing inputs.
|
|
1698
|
+
//
|
|
1699
|
+
// Hetzner server_type changes are in-place resizes (not replaces) per
|
|
1700
|
+
// the master-replace guard's design comment above — the public IP
|
|
1701
|
+
// survives the type bump, so the persisted `servers[]` block still
|
|
1702
|
+
// points at the right hosts after scale. We only need to update the
|
|
1703
|
+
// sizing inputs (worker/master/supabase serverType). The serverType
|
|
1704
|
+
// field on each `servers[]` entry is left alone since failover/destroy
|
|
1705
|
+
// don't read it (status reads from the top-level `*ServerType` fields
|
|
1706
|
+
// below).
|
|
1707
|
+
const updatedEnvFields = {
|
|
1708
|
+
...(newValues.workerType && {
|
|
1709
|
+
workerServerType: newValues.workerType,
|
|
1710
|
+
serverType: newValues.workerType,
|
|
1711
|
+
}),
|
|
1712
|
+
...(newValues.masterType && { masterServerType: newValues.masterType }),
|
|
1713
|
+
...(newValues.supabaseType && { supabaseServerType: newValues.supabaseType }),
|
|
1714
|
+
// Phase 8: persist CA bounds so subsequent `vibecarbon deploy` and
|
|
1715
|
+
// `vibecarbon scale` runs replay the operator's most recent choice
|
|
1716
|
+
// rather than falling back to defaults.
|
|
1717
|
+
...(newValues.minWorkers != null && { minWorkers: newValues.minWorkers }),
|
|
1718
|
+
...(newValues.maxWorkers != null && { maxWorkers: newValues.maxWorkers }),
|
|
1719
|
+
};
|
|
1720
|
+
|
|
1721
|
+
const updatedConfig = {
|
|
1722
|
+
...projectConfig,
|
|
1723
|
+
environments: {
|
|
1724
|
+
...projectConfig.environments,
|
|
1725
|
+
[environment]: { ...envConfig, ...updatedEnvFields },
|
|
1726
|
+
},
|
|
1727
|
+
};
|
|
1728
|
+
saveProjectConfig(updatedConfig);
|
|
1729
|
+
|
|
1730
|
+
p.outro(`${c.success('Scale complete!')} Configuration updated in .vibecarbon.json`);
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
// Exported for tests.
|
|
1734
|
+
export { SPEC };
|