vibecarbon 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/destroy.js +903 -899
- package/src/lib/deploy/compose/index.js +43 -37
- package/src/lib/deploy/compose/single.js +421 -0
- package/src/lib/deploy/image.js +19 -7
- package/src/lib/deploy/k8s/k3s.js +346 -265
- package/src/lib/deploy/orchestrator.js +110 -459
- package/src/lib/deploy/tier-registry.js +30 -0
- package/src/lib/retry.js +59 -0
- package/src/lib/scale-plan.js +159 -0
- package/src/scale.js +101 -131
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import dns from 'node:dns';
|
|
7
|
-
import { readFileSync } from 'node:fs';
|
|
8
7
|
import { join } from 'node:path';
|
|
9
8
|
import * as p from '@clack/prompts';
|
|
10
9
|
import { c } from '../colors.js';
|
|
@@ -15,17 +14,13 @@ import {
|
|
|
15
14
|
registerProject,
|
|
16
15
|
saveProjectConfig,
|
|
17
16
|
} from '../config.js';
|
|
18
|
-
import { waitForDNSPropagation } from '../dns-propagation.js';
|
|
19
|
-
import { knownHostsPathForKey, seedKnownHosts } from '../host-keys.js';
|
|
20
17
|
import { ensureOperatorIpAccess } from '../operator-ip.js';
|
|
21
18
|
import { perfAsync, perfTimer } from '../perf.js';
|
|
22
19
|
import { createTracker } from '../tracker.js';
|
|
23
20
|
import { useDnsChallenge } from './acme.js';
|
|
24
21
|
import { renderBundle } from './bundle.js';
|
|
25
|
-
import { sideloadCompose } from './image.js';
|
|
26
|
-
import { buildRemote } from './remote-build.js';
|
|
27
22
|
import { StateTracker } from './state.js';
|
|
28
|
-
import {
|
|
23
|
+
import { isComposeTier, isHATier, isK8sTier, resolveTier } from './tier-registry.js';
|
|
29
24
|
|
|
30
25
|
/**
|
|
31
26
|
* Probe the public /api/health endpoint to verify the app is reachable
|
|
@@ -195,8 +190,7 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
195
190
|
} = gatheredConfig;
|
|
196
191
|
|
|
197
192
|
const { deployMode } = config;
|
|
198
|
-
const
|
|
199
|
-
const isHA = deployMode === 'compose-ha' || !!config.ha;
|
|
193
|
+
const tier = resolveTier({ deployMode, ha: config.ha });
|
|
200
194
|
|
|
201
195
|
// Operator-IP access: detect + persist + (when env already deployed) patch
|
|
202
196
|
// the live Hetzner firewall. On a fresh first deploy this just appends to
|
|
@@ -206,7 +200,7 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
206
200
|
const accessResult = await ensureOperatorIpAccess({
|
|
207
201
|
projectConfig,
|
|
208
202
|
environment,
|
|
209
|
-
isHA,
|
|
203
|
+
isHA: isHATier(tier),
|
|
210
204
|
apiToken,
|
|
211
205
|
yes: !!args.yes,
|
|
212
206
|
onMessage: (msg) => p.log.info(msg),
|
|
@@ -505,8 +499,7 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
505
499
|
const imageResolveTimer = perfTimer('deploy.image.resolve');
|
|
506
500
|
const buildMode = resolveBuildMode(args, process.cwd(), deployMode);
|
|
507
501
|
const isDirectDeploy = buildMode === 'direct';
|
|
508
|
-
const
|
|
509
|
-
const isComposeLocal = buildMode === 'local' && isComposeMode;
|
|
502
|
+
const isComposeLocal = buildMode === 'local' && isComposeTier(tier);
|
|
510
503
|
let imageReadyPromise;
|
|
511
504
|
// For compose-local: kicked off here in parallel with iac.upStack below.
|
|
512
505
|
// The build is awaited later (just before sideload). Other modes leave
|
|
@@ -620,7 +613,7 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
620
613
|
// DNS-propagation poll or the Traefik cert self-heal below. `manual` keeps
|
|
621
614
|
// HTTP-01. Scoped to compose: k8s issues certs via cert-manager and ignores
|
|
622
615
|
// this Traefik override entirely.
|
|
623
|
-
const dnsChallenge =
|
|
616
|
+
const dnsChallenge = isComposeTier(tier) && useDnsChallenge(dnsProvider);
|
|
624
617
|
const bundlePath = renderBundle(projectConfig.projectName, {
|
|
625
618
|
domain,
|
|
626
619
|
image: imageRef,
|
|
@@ -670,9 +663,79 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
670
663
|
: Promise.resolve(null);
|
|
671
664
|
|
|
672
665
|
// --- STEP 5: Architecture Dispatch ---
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
666
|
+
// One entry per tier, keyed by the registry. deploymentConfig is a cheap,
|
|
667
|
+
// pure object literal built unconditionally above the table; its k8s-only
|
|
668
|
+
// fields are simply unused on the compose paths.
|
|
669
|
+
const deploymentConfig = {
|
|
670
|
+
projectName: projectConfig.projectName,
|
|
671
|
+
environment: config.environment,
|
|
672
|
+
provider: config.provider,
|
|
673
|
+
region,
|
|
674
|
+
secondaryRegion,
|
|
675
|
+
masterServerType,
|
|
676
|
+
supabaseServerType,
|
|
677
|
+
workerServerType,
|
|
678
|
+
serverType,
|
|
679
|
+
minWorkers,
|
|
680
|
+
maxWorkers,
|
|
681
|
+
domain,
|
|
682
|
+
ha: config.ha,
|
|
683
|
+
observability: config.observability,
|
|
684
|
+
services,
|
|
685
|
+
apiToken,
|
|
686
|
+
cloudflareApiToken,
|
|
687
|
+
cloudflareZoneId,
|
|
688
|
+
hetznerDnsZoneId,
|
|
689
|
+
dnsProvider,
|
|
690
|
+
s3Config,
|
|
691
|
+
backupConfig,
|
|
692
|
+
backupBucketName: backupS3Config?.bucket || null,
|
|
693
|
+
// DR: when set (`deploy -restore latest|<ts>`), the db pod's init
|
|
694
|
+
// container seeds PGDATA from S3 via wal-g and applyMigrations is
|
|
695
|
+
// skipped (the restored DB is authoritative).
|
|
696
|
+
restore: args.restore || null,
|
|
697
|
+
// Finding #1: hard-gate k8s-HA replication unless the operator opts into a
|
|
698
|
+
// warm/degraded standby via `deploy -allow-degraded`.
|
|
699
|
+
allowDegraded: !!args.allowDegraded,
|
|
700
|
+
operatorCidrs: projectConfig.operatorCidrs ?? [],
|
|
701
|
+
imageReadyPromise,
|
|
702
|
+
tracker,
|
|
703
|
+
state,
|
|
704
|
+
};
|
|
705
|
+
|
|
706
|
+
const deployCtx = {
|
|
707
|
+
projectConfig,
|
|
708
|
+
environment,
|
|
709
|
+
envConfig,
|
|
710
|
+
region,
|
|
711
|
+
serverType,
|
|
712
|
+
services,
|
|
713
|
+
domain,
|
|
714
|
+
dnsProvider,
|
|
715
|
+
apiToken,
|
|
716
|
+
cloudflareApiToken,
|
|
717
|
+
cloudflareZoneId,
|
|
718
|
+
hetznerDnsZoneId,
|
|
719
|
+
backupConfig,
|
|
720
|
+
imageRef,
|
|
721
|
+
bundlePath,
|
|
722
|
+
ciReady,
|
|
723
|
+
s3Config,
|
|
724
|
+
state,
|
|
725
|
+
dnsChallenge,
|
|
726
|
+
dnsWarmupPromise,
|
|
727
|
+
isDirectDeploy,
|
|
728
|
+
isComposeLocal,
|
|
729
|
+
localImageTag,
|
|
730
|
+
composeLocalBuildPromise,
|
|
731
|
+
};
|
|
732
|
+
|
|
733
|
+
const TIER_DEPLOYERS = {
|
|
734
|
+
compose: async () => {
|
|
735
|
+
const { deployComposeSingle } = await import('./compose/single.js');
|
|
736
|
+
return deployComposeSingle(deployCtx);
|
|
737
|
+
},
|
|
738
|
+
'compose-ha': async () => {
|
|
676
739
|
const { deployComposeHA } = await import('./compose/ha.js');
|
|
677
740
|
const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}`);
|
|
678
741
|
const s = tracker.spinner();
|
|
@@ -680,7 +743,7 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
680
743
|
// Wrap the entire compose-HA pipeline so the perf trace shows the
|
|
681
744
|
// headline cost (which contains compose-ha.deploy.* sub-stages from
|
|
682
745
|
// compose/ha.js).
|
|
683
|
-
|
|
746
|
+
const result = await perfAsync('deploy.ha.compose.full', () =>
|
|
684
747
|
deployComposeHA({
|
|
685
748
|
projectName: projectConfig.projectName,
|
|
686
749
|
environment,
|
|
@@ -711,433 +774,22 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
711
774
|
}),
|
|
712
775
|
);
|
|
713
776
|
s.stop('Compose HA deployment complete');
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
setupComposeBackupCron,
|
|
722
|
-
} = await import('./compose/index.js');
|
|
723
|
-
const { setupServerFiles, waitForSSH: waitForComposeSSH } = await import(
|
|
724
|
-
'./compose/index.js'
|
|
725
|
-
);
|
|
726
|
-
const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}`);
|
|
727
|
-
|
|
728
|
-
let serverIp = envConfig.servers?.[0]?.ip || null;
|
|
729
|
-
// Track the Hetzner server's API id + literal name so destroy can find
|
|
730
|
-
// the VPS without falling back to the literal string "master" (which
|
|
731
|
-
// never matches the Pulumi-assigned name `${projectName}-${env}`).
|
|
732
|
-
// Without this, destroy → re-deploy hits 'server name is already used
|
|
733
|
-
// (uniqueness_error)' because the previous VPS was never deleted
|
|
734
|
-
// (caught in compose restore matrix runs).
|
|
735
|
-
let serverHetznerId = envConfig.servers?.[0]?.id || null;
|
|
736
|
-
let serverHetznerName = envConfig.servers?.[0]?.hetznerName || null;
|
|
737
|
-
if (!serverIp) {
|
|
738
|
-
generateSSHKeyPair(sshKeyPath);
|
|
739
|
-
const { upStack } = await import('../iac/index.js');
|
|
740
|
-
const { buildHetznerComposeProgram } = await import('../iac/programs/hetzner-compose.js');
|
|
741
|
-
const program = buildHetznerComposeProgram({
|
|
742
|
-
projectName: projectConfig.projectName,
|
|
743
|
-
environment,
|
|
744
|
-
sshPublicKey: readFileSync(`${sshKeyPath}.pub`, 'utf-8').trim(),
|
|
745
|
-
location: region,
|
|
746
|
-
serverType: serverType || 'cx23',
|
|
747
|
-
labels: { 'managed-by': 'vibecarbon' },
|
|
748
|
-
allowedSshIps: (projectConfig.operatorCidrs ?? []).map((e) => e.cidr),
|
|
749
|
-
});
|
|
750
|
-
// VM provisioning via Pulumi — typical 30-90s on Hetzner cx23.
|
|
751
|
-
const result = await perfAsync('deploy.iac.upStack', () =>
|
|
752
|
-
upStack(environment, program, { hcloudToken: apiToken, s3Config }),
|
|
753
|
-
);
|
|
754
|
-
serverIp = result.outputs.serverIp;
|
|
755
|
-
serverHetznerId = result.outputs.serverId || null;
|
|
756
|
-
serverHetznerName = `${projectConfig.projectName}-${environment}`;
|
|
757
|
-
await perfAsync('deploy.iac.waitForSSH', () => waitForComposeSSH(serverIp, sshKeyPath, 30));
|
|
758
|
-
// Trusted host-key seed on fresh provision — captures the new VPS's
|
|
759
|
-
// host key via ssh-keyscan into the per-env known_hosts so the pinned
|
|
760
|
-
// SSH opts (accept-new) become a strict pin for this deploy and every
|
|
761
|
-
// later command. Re-seeding here also cleanly re-pins a Hetzner-
|
|
762
|
-
// recycled IP (destroy → redeploy). Best-effort: falls back to
|
|
763
|
-
// accept-new TOFU if the scan can't reach the server yet. See
|
|
764
|
-
// host-keys.js seedKnownHosts for the full security rationale.
|
|
765
|
-
await seedKnownHosts(knownHostsPathForKey(sshKeyPath), serverIp);
|
|
766
|
-
}
|
|
767
|
-
|
|
768
|
-
// --- Warm path (StateTracker) ---
|
|
769
|
-
// On a second deploy against an already-provisioned server, skip the
|
|
770
|
-
// idempotent-but-slow steps (cloud-init probe, GHCR login, bundle rsync)
|
|
771
|
-
// when their inputs haven't changed. `docker compose up -d` is cheap
|
|
772
|
-
// and always runs — it applies diffs to the live stack.
|
|
773
|
-
const setupInputs = { serverIp };
|
|
774
|
-
if (!state.shouldSkip('compose-setup-server', setupInputs)) {
|
|
775
|
-
state.startStep('compose-setup-server', setupInputs);
|
|
776
|
-
// Cloud-init readiness probe — first-deploy spends 20-40s here while
|
|
777
|
-
// ufw + unattended-upgrades land. Warm deploys hit the ready marker
|
|
778
|
-
// immediately, so this is mostly cold-deploy noise.
|
|
779
|
-
await perfAsync('deploy.compose.cloudInitReady', async () => {
|
|
780
|
-
if (isDirectDeploy || isComposeLocal) {
|
|
781
|
-
// Both direct (build over DOCKER_HOST=ssh://) and local
|
|
782
|
-
// (sideload via docker save | ssh docker load) need dockerd
|
|
783
|
-
// up on the server before they can proceed. Push mode doesn't
|
|
784
|
-
// because the server pulls from GHCR after compose-up starts.
|
|
785
|
-
const { waitForDockerReady } = await import('./compose/index.js');
|
|
786
|
-
await setupServer(serverIp, sshKeyPath);
|
|
787
|
-
await waitForDockerReady(serverIp, sshKeyPath);
|
|
788
|
-
} else {
|
|
789
|
-
await setupServer(serverIp, sshKeyPath);
|
|
790
|
-
}
|
|
791
|
-
});
|
|
792
|
-
state.completeStep('compose-setup-server', { serverIp });
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
// --- Image transfer to server (mode-dependent) ---
|
|
796
|
-
if (isComposeLocal) {
|
|
797
|
-
// Local-build path: the build was kicked off in parallel with
|
|
798
|
-
// iac.upStack at the top of this function. By the time we reach
|
|
799
|
-
// here (post-cloudInitReady), the build has almost always finished.
|
|
800
|
-
// Await the promise (cheap if already settled) then sideload the
|
|
801
|
-
// image tarball over SSH. Sideload uses docker save | gzip | ssh
|
|
802
|
-
// | docker load — gzip drops ~600MB → ~150-200MB on the wire.
|
|
803
|
-
//
|
|
804
|
-
// Both run silently (build is silent:true; sideload streams no stdout),
|
|
805
|
-
// so without a spinner this is a 15-30s dead pause right after "Proceed"
|
|
806
|
-
// on warm redeploys (where the upStack spinner that normally overlaps
|
|
807
|
-
// the build was skipped). Drive a spinner so the operator sees progress.
|
|
808
|
-
const imageSpinner = p.spinner();
|
|
809
|
-
imageSpinner.start('Building app image');
|
|
810
|
-
await composeLocalBuildPromise;
|
|
811
|
-
imageSpinner.message('Sideloading app image to the server');
|
|
812
|
-
await perfAsync('deploy.image.sideload', async () =>
|
|
813
|
-
sideloadCompose({
|
|
814
|
-
tag: localImageTag,
|
|
815
|
-
sshTarget: `root@${serverIp}`,
|
|
816
|
-
sshKey: sshKeyPath,
|
|
817
|
-
}),
|
|
818
|
-
);
|
|
819
|
-
imageSpinner.stop('App image built and sideloaded');
|
|
820
|
-
} else if (isDirectDeploy) {
|
|
821
|
-
// Direct path: build the image on the server via DOCKER_HOST=ssh://.
|
|
822
|
-
// BuildKit caches aggressively, so unchanged source rebuilds in 1-3s.
|
|
823
|
-
// Slower than local+sideload (no upStack overlap) but works when the
|
|
824
|
-
// operator has no local Docker — selected by --direct or by
|
|
825
|
-
// resolveBuildMode's docker-not-found fallback. VITE_* args plumbed
|
|
826
|
-
// through here for the same reason as the local-build path above.
|
|
827
|
-
const { collectComposeBuildArgs } = await import('./compose/build-args.js');
|
|
828
|
-
const directBuildArgs = collectComposeBuildArgs(process.cwd(), {
|
|
829
|
-
projectName: projectConfig.projectName,
|
|
830
|
-
domain,
|
|
831
|
-
});
|
|
832
|
-
const success = await perfAsync('deploy.image.directBuild', () =>
|
|
833
|
-
buildRemote(serverIp, sshKeyPath, localImageTag, process.cwd(), directBuildArgs),
|
|
834
|
-
);
|
|
835
|
-
if (!success) process.exit(1);
|
|
836
|
-
}
|
|
837
|
-
|
|
838
|
-
// --- Docker Hub login (skip if creds unchanged) ---
|
|
839
|
-
// Without this, the reconcile script's `docker compose pull` for
|
|
840
|
-
// Supabase service images (rest, auth, storage, kong, imgproxy, etc.)
|
|
841
|
-
// hits Docker Hub anonymously. A fresh Hetzner VPS shares a NAT range
|
|
842
|
-
// with many other deploys, so the per-IP unauthenticated pull quota
|
|
843
|
-
// is routinely exhausted — observed 2026-04-26 matrix run #3
|
|
844
|
-
// ("toomanyrequests: You have reached your unauthenticated pull rate
|
|
845
|
-
// limit"). Compose-HA + scale already log in; this brings the
|
|
846
|
-
// single-compose deploy path in line.
|
|
847
|
-
const dockerHubCreds = loadCredentials().dockerHub || null;
|
|
848
|
-
if (dockerHubCreds) {
|
|
849
|
-
const dhLoginInputs = {
|
|
850
|
-
serverIp,
|
|
851
|
-
registry: 'docker.io',
|
|
852
|
-
user: dockerHubCreds.username,
|
|
853
|
-
tokenFp: dockerHubCreds.token ? dockerHubCreds.token.slice(0, 8) : '',
|
|
854
|
-
};
|
|
855
|
-
if (!state.shouldSkip('compose-dockerhub-login', dhLoginInputs)) {
|
|
856
|
-
state.startStep('compose-dockerhub-login', dhLoginInputs);
|
|
857
|
-
await dockerLoginOnServer(serverIp, sshKeyPath, dockerHubCreds);
|
|
858
|
-
state.completeStep('compose-dockerhub-login', { serverIp });
|
|
859
|
-
}
|
|
860
|
-
}
|
|
861
|
-
|
|
862
|
-
// --- GHCR login (skip if creds unchanged) ---
|
|
863
|
-
if (ciReady.ghcrPullCreds) {
|
|
864
|
-
const loginInputs = {
|
|
865
|
-
serverIp,
|
|
866
|
-
registry: 'ghcr.io',
|
|
867
|
-
user: ciReady.ghcrPullCreds.owner,
|
|
868
|
-
// Hash the token's fingerprint, not the token itself — we don't
|
|
869
|
-
// want the token written to .vibecarbon/deploy-state-*.json.
|
|
870
|
-
tokenFp: ciReady.ghcrPullCreds.token ? ciReady.ghcrPullCreds.token.slice(0, 8) : '',
|
|
871
|
-
};
|
|
872
|
-
if (!state.shouldSkip('compose-ghcr-login', loginInputs)) {
|
|
873
|
-
state.startStep('compose-ghcr-login', loginInputs);
|
|
874
|
-
await dockerLoginOnServer(serverIp, sshKeyPath, {
|
|
875
|
-
username: ciReady.ghcrPullCreds.owner,
|
|
876
|
-
token: ciReady.ghcrPullCreds.token,
|
|
877
|
-
registry: 'ghcr.io',
|
|
878
|
-
});
|
|
879
|
-
state.completeStep('compose-ghcr-login', { serverIp });
|
|
880
|
-
}
|
|
881
|
-
}
|
|
882
|
-
|
|
883
|
-
// --- Bundle rsync (skip when bundle inputs unchanged) ---
|
|
884
|
-
const bundleInputs = {
|
|
885
|
-
serverIp,
|
|
886
|
-
projectName: projectConfig.projectName,
|
|
887
|
-
imageRef,
|
|
888
|
-
domain,
|
|
889
|
-
services,
|
|
890
|
-
};
|
|
891
|
-
if (!state.shouldSkip('compose-setup-files', bundleInputs)) {
|
|
892
|
-
state.startStep('compose-setup-files', bundleInputs);
|
|
893
|
-
// setupServerFiles tars the bundle locally + streams over ssh +
|
|
894
|
-
// extracts on the server in one pipeline. Bucket the whole upload
|
|
895
|
-
// here; the inner tar/upload split is captured in compose/index.js.
|
|
896
|
-
await perfAsync('deploy.bundle.upload', () =>
|
|
897
|
-
setupServerFiles(serverIp, sshKeyPath, projectConfig.projectName, {
|
|
898
|
-
...services,
|
|
899
|
-
domain,
|
|
900
|
-
image: imageRef,
|
|
901
|
-
bundlePath,
|
|
902
|
-
}),
|
|
903
|
-
);
|
|
904
|
-
state.completeStep('compose-setup-files', { serverIp });
|
|
905
|
-
}
|
|
906
|
-
|
|
907
|
-
// --- DNS update + propagation wait BEFORE compose-up ---
|
|
908
|
-
//
|
|
909
|
-
// Traefik attempts ACME HTTP-01 challenges immediately on container
|
|
910
|
-
// start and bursts ~7 attempts in ~30s before giving up. If LE's
|
|
911
|
-
// DNS resolver sees stale or absent records on those attempts,
|
|
912
|
-
// every challenge fails with "no valid A records found" and
|
|
913
|
-
// acme.json ends the deploy with zero certs issued — browser
|
|
914
|
-
// sees TRAEFIK DEFAULT CERT and NET::ERR_CERT_AUTHORITY_INVALID.
|
|
915
|
-
//
|
|
916
|
-
// The warm-up that fires in Step 4 writes a 0.0.0.0 placeholder;
|
|
917
|
-
// LE specifically rejects 0.0.0.0 as not-a-valid-A-record. So even
|
|
918
|
-
// when warm-up "succeeds," it doesn't help ACME — it actively
|
|
919
|
-
// poisons the resolver state until the real IP overwrites it.
|
|
920
|
-
//
|
|
921
|
-
// Fix (RCA from vibecarbon.com cold-deploy 2026-05-19): swap the
|
|
922
|
-
// ordering. Write the real IP now, wait for it to propagate to
|
|
923
|
-
// public resolvers, then start compose. Traefik's first ACME
|
|
924
|
-
// attempt sees real DNS, succeeds on the first try.
|
|
925
|
-
//
|
|
926
|
-
// The compose-HA path already writes DNS before compose-up at
|
|
927
|
-
// src/lib/deploy/compose/ha.js — single-region compose was the
|
|
928
|
-
// outlier that ran startComposeStack first then updated DNS.
|
|
929
|
-
if (domain && dnsProvider && dnsProvider !== 'manual') {
|
|
930
|
-
await perfAsync('deploy.dns.warm', async () => {
|
|
931
|
-
await dnsWarmupPromise;
|
|
932
|
-
});
|
|
933
|
-
const dnsUpdateInputs = { domain, dnsProvider, serverIp };
|
|
934
|
-
if (!state.shouldSkip('compose-dns-update', dnsUpdateInputs)) {
|
|
935
|
-
state.startStep('compose-dns-update', dnsUpdateInputs);
|
|
936
|
-
const { setupSimple: updateDnsIp } =
|
|
937
|
-
dnsProvider === 'cloudflare'
|
|
938
|
-
? await import('../cloudflare.js')
|
|
939
|
-
: await import('../hetzner-dns.js');
|
|
940
|
-
const token = dnsProvider === 'cloudflare' ? cloudflareApiToken : apiToken;
|
|
941
|
-
const zoneId = dnsProvider === 'cloudflare' ? cloudflareZoneId : hetznerDnsZoneId;
|
|
942
|
-
const dnsSpinner = p.spinner();
|
|
943
|
-
dnsSpinner.start(`Pointing ${domain} → ${serverIp}...`);
|
|
944
|
-
await perfAsync('deploy.dns.create', () => updateDnsIp(token, zoneId, domain, serverIp));
|
|
945
|
-
dnsSpinner.stop(`DNS A record set: ${domain} → ${serverIp}`);
|
|
946
|
-
state.completeStep('compose-dns-update', { serverIp });
|
|
947
|
-
}
|
|
948
|
-
// HTTP-01 only: block compose-up until public resolvers see the real
|
|
949
|
-
// IP. 120s budget covers Cloudflare's typical edge propagation (under
|
|
950
|
-
// 60s) with margin. Returns false on timeout; we proceed anyway —
|
|
951
|
-
// Traefik can still acquire certs on its own retries, the customer
|
|
952
|
-
// just sees a brief self-signed-cert window. Hard-failing here would
|
|
953
|
-
// mask deploys that succeed eventually.
|
|
954
|
-
//
|
|
955
|
-
// DNS-01 (managed providers) skips this entirely: lego validates via a
|
|
956
|
-
// TXT record it writes through the provider API, so cert issuance does
|
|
957
|
-
// not wait on the A record propagating. The A record we just wrote
|
|
958
|
-
// propagates in the background; the post-deploy health probe uses
|
|
959
|
-
// localhost + Host header and never depends on it.
|
|
960
|
-
//
|
|
961
|
-
// Visible spinner so the operator isn't staring at a frozen
|
|
962
|
-
// "Yes" screen for up to 120s on a cold deploy where the DNS
|
|
963
|
-
// edge is slow. `timer` indicator forces the rendered string to
|
|
964
|
-
// change every second so progress remains visibly alive.
|
|
965
|
-
if (!dnsChallenge) {
|
|
966
|
-
const dnsSpinner = p.spinner({ indicator: 'timer' });
|
|
967
|
-
dnsSpinner.start(`Waiting for ${domain} to resolve to ${serverIp}`);
|
|
968
|
-
const dnsPropagated = await perfAsync('deploy.dns.waitForPropagation', () =>
|
|
969
|
-
waitForDNSPropagation(domain, serverIp, 120_000),
|
|
970
|
-
);
|
|
971
|
-
dnsSpinner.stop(
|
|
972
|
-
dnsPropagated
|
|
973
|
-
? `DNS resolves: ${domain} → ${serverIp}`
|
|
974
|
-
: `DNS propagation timed out — proceeding anyway`,
|
|
975
|
-
);
|
|
976
|
-
}
|
|
977
|
-
}
|
|
978
|
-
|
|
979
|
-
// `docker compose up -d` is idempotent and cheap — always run so a
|
|
980
|
-
// config tweak that didn't change the bundle hash (e.g. env var)
|
|
981
|
-
// still takes effect. Reconcile.sh on the server runs `docker compose
|
|
982
|
-
// pull` (when not local image) + `docker compose up -d` — see
|
|
983
|
-
// compose/index.js for the inner perf instrumentation.
|
|
984
|
-
await perfAsync('deploy.reconcile.run', () =>
|
|
985
|
-
startComposeStack(serverIp, sshKeyPath, projectConfig.projectName, services),
|
|
986
|
-
);
|
|
987
|
-
|
|
988
|
-
// Apply app migrations + reload PostgREST. The orchestrator's inline
|
|
989
|
-
// single-compose path previously skipped this (only deployComposeHA ran
|
|
990
|
-
// runMigrations), so compose-single shipped an EMPTY app schema — 0 public
|
|
991
|
-
// tables, every DB-backed feature 500'd, and db_schema verify failed with
|
|
992
|
-
// PGRST205. runMigrations waits for supabase_admin, applies each
|
|
993
|
-
// supabase/migrations/* with ON_ERROR_STOP=1 (a real failure aborts the
|
|
994
|
-
// deploy rather than silently shipping a schema-less prod), then reloads
|
|
995
|
-
// PostgREST's schema cache. Mirrors the HA path.
|
|
996
|
-
await perfAsync('deploy.compose.migrations', () =>
|
|
997
|
-
runMigrations(serverIp, sshKeyPath, projectConfig.projectName),
|
|
998
|
-
);
|
|
999
|
-
|
|
1000
|
-
// Create the production app super-admin in auth.users via GoTrue's admin
|
|
1001
|
-
// API. Same class of trap as runMigrations above: this inline single-
|
|
1002
|
-
// compose path previously skipped it (only deployComposeHA called
|
|
1003
|
-
// createAdminUser), so a single-server deploy shipped a prod app the
|
|
1004
|
-
// operator couldn't actually log into — the only admin user lived in
|
|
1005
|
-
// their local Docker. Idempotent (422 = already exists); a failure here
|
|
1006
|
-
// is a warning, not a hard abort, so a transient GoTrue blip doesn't
|
|
1007
|
-
// fail an otherwise-healthy deploy — re-running `vibecarbon deploy`
|
|
1008
|
-
// retries it.
|
|
1009
|
-
const adminResult = await perfAsync('deploy.compose.createAdminUser', () =>
|
|
1010
|
-
createAdminUser(serverIp, sshKeyPath, projectConfig.projectName),
|
|
1011
|
-
);
|
|
1012
|
-
if (adminResult.success) {
|
|
1013
|
-
p.log.success(adminResult.message);
|
|
1014
|
-
} else {
|
|
1015
|
-
p.log.warn(`${adminResult.message}. Run \`vibecarbon deploy\` again to retry.`);
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
// --- Verifiable success gate ---
|
|
1019
|
-
// Probe the app's own /api/health endpoint on the server itself (via
|
|
1020
|
-
// localhost + Host header), bypassing DNS/TLS. A successful deploy
|
|
1021
|
-
// must yield an HTTP 2xx here; otherwise fail the deploy loudly with
|
|
1022
|
-
// container state + log tail.
|
|
1023
|
-
//
|
|
1024
|
-
// verifyAppHealth polls every 10s for up to 3 minutes while the app
|
|
1025
|
-
// container restarts and binds port 3000. Without a visible spinner
|
|
1026
|
-
// the operator just sees a frozen "Stack reconciled" line for that
|
|
1027
|
-
// entire window — observed 2026-05-19 vibecarbon.com re-deploy
|
|
1028
|
-
// where compose-up finished in ~5s but the app took ~3min to come
|
|
1029
|
-
// back, leaving the screen looking hung. `timer` indicator so the
|
|
1030
|
-
// elapsed-seconds counter ticks visibly even on terminals where
|
|
1031
|
-
// the spinner frame-character animation gets throttled under load.
|
|
1032
|
-
const { verifyAppHealth } = await import('./compose/index.js');
|
|
1033
|
-
const healthSpinner = p.spinner({ indicator: 'timer' });
|
|
1034
|
-
healthSpinner.start('Waiting for app to start serving requests');
|
|
1035
|
-
let health;
|
|
1036
|
-
try {
|
|
1037
|
-
health = await perfAsync('deploy.health.probe', () =>
|
|
1038
|
-
verifyAppHealth(serverIp, sshKeyPath, projectConfig.projectName, {
|
|
1039
|
-
domain,
|
|
1040
|
-
}),
|
|
1041
|
-
);
|
|
1042
|
-
} catch (err) {
|
|
1043
|
-
healthSpinner.stop('App health probe errored', 1);
|
|
1044
|
-
throw err;
|
|
1045
|
-
}
|
|
1046
|
-
if (!health.healthy) {
|
|
1047
|
-
healthSpinner.stop(`App not serving requests (status: ${health.status})`, 1);
|
|
1048
|
-
p.log.error(
|
|
1049
|
-
`Deploy produced an unhealthy app (probe status: ${health.status}):\n${health.details}`,
|
|
1050
|
-
);
|
|
1051
|
-
process.exit(1);
|
|
1052
|
-
}
|
|
1053
|
-
healthSpinner.stop(`App is serving requests (HTTP ${health.status})`);
|
|
1054
|
-
|
|
1055
|
-
// Install the scheduled wal-g backup cron on the server. Without this,
|
|
1056
|
-
// a fresh compose deploy collects + persists backupConfig (envConfig.backup)
|
|
1057
|
-
// but never schedules a backup — backups only ran after a `vibecarbon scale`
|
|
1058
|
-
// event (scale.js was the lone caller). setupComposeBackupCron defaults the
|
|
1059
|
-
// schedule + retain when backupConfig is absent, so it always installs a
|
|
1060
|
-
// sensible cron. wal-g reads its S3 config from the db container env
|
|
1061
|
-
// (rendered into .env at deploy) — the cron just runs compose-backup.sh,
|
|
1062
|
-
// which self-skips (exit 0) when no S3 backup target is configured, so an
|
|
1063
|
-
// always-installed cron never hard-fails on a no-S3 deploy.
|
|
1064
|
-
//
|
|
1065
|
-
// A cron-install failure must NOT fail an already-healthy deploy: the app
|
|
1066
|
-
// is serving by this point. Downgrade to a warning so a transient ssh/cron
|
|
1067
|
-
// hiccup doesn't roll back a successful deploy.
|
|
1068
|
-
try {
|
|
1069
|
-
await perfAsync('deploy.compose.backupCron', async () =>
|
|
1070
|
-
setupComposeBackupCron(serverIp, sshKeyPath, projectConfig.projectName, backupConfig),
|
|
1071
|
-
);
|
|
1072
|
-
} catch (err) {
|
|
1073
|
-
p.log.warn(`Scheduled backup cron install failed (deploy still succeeded): ${err.message}`);
|
|
1074
|
-
}
|
|
1075
|
-
|
|
1076
|
-
// No ACME self-heal needed here. Managed-DNS deploys (cloudflare/hetzner)
|
|
1077
|
-
// issue certs via DNS-01, which has no HTTP-01-vs-DNS-propagation race to
|
|
1078
|
-
// recover from; `manual` DNS was never self-healed (we don't manage its
|
|
1079
|
-
// records). The old acme.json poll + Traefik restart (acme-verify.js)
|
|
1080
|
-
// existed only for the HTTP-01 race the DNS-01 switch eliminates.
|
|
1081
|
-
|
|
1082
|
-
deployResult = {
|
|
1083
|
-
masterIp: serverIp,
|
|
1084
|
-
serverId: serverHetznerId || 'manual',
|
|
1085
|
-
serverName: serverHetznerName,
|
|
1086
|
-
};
|
|
1087
|
-
}
|
|
1088
|
-
} else {
|
|
1089
|
-
// Kubernetes (k3s on Ubuntu)
|
|
1090
|
-
const { deployK3s } = await import('./k8s/k3s.js');
|
|
1091
|
-
const { deployK8sHA } = await import('./k8s/ha/index.js');
|
|
1092
|
-
|
|
1093
|
-
const deploymentConfig = {
|
|
1094
|
-
projectName: projectConfig.projectName,
|
|
1095
|
-
environment: config.environment,
|
|
1096
|
-
provider: config.provider,
|
|
1097
|
-
region,
|
|
1098
|
-
secondaryRegion,
|
|
1099
|
-
masterServerType,
|
|
1100
|
-
supabaseServerType,
|
|
1101
|
-
workerServerType,
|
|
1102
|
-
serverType,
|
|
1103
|
-
minWorkers,
|
|
1104
|
-
maxWorkers,
|
|
1105
|
-
domain,
|
|
1106
|
-
ha: config.ha,
|
|
1107
|
-
observability: config.observability,
|
|
1108
|
-
services,
|
|
1109
|
-
apiToken,
|
|
1110
|
-
cloudflareApiToken,
|
|
1111
|
-
cloudflareZoneId,
|
|
1112
|
-
hetznerDnsZoneId,
|
|
1113
|
-
dnsProvider,
|
|
1114
|
-
s3Config,
|
|
1115
|
-
backupConfig,
|
|
1116
|
-
backupBucketName: backupS3Config?.bucket || null,
|
|
1117
|
-
// DR: when set (`deploy -restore latest|<ts>`), the db pod's init
|
|
1118
|
-
// container seeds PGDATA from S3 via wal-g and applyMigrations is
|
|
1119
|
-
// skipped (the restored DB is authoritative).
|
|
1120
|
-
restore: args.restore || null,
|
|
1121
|
-
// Finding #1: hard-gate k8s-HA replication unless the operator opts into a
|
|
1122
|
-
// warm/degraded standby via `deploy -allow-degraded`.
|
|
1123
|
-
allowDegraded: !!args.allowDegraded,
|
|
1124
|
-
operatorCidrs: projectConfig.operatorCidrs ?? [],
|
|
1125
|
-
imageReadyPromise,
|
|
1126
|
-
tracker,
|
|
1127
|
-
state,
|
|
1128
|
-
};
|
|
1129
|
-
|
|
1130
|
-
if (config.ha) {
|
|
777
|
+
return result;
|
|
778
|
+
},
|
|
779
|
+
k8s: async () => {
|
|
780
|
+
const { deployK3s } = await import('./k8s/k3s.js');
|
|
781
|
+
return perfAsync('deploy.k3s.full', () => deployK3s(deploymentConfig));
|
|
782
|
+
},
|
|
783
|
+
'k8s-ha': async () => {
|
|
1131
784
|
// K8s HA = 2 clusters (primary + standby) provisioned in parallel,
|
|
1132
785
|
// each running k3s + Helm + sideload. Cold HA observed ~25-40 min.
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
786
|
+
const { deployK8sHA } = await import('./k8s/ha/index.js');
|
|
787
|
+
return perfAsync('deploy.ha.k8s.full', () => deployK8sHA(deploymentConfig));
|
|
788
|
+
},
|
|
789
|
+
};
|
|
790
|
+
const deployResult = await TIER_DEPLOYERS[tier]();
|
|
1137
791
|
|
|
1138
|
-
|
|
1139
|
-
// to every node, and applied k8s manifests inline. Layer in Flux + GHA
|
|
1140
|
-
// via `vibecarbon configure cicd <env>` (PR 7) after the cluster is up.
|
|
792
|
+
if (isK8sTier(tier)) {
|
|
1141
793
|
{
|
|
1142
794
|
const kubeconfigPath = join(process.cwd(), '.vibecarbon', `kubeconfig-${environment}`);
|
|
1143
795
|
const masterIp = deployResult.primary?.masterIp || deployResult.masterIp;
|
|
@@ -1246,15 +898,14 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
1246
898
|
// root cause without requiring a follow-up `vibecarbon diagnose`.
|
|
1247
899
|
// The cluster is about to be destroyed by the test runner; once
|
|
1248
900
|
// it's gone, kubectl probes return nothing. We spawn each tool via
|
|
1249
|
-
//
|
|
901
|
+
// runCommandAsync (array argv, no shell) — domain and environment come
|
|
1250
902
|
// from validated config, but we still avoid shell-string interp.
|
|
1251
903
|
try {
|
|
1252
|
-
const { execFileSync } = await import('node:child_process');
|
|
1253
904
|
// HA deploys split into kubeconfig-<env>-primary / -standby; standalone
|
|
1254
905
|
// uses just kubeconfig-<env>. Prefer primary (the user-facing cluster
|
|
1255
906
|
// — Traefik/cert-manager state we want to inspect lives there) and
|
|
1256
907
|
// fall back to the standalone path. fs.existsSync check would race
|
|
1257
|
-
// against the FS, so prefer the file-not-found error in
|
|
908
|
+
// against the FS, so prefer the file-not-found error in runCommandAsync.
|
|
1258
909
|
const { existsSync } = await import('node:fs');
|
|
1259
910
|
const kubeconfigCandidates = [
|
|
1260
911
|
join(process.cwd(), '.vibecarbon', `kubeconfig-${environment}-primary`),
|
|
@@ -1262,18 +913,18 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
1262
913
|
];
|
|
1263
914
|
const kubeconfig =
|
|
1264
915
|
kubeconfigCandidates.find((c) => existsSync(c)) ?? kubeconfigCandidates[1];
|
|
1265
|
-
// Run each tool via
|
|
916
|
+
// Run each tool via runCommandAsync. Capture stderr alongside stdout
|
|
1266
917
|
// (kubectl prints errors to stderr; without merging we get the
|
|
1267
918
|
// useless first line of err.message — observed 2026-04-26 run #5
|
|
1268
919
|
// where every kubectl probe surfaced as "(kubectl failed: Command
|
|
1269
920
|
// failed: kubectl ...)" with no actual diagnostic).
|
|
1270
|
-
const safeRun = (cmd, argv) => {
|
|
921
|
+
const safeRun = async (cmd, argv) => {
|
|
1271
922
|
try {
|
|
1272
|
-
|
|
1273
|
-
|
|
923
|
+
const output = await runCommandAsync([cmd, ...argv], {
|
|
924
|
+
silent: true,
|
|
1274
925
|
timeout: 30_000,
|
|
1275
|
-
|
|
1276
|
-
|
|
926
|
+
});
|
|
927
|
+
return output.trim();
|
|
1277
928
|
} catch (err) {
|
|
1278
929
|
const stdout = err?.stdout?.toString?.()?.trim() ?? '';
|
|
1279
930
|
const stderr = err?.stderr?.toString?.()?.trim() ?? '';
|
|
@@ -1281,10 +932,10 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
1281
932
|
return `(${cmd} failed exit=${err?.status ?? '?'}):\n${tail || (err instanceof Error ? err.message : String(err))}`;
|
|
1282
933
|
}
|
|
1283
934
|
};
|
|
1284
|
-
const dig = safeRun('dig', ['+short', domain, '@1.1.1.1']);
|
|
935
|
+
const dig = await safeRun('dig', ['+short', domain, '@1.1.1.1']);
|
|
1285
936
|
const kcExists = existsSync(kubeconfig);
|
|
1286
937
|
const certs = kcExists
|
|
1287
|
-
? safeRun('kubectl', [
|
|
938
|
+
? await safeRun('kubectl', [
|
|
1288
939
|
'--kubeconfig',
|
|
1289
940
|
kubeconfig,
|
|
1290
941
|
'get',
|
|
@@ -1296,7 +947,7 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
1296
947
|
])
|
|
1297
948
|
: `(kubeconfig not present at ${kubeconfig} — checked ${kubeconfigCandidates.join(', ')})`;
|
|
1298
949
|
const events = kcExists
|
|
1299
|
-
? safeRun('kubectl', [
|
|
950
|
+
? await safeRun('kubectl', [
|
|
1300
951
|
'--kubeconfig',
|
|
1301
952
|
kubeconfig,
|
|
1302
953
|
'get',
|
|
@@ -1313,13 +964,13 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
1313
964
|
// kong 503, etc.). curl exits non-zero on connect/TLS errors so we
|
|
1314
965
|
// route through safeRun. -k skips cert validation — we want the
|
|
1315
966
|
// body even if origin's cert is staging or self-signed.
|
|
1316
|
-
const curlVerbose = safeRun('curl', [
|
|
967
|
+
const curlVerbose = await safeRun('curl', [
|
|
1317
968
|
'-skvI',
|
|
1318
969
|
'--max-time',
|
|
1319
970
|
'10',
|
|
1320
971
|
`https://${domain}/api/health`,
|
|
1321
972
|
]);
|
|
1322
|
-
const curlPodIp = safeRun('curl', [
|
|
973
|
+
const curlPodIp = await safeRun('curl', [
|
|
1323
974
|
'-skv',
|
|
1324
975
|
'--max-time',
|
|
1325
976
|
'10',
|
|
@@ -1335,7 +986,7 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
1335
986
|
// turns "deploy probe got 404" from a riddle into a directly
|
|
1336
987
|
// actionable error.
|
|
1337
988
|
const pods = kcExists
|
|
1338
|
-
? safeRun('kubectl', [
|
|
989
|
+
? await safeRun('kubectl', [
|
|
1339
990
|
'--kubeconfig',
|
|
1340
991
|
kubeconfig,
|
|
1341
992
|
'get',
|
|
@@ -1347,7 +998,7 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
1347
998
|
])
|
|
1348
999
|
: '(skipped — no kubeconfig)';
|
|
1349
1000
|
const endpoints = kcExists
|
|
1350
|
-
? safeRun('kubectl', [
|
|
1001
|
+
? await safeRun('kubectl', [
|
|
1351
1002
|
'--kubeconfig',
|
|
1352
1003
|
kubeconfig,
|
|
1353
1004
|
'get',
|
|
@@ -1359,7 +1010,7 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
1359
1010
|
// App pod logs (last 100 lines). If multiple replicas, this picks
|
|
1360
1011
|
// by deployment selector — same as `kubectl logs deploy/app`.
|
|
1361
1012
|
const appLogs = kcExists
|
|
1362
|
-
? safeRun('kubectl', [
|
|
1013
|
+
? await safeRun('kubectl', [
|
|
1363
1014
|
'--kubeconfig',
|
|
1364
1015
|
kubeconfig,
|
|
1365
1016
|
'logs',
|
|
@@ -1481,11 +1132,11 @@ export async function executeDeployment(args, gatheredConfig) {
|
|
|
1481
1132
|
supabaseIp: deployResult.supabaseIp,
|
|
1482
1133
|
region,
|
|
1483
1134
|
serverType,
|
|
1484
|
-
...(
|
|
1135
|
+
...(isComposeTier(tier) ? {} : { role: 'master' }),
|
|
1485
1136
|
});
|
|
1486
1137
|
// K8s only — compose has neither a separate supabase node nor worker
|
|
1487
1138
|
// pool. Compose deploys leave servers as a single-element array.
|
|
1488
|
-
if (
|
|
1139
|
+
if (isK8sTier(tier)) {
|
|
1489
1140
|
if (deployResult.supabaseIp) {
|
|
1490
1141
|
servers.push({
|
|
1491
1142
|
name: 'supabase',
|