vibecarbon 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +110 -663
- package/README.md +4 -4
- package/carbon/.env.example +11 -0
- package/carbon/backup/compose-backup.sh +8 -0
- package/carbon/docker-compose.yml +11 -0
- package/carbon/ha/primary-init.sql +10 -2
- package/carbon/k8s/base/backup/cronjob.yaml +9 -0
- package/carbon/k8s/base/repl-gateway/kustomization.yaml +11 -0
- package/carbon/k8s/base/repl-gateway/repl-gateway.yaml +92 -0
- package/carbon/k8s/values/supabase.values.yaml +6 -1
- package/carbon/src/client/lib/supabase.ts +41 -0
- package/carbon/src/client/locales/de.json +27 -12
- package/carbon/src/client/locales/en.json +30 -15
- package/carbon/src/client/locales/es.json +27 -12
- package/carbon/src/client/locales/fr.json +27 -12
- package/carbon/src/client/locales/pt.json +27 -12
- package/carbon/src/client/pages/Checkout.tsx +1 -1
- package/carbon/src/server/billing/providers/paddle.ts +21 -2
- package/carbon/src/server/billing/providers/polar.ts +14 -0
- package/carbon/src/server/index.ts +12 -9
- package/carbon/src/server/lib/client-ip.ts +68 -0
- package/carbon/src/server/lib/env.ts +20 -6
- package/carbon/src/server/lib/rate-limiter.ts +11 -65
- package/carbon/src/server/lib/supabase.ts +10 -4
- package/carbon/src/server/middleware/requireOrgRole.ts +76 -0
- package/carbon/src/server/middleware/requirePlan.ts +64 -34
- package/carbon/src/server/middleware/requireSuperAdmin.ts +26 -0
- package/carbon/src/server/routes/v1/admin/contact.ts +5 -20
- package/carbon/src/server/routes/v1/admin/jobs.ts +6 -20
- package/carbon/src/server/routes/v1/admin/newsletter.ts +26 -23
- package/carbon/src/server/routes/v1/auth.ts +5 -2
- package/carbon/src/server/routes/v1/billing.ts +24 -64
- package/carbon/src/server/routes/v1/index.ts +8 -1
- package/carbon/src/server/routes/v1/newsletter.ts +18 -12
- package/carbon/src/server/routes/v1/theme.ts +36 -15
- package/carbon/supabase/migrations/00003_pg_cron.sql +2 -2
- package/carbon/supabase/migrations/00004_contact_newsletter.sql +24 -14
- package/carbon/tests/component/supabase-env-guard.test.ts +54 -0
- package/carbon/tests/integration/server/middleware/rate-limit-ip-spoof.test.ts +48 -0
- package/carbon/tests/integration/server/middleware/require-plan.test.ts +101 -0
- package/carbon/tests/integration/server/routes/admin-newsletter.test.ts +93 -0
- package/carbon/tests/integration/server/routes/billing-subscription-idor.test.ts +92 -0
- package/carbon/tests/integration/server/routes/newsletter-unsubscribe.test.ts +65 -0
- package/carbon/tests/integration/server/routes/theme-validation.test.ts +66 -0
- package/carbon/tests/unit/server/client-ip.test.ts +76 -0
- package/carbon/tests/unit/server/env-bool.test.ts +63 -0
- package/carbon/tests/unit/server/paddle-webhook.test.ts +57 -0
- package/carbon/tests/unit/server/polar-webhook.test.ts +63 -0
- package/carbon/tests/unit/server/supabase-client.test.ts +30 -0
- package/carbon/volumes/db/wal-archive.sh +36 -0
- package/package.json +2 -2
- package/src/backup.js +0 -2
- package/src/cli.js +16 -0
- package/src/deploy.js +16 -5
- package/src/destroy.js +968 -886
- package/src/failover.js +118 -237
- package/src/lib/command.js +19 -1
- package/src/lib/deploy/compose/ha.js +180 -102
- package/src/lib/deploy/compose/index.js +106 -57
- package/src/lib/deploy/compose/single.js +421 -0
- package/src/lib/deploy/image.js +38 -30
- package/src/lib/deploy/k8s/ha/index.js +397 -150
- package/src/lib/deploy/k8s/k3s.js +411 -279
- package/src/lib/deploy/orchestrator.js +274 -453
- package/src/lib/deploy/remote-build.js +8 -1
- package/src/lib/deploy/replication.js +540 -0
- package/src/lib/deploy/state.js +62 -5
- package/src/lib/deploy/tier-registry.js +30 -0
- package/src/lib/deploy/utils.js +30 -2
- package/src/lib/deploy/wireguard.js +146 -0
- package/src/lib/host-keys.js +120 -5
- package/src/lib/iac/index.js +57 -23
- package/src/lib/licensing/gate.js +59 -0
- package/src/lib/licensing/index.js +1 -1
- package/src/lib/licensing/tiers.js +6 -5
- package/src/lib/prod-confirm.js +65 -0
- package/src/lib/providers/hetzner-s3.js +85 -0
- package/src/lib/retry.js +59 -0
- package/src/lib/scale-plan.js +159 -0
- package/src/lib/ssh.js +35 -35
- package/src/restore.js +102 -14
- package/src/scale.js +105 -133
- package/src/status.js +122 -0
- package/src/upgrade.js +0 -4
- package/carbon/ha/activate-standby.sh +0 -52
- package/carbon/ha/standby-init.sh +0 -74
- package/carbon/src/server/lib/request.ts +0 -34
- package/src/lib/pod-backups.js +0 -53
package/src/scale.js
CHANGED
|
@@ -38,14 +38,16 @@ import {
|
|
|
38
38
|
waitForSSH,
|
|
39
39
|
} from './lib/deploy/compose/index.js';
|
|
40
40
|
import { buildRemote } from './lib/deploy/remote-build.js';
|
|
41
|
+
import { isHATier, resolveTier } from './lib/deploy/tier-registry.js';
|
|
41
42
|
import { DEFAULT_WORKER_MAX, DEFAULT_WORKER_MIN } from './lib/deploy/utils.js';
|
|
42
43
|
import { waitForDNSPropagation } from './lib/dns-propagation.js';
|
|
43
44
|
import { getApiToken, getS3Credentials } from './lib/hetzner-guided-setup.js';
|
|
44
|
-
import { requireLicense } from './lib/licensing/index.js';
|
|
45
45
|
import { ensureOperatorIpAccess } from './lib/operator-ip.js';
|
|
46
46
|
import { perfAsync } from './lib/perf.js';
|
|
47
47
|
import { assertInProjectDir } from './lib/project-guard.js';
|
|
48
48
|
import { HETZNER_PRICING_URL, HetznerProvider } from './lib/providers/hetzner.js';
|
|
49
|
+
import { pollUntil } from './lib/retry.js';
|
|
50
|
+
import { buildProgramConfig, planK8sScaleChanges, updateCaNodesArgs } from './lib/scale-plan.js';
|
|
49
51
|
import { buildComposeTypeOptions, buildSimpleTypeOptions } from './lib/server-types.js';
|
|
50
52
|
import { parseDotenv } from './lib/shell.js';
|
|
51
53
|
import { sshRun, sshRunScript } from './lib/ssh.js';
|
|
@@ -99,6 +101,7 @@ const SPEC = {
|
|
|
99
101
|
// ============================================================================
|
|
100
102
|
|
|
101
103
|
async function scaleCompose(environment, envConfig, projectConfig, options = {}) {
|
|
104
|
+
const tier = options.tier || resolveTier(envConfig);
|
|
102
105
|
const servers = envConfig.servers || [];
|
|
103
106
|
const projectName = projectConfig.projectName;
|
|
104
107
|
|
|
@@ -132,7 +135,7 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
|
|
|
132
135
|
|
|
133
136
|
// Ensure operator IP is in the firewall allowlist before any SSH work.
|
|
134
137
|
try {
|
|
135
|
-
const isHA =
|
|
138
|
+
const isHA = isHATier(tier);
|
|
136
139
|
const result = await ensureOperatorIpAccess({
|
|
137
140
|
projectConfig,
|
|
138
141
|
environment,
|
|
@@ -175,7 +178,7 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
|
|
|
175
178
|
|
|
176
179
|
// For compose-ha: select which server(s) to scale
|
|
177
180
|
let targetServers = resizableServers;
|
|
178
|
-
if (
|
|
181
|
+
if (isHATier(tier) && resizableServers.length > 1) {
|
|
179
182
|
const primaryServer = resizableServers.find((s) => s.role === 'primary');
|
|
180
183
|
const standbyServer = resizableServers.find((s) => s.role === 'standby');
|
|
181
184
|
|
|
@@ -233,7 +236,7 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
|
|
|
233
236
|
return;
|
|
234
237
|
}
|
|
235
238
|
|
|
236
|
-
const ha =
|
|
239
|
+
const ha = isHATier(tier);
|
|
237
240
|
const services = projectConfig.services || {};
|
|
238
241
|
p.note(`Current Hetzner pricing: ${HETZNER_PRICING_URL}`, 'Cost');
|
|
239
242
|
|
|
@@ -289,8 +292,7 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
|
|
|
289
292
|
// exactly 2× single-server scale (711s vs ~365s); parallelizing makes it
|
|
290
293
|
// ~max(primary, standby) ≈ 6m off the critical path.
|
|
291
294
|
const scaleSingleServer = async (server) => {
|
|
292
|
-
const label =
|
|
293
|
-
envConfig.deployMode === 'compose-ha' ? `${server.role} (${server.ip})` : server.ip;
|
|
295
|
+
const label = isHATier(tier) ? `${server.role} (${server.ip})` : server.ip;
|
|
294
296
|
const s = p.spinner();
|
|
295
297
|
|
|
296
298
|
// 1. Backup database on old server via wal-g (pushes directly to S3)
|
|
@@ -740,6 +742,17 @@ async function updateDNS(envConfig, apiToken, domain, newIp) {
|
|
|
740
742
|
// MAIN
|
|
741
743
|
// ============================================================================
|
|
742
744
|
|
|
745
|
+
// Tier → scale strategy. scaleCompose / scaleK8s are hoisted function
|
|
746
|
+
// declarations, so referencing them here at module scope is safe. Compose and
|
|
747
|
+
// compose-ha share one strategy (it branches on isHATier internally); likewise
|
|
748
|
+
// k8s / k8s-ha.
|
|
749
|
+
const SCALE_STRATEGIES = {
|
|
750
|
+
compose: scaleCompose,
|
|
751
|
+
'compose-ha': scaleCompose,
|
|
752
|
+
k8s: scaleK8s,
|
|
753
|
+
'k8s-ha': scaleK8s,
|
|
754
|
+
};
|
|
755
|
+
|
|
743
756
|
export async function run(args) {
|
|
744
757
|
const { values, positional, errors } = parseFlags(args, SPEC);
|
|
745
758
|
|
|
@@ -764,7 +777,6 @@ export async function run(args) {
|
|
|
764
777
|
// scale` from a parent directory emits the canonical message.
|
|
765
778
|
const projectConfig = assertInProjectDir();
|
|
766
779
|
|
|
767
|
-
requireLicense('scale');
|
|
768
780
|
printBanner();
|
|
769
781
|
p.intro(`${c.bold('vibecarbon scale')} ${c.dim(`v${VERSION}`)}`);
|
|
770
782
|
|
|
@@ -831,15 +843,18 @@ export async function run(args) {
|
|
|
831
843
|
process.exit(1);
|
|
832
844
|
}
|
|
833
845
|
|
|
834
|
-
//
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
846
|
+
// Dispatch to the tier's scale strategy. The 4th arg carries the parsed
|
|
847
|
+
// flags plus the resolved `tier` so each strategy dispatches on tier /
|
|
848
|
+
// isHATier(tier) instead of re-deriving deployMode+ha.
|
|
849
|
+
const tier = resolveTier(envConfig);
|
|
850
|
+
await SCALE_STRATEGIES[tier](environment, envConfig, projectConfig, { ...parsed, tier });
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
// ============================================================================
|
|
854
|
+
// KUBERNETES SCALING
|
|
855
|
+
// ============================================================================
|
|
842
856
|
|
|
857
|
+
async function scaleK8s(environment, envConfig, projectConfig, parsed) {
|
|
843
858
|
const region = envConfig.region || 'fsn1';
|
|
844
859
|
|
|
845
860
|
// Pre-fetch live server types (non-blocking, uses saved credentials if available)
|
|
@@ -884,60 +899,28 @@ export async function run(args) {
|
|
|
884
899
|
const regionTypes = HetznerProvider.getServerTypesForRegion(region);
|
|
885
900
|
const serverTypeOptions = buildSimpleTypeOptions(regionTypes, { filterSharedCpu: false });
|
|
886
901
|
|
|
902
|
+
// Non-interactive plan (pure): `-yes -type` → resize every role (matrix run
|
|
903
|
+
// #10 caught the old behavior of only changing worker pools); `-yes` + bounds
|
|
904
|
+
// → cluster-autoscaler bounds only. `null` means no scripted plan matched —
|
|
905
|
+
// fall through to the interactive multiselect. See src/lib/scale-plan.js for
|
|
906
|
+
// the branch rationale (in-place resize vs. master-replace guard, phase-8
|
|
907
|
+
// bounds TODO). The user-facing logging stays here (side-effectful).
|
|
887
908
|
let changes;
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
// ~2 min of control-plane downtime; running under -y means the
|
|
903
|
-
// operator accepted the cost.
|
|
904
|
-
//
|
|
905
|
-
// Per-role overrides remain available through the interactive
|
|
906
|
-
// multiselect when finer control is needed.
|
|
907
|
-
changes = ['masterType', 'supabaseType', 'workerType'];
|
|
908
|
-
newValues.masterType = parsed.type;
|
|
909
|
-
newValues.supabaseType = parsed.type;
|
|
910
|
-
newValues.workerType = parsed.type;
|
|
911
|
-
// Allow --type and bounds to be combined in one invocation. CA
|
|
912
|
-
// re-patch happens regardless once changes include 'workerBounds'.
|
|
913
|
-
if (parsed.minWorkers != null || parsed.maxWorkers != null) {
|
|
914
|
-
changes.push('workerBounds');
|
|
915
|
-
if (parsed.minWorkers != null) newValues.minWorkers = parsed.minWorkers;
|
|
916
|
-
if (parsed.maxWorkers != null) newValues.maxWorkers = parsed.maxWorkers;
|
|
909
|
+
const plan = planK8sScaleChanges(parsed, envConfig);
|
|
910
|
+
if (plan) {
|
|
911
|
+
changes = plan.changes;
|
|
912
|
+
Object.assign(newValues, plan.newValues);
|
|
913
|
+
if (changes.includes('masterType')) {
|
|
914
|
+
p.log.info(
|
|
915
|
+
`Scaling all node roles (master, supabase, worker) to ${c.bold(parsed.type)} ${c.dim('(in-place resize, ~2 min downtime per node)')}`,
|
|
916
|
+
);
|
|
917
|
+
} else {
|
|
918
|
+
p.log.info(
|
|
919
|
+
`Updating cluster-autoscaler bounds: ${c.bold(
|
|
920
|
+
`min=${newValues.minWorkers ?? envConfig.minWorkers ?? DEFAULT_WORKER_MIN}`,
|
|
921
|
+
)} ${c.bold(`max=${newValues.maxWorkers ?? envConfig.maxWorkers ?? DEFAULT_WORKER_MAX}`)}`,
|
|
922
|
+
);
|
|
917
923
|
}
|
|
918
|
-
p.log.info(
|
|
919
|
-
`Scaling all node roles (master, supabase, worker) to ${c.bold(parsed.type)} ${c.dim('(in-place resize, ~2 min downtime per node)')}`,
|
|
920
|
-
);
|
|
921
|
-
} else if (parsed.yes && (parsed.minWorkers != null || parsed.maxWorkers != null)) {
|
|
922
|
-
// Non-interactive bounds-only path: --min-workers and/or --max-workers
|
|
923
|
-
// without --type. We do not run Pulumi here unless minWorkers rose above
|
|
924
|
-
// the previously persisted floor (which would require new static workers
|
|
925
|
-
// to be provisioned).
|
|
926
|
-
//
|
|
927
|
-
// TODO(phase 8 follow-up): bounds-driven Pulumi changes — when
|
|
928
|
-
// minWorkers rises above the persisted floor, run Pulumi to add static
|
|
929
|
-
// workers; when minWorkers drops, leave Pulumi alone (don't reap static
|
|
930
|
-
// workers on bounds relax). For now we patch CA only; bumping min above
|
|
931
|
-
// the persisted floor will not provision new static workers until the
|
|
932
|
-
// next deploy.
|
|
933
|
-
changes = ['workerBounds'];
|
|
934
|
-
if (parsed.minWorkers != null) newValues.minWorkers = parsed.minWorkers;
|
|
935
|
-
if (parsed.maxWorkers != null) newValues.maxWorkers = parsed.maxWorkers;
|
|
936
|
-
p.log.info(
|
|
937
|
-
`Updating cluster-autoscaler bounds: ${c.bold(
|
|
938
|
-
`min=${newValues.minWorkers ?? envConfig.minWorkers ?? DEFAULT_WORKER_MIN}`,
|
|
939
|
-
)} ${c.bold(`max=${newValues.maxWorkers ?? envConfig.maxWorkers ?? DEFAULT_WORKER_MAX}`)}`,
|
|
940
|
-
);
|
|
941
924
|
} else {
|
|
942
925
|
changes = await p.multiselect({
|
|
943
926
|
message: 'What would you like to change?',
|
|
@@ -1005,7 +988,11 @@ export async function run(args) {
|
|
|
1005
988
|
(c) => c === 'workerType' || c === 'masterType' || c === 'supabaseType',
|
|
1006
989
|
);
|
|
1007
990
|
|
|
1008
|
-
|
|
991
|
+
// isHATier(tier) is behavior-identical to `envConfig.ha?.enabled` for the k8s
|
|
992
|
+
// tier; the `&& secondaryRegion` conjunct is kept because HA cluster fan-out
|
|
993
|
+
// below also requires a configured standby region (registry predicate alone
|
|
994
|
+
// is NOT equivalent here).
|
|
995
|
+
const isHA = isHATier(parsed.tier) && envConfig.secondaryRegion;
|
|
1009
996
|
p.note(`Current Hetzner pricing: ${HETZNER_PRICING_URL}`, 'Cost');
|
|
1010
997
|
|
|
1011
998
|
// 6. Require API token if infra changes (server type swap) needed
|
|
@@ -1023,11 +1010,12 @@ export async function run(args) {
|
|
|
1023
1010
|
// a no-op scale that just returns) since both paths below require token.
|
|
1024
1011
|
if (apiToken) {
|
|
1025
1012
|
try {
|
|
1026
|
-
|
|
1013
|
+
// Firewall allowlist gate uses ha-vs-not only (no secondaryRegion
|
|
1014
|
+
// conjunct) — isHATier(tier) === `!!envConfig.ha?.enabled` for k8s.
|
|
1027
1015
|
const result = await ensureOperatorIpAccess({
|
|
1028
1016
|
projectConfig,
|
|
1029
1017
|
environment,
|
|
1030
|
-
isHA,
|
|
1018
|
+
isHA: isHATier(parsed.tier),
|
|
1031
1019
|
apiToken,
|
|
1032
1020
|
yes: !!parsed.yes,
|
|
1033
1021
|
onMessage: (msg) => p.log.info(msg),
|
|
@@ -1056,14 +1044,6 @@ export async function run(args) {
|
|
|
1056
1044
|
);
|
|
1057
1045
|
const { buildHetznerK8sProgram } = await import('./lib/iac/programs/hetzner-k8s.js');
|
|
1058
1046
|
const { K3S_VERSION } = await import('./lib/deploy/k8s/k3s.js');
|
|
1059
|
-
const REGION_TO_NETWORK_ZONE = {
|
|
1060
|
-
nbg1: 'eu-central',
|
|
1061
|
-
hel1: 'eu-central',
|
|
1062
|
-
fsn1: 'eu-central',
|
|
1063
|
-
hil: 'us-west',
|
|
1064
|
-
ash: 'us-east',
|
|
1065
|
-
sin: 'ap-southeast',
|
|
1066
|
-
};
|
|
1067
1047
|
// SSH key location depends on deploy topology:
|
|
1068
1048
|
// - Single-cluster k8s: deployK3s' ensureSshKey writes
|
|
1069
1049
|
// `.vibecarbon/ssh-<env>` and lets Pulumi manage the SshKey resource.
|
|
@@ -1140,6 +1120,10 @@ export async function run(args) {
|
|
|
1140
1120
|
bucket: envConfig.s3.bucket,
|
|
1141
1121
|
region: envConfig.s3.region,
|
|
1142
1122
|
endpoint: envConfig.s3.endpoint,
|
|
1123
|
+
// Dedicated Pulumi-state bucket. Undefined for envs deployed before
|
|
1124
|
+
// the state-bucket split — resolveBackendUrl then falls back to the
|
|
1125
|
+
// app bucket, matching where their state actually still lives.
|
|
1126
|
+
stateBucket: envConfig.s3.stateBucket,
|
|
1143
1127
|
accessKey: s3Creds.accessKey,
|
|
1144
1128
|
secretKey: s3Creds.secretKey,
|
|
1145
1129
|
};
|
|
@@ -1152,45 +1136,30 @@ export async function run(args) {
|
|
|
1152
1136
|
// destructive replace. The k3sToken probe-and-replay (further below)
|
|
1153
1137
|
// keeps userData stable; HA needed `existingSshKeyId` to also be
|
|
1154
1138
|
// mirrored or master.sshKeys would still drift and replace etcd.
|
|
1139
|
+
// Byte-identical to deployK3s' program inputs — every field that flows
|
|
1140
|
+
// into a Server's sshKeys/firewallIds/labels/userData is a destructive-
|
|
1141
|
+
// replace tripwire (replacing master = etcd wipe). buildProgramConfig
|
|
1142
|
+
// (src/lib/scale-plan.js) owns the assembly + the region→zone map + the
|
|
1143
|
+
// newValues/persisted/default `??` resolution. k3sVersion and labels MUST
|
|
1144
|
+
// match deployK3s (both are interpolated into userData / resource labels).
|
|
1155
1145
|
const allowedCidrs = (projectConfig.operatorCidrs ?? []).map((e) => e.cidr);
|
|
1156
|
-
const programConfig = {
|
|
1146
|
+
const programConfig = buildProgramConfig({
|
|
1157
1147
|
projectName: projectConfig.projectName,
|
|
1158
1148
|
environment: clusterEnv,
|
|
1159
1149
|
sshPublicKey,
|
|
1160
|
-
|
|
1161
|
-
allowedK8sApiIps: allowedCidrs,
|
|
1162
|
-
// HA deploys share one Hetzner SshKey across primary + standby and
|
|
1163
|
-
// pass `existingSshKeyId` so neither stack manages the resource.
|
|
1164
|
-
// Single-cluster lets Pulumi own the SshKey (existingSshKeyId =
|
|
1165
|
-
// undefined). Mirror whichever the deploy did or master.sshKeys
|
|
1166
|
-
// changes and Pulumi replaces every node.
|
|
1150
|
+
allowedCidrs,
|
|
1167
1151
|
existingSshKeyId,
|
|
1168
1152
|
location: clusterRegion,
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
// maxWorkers is not consumed by Pulumi; it flows to the CA Deployment
|
|
1176
|
-
// patch below. If the operator passed --min-workers/--max-workers
|
|
1177
|
-
// we honor them; otherwise replay the previously persisted values
|
|
1178
|
-
// (which are needed verbatim to keep Pulumi's plan a no-op for the
|
|
1179
|
-
// worker pool when the operator is just resizing types).
|
|
1180
|
-
minWorkers: newValues.minWorkers ?? envConfig.minWorkers ?? DEFAULT_WORKER_MIN,
|
|
1181
|
-
maxWorkers: newValues.maxWorkers ?? envConfig.maxWorkers ?? DEFAULT_WORKER_MAX,
|
|
1182
|
-
// MUST match the K3S_VERSION used by deployK3s — k3sVersion is
|
|
1183
|
-
// interpolated into each server's userData (cloud-init script), so a
|
|
1184
|
-
// mismatch makes Pulumi see ~userData on master/supabase/workers and
|
|
1185
|
-
// tries to replace them. Replacing master = etcd wipe.
|
|
1153
|
+
newValues,
|
|
1154
|
+
currentMasterType,
|
|
1155
|
+
currentSupabaseType,
|
|
1156
|
+
currentWorkerType,
|
|
1157
|
+
persistedMinWorkers: envConfig.minWorkers,
|
|
1158
|
+
persistedMaxWorkers: envConfig.maxWorkers,
|
|
1186
1159
|
k3sVersion: K3S_VERSION,
|
|
1187
|
-
// MUST match the labels passed by deployK3s (src/lib/deploy/k8s/k3s.js).
|
|
1188
|
-
// Mismatched labels make Pulumi mark every resource as ~labels and
|
|
1189
|
-
// attempt a replace — for hcloud.SshKey, replace = delete+create with
|
|
1190
|
-
// the same public-key bytes, which Hetzner rejects with HTTP 409.
|
|
1191
1160
|
labels: { 'managed-by': 'vibecarbon', 'os-flavor': 'k3s' },
|
|
1192
1161
|
apiToken,
|
|
1193
|
-
};
|
|
1162
|
+
});
|
|
1194
1163
|
|
|
1195
1164
|
// Probe prior stack outputs for the k3sToken so we can replay it into
|
|
1196
1165
|
// the real program. classifyK3sTokenProbe normalizes recovered/empty/
|
|
@@ -1386,13 +1355,7 @@ export async function run(args) {
|
|
|
1386
1355
|
} catch {
|
|
1387
1356
|
argList = [];
|
|
1388
1357
|
}
|
|
1389
|
-
const updated = argList
|
|
1390
|
-
typeof a === 'string' && a.startsWith('--nodes=') ? `--nodes=${caNodesSpec}` : a,
|
|
1391
|
-
);
|
|
1392
|
-
// If for some reason --nodes wasn't there, append it.
|
|
1393
|
-
if (!updated.some((a) => typeof a === 'string' && a.startsWith('--nodes='))) {
|
|
1394
|
-
updated.push(`--nodes=${caNodesSpec}`);
|
|
1395
|
-
}
|
|
1358
|
+
const updated = updateCaNodesArgs(argList, caNodesSpec);
|
|
1396
1359
|
const patch = JSON.stringify({
|
|
1397
1360
|
spec: {
|
|
1398
1361
|
template: { spec: { containers: [{ name: 'cluster-autoscaler', args: updated }] } },
|
|
@@ -1515,31 +1478,40 @@ export async function run(args) {
|
|
|
1515
1478
|
// plane has truly stabilized post-resize.
|
|
1516
1479
|
const apiSpinner = p.spinner();
|
|
1517
1480
|
apiSpinner.start(`Waiting for API server to be reachable (${label})`);
|
|
1518
|
-
const apiStart = Date.now();
|
|
1519
1481
|
const API_READY_BUDGET_MS = 300_000;
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1482
|
+
// Fixed 5s interval (backoffFactor: 1) under a 300s budget — the probe
|
|
1483
|
+
// throws while the floating IP is re-attaching (runCommand rejects on
|
|
1484
|
+
// nonzero exit when silent), and pollUntil retries a throwing probe.
|
|
1485
|
+
// Timeout still throws (preserving the old fail-fast); we re-wrap so the
|
|
1486
|
+
// operator-facing message and last-error detail are unchanged.
|
|
1487
|
+
try {
|
|
1488
|
+
await pollUntil(
|
|
1489
|
+
() => {
|
|
1490
|
+
runCommand(['kubectl', '--kubeconfig', kubeconfigPath, 'get', '--raw=/healthz'], {
|
|
1491
|
+
silent: true,
|
|
1492
|
+
timeout: 10_000,
|
|
1493
|
+
});
|
|
1494
|
+
return true;
|
|
1495
|
+
},
|
|
1496
|
+
{
|
|
1497
|
+
budgetMs: API_READY_BUDGET_MS,
|
|
1498
|
+
initialDelayMs: 5000,
|
|
1499
|
+
backoffFactor: 1,
|
|
1500
|
+
description: 'k8s API healthy after resize',
|
|
1501
|
+
},
|
|
1502
|
+
);
|
|
1503
|
+
apiSpinner.stop(`API server reachable (${label})`);
|
|
1504
|
+
} catch (err) {
|
|
1536
1505
|
apiSpinner.stop(`API server (${label}) not reachable within 5 min`);
|
|
1506
|
+
const cause = err.cause;
|
|
1507
|
+
const lastApiErr = cause
|
|
1508
|
+
? cause.message?.split('\n')[0]?.slice(0, 120) || String(cause).slice(0, 120)
|
|
1509
|
+
: '';
|
|
1537
1510
|
throw new Error(
|
|
1538
1511
|
`Kube API server for ${label} never returned /healthz=ok within 5 min after Pulumi resize. ` +
|
|
1539
1512
|
`Last error: ${lastApiErr}`,
|
|
1540
1513
|
);
|
|
1541
1514
|
}
|
|
1542
|
-
apiSpinner.stop(`API server reachable (${label})`);
|
|
1543
1515
|
// Wait for the Hetzner CSI driver DaemonSet to register on every
|
|
1544
1516
|
// node BEFORE checking pod-Ready. RCA from k8s-ha standby failure
|
|
1545
1517
|
// 2026-04-28 (live cluster repro under triple-node-reboot):
|
package/src/status.js
CHANGED
|
@@ -20,6 +20,7 @@ import { c, printBanner } from './lib/colors.js';
|
|
|
20
20
|
import { runCommand } from './lib/command.js';
|
|
21
21
|
import { cleanStaleProjects, loadGlobalRegistry, loadProjectConfig } from './lib/config.js';
|
|
22
22
|
import { HetznerProvider } from './lib/providers/hetzner.js';
|
|
23
|
+
import { getPostgresPod, getSSHKeyPath, sshKubectl, sshRun } from './lib/ssh.js';
|
|
23
24
|
import { VERSION } from './lib/version.js';
|
|
24
25
|
|
|
25
26
|
/** @type {import('./lib/cli/parse-flags.js').CommandSpec & { summary?: string, description?: string, examples?: Array<{ command: string, description?: string }> }} */
|
|
@@ -324,6 +325,108 @@ async function checkRemoteHealth(domain) {
|
|
|
324
325
|
};
|
|
325
326
|
}
|
|
326
327
|
|
|
328
|
+
/**
|
|
329
|
+
* Read the REAL streaming-replication state for an HA environment by querying
|
|
330
|
+
* `pg_stat_replication` on the CURRENT primary. Surfaces honest DR state instead
|
|
331
|
+
* of a hardcoded "streaming" string (finding #4). Best-effort and hard-bounded:
|
|
332
|
+
* any failure (unreachable, db down, non-HA) resolves to a value the renderer
|
|
333
|
+
* can degrade on, and the whole probe is raced against a short timeout so
|
|
334
|
+
* `status` never hangs.
|
|
335
|
+
*
|
|
336
|
+
* @returns {Promise<{ streaming: boolean, state: string, lagBytes: number|null } | null>}
|
|
337
|
+
*/
|
|
338
|
+
async function checkReplication(envName, envConfig, projectName, deps = {}) {
|
|
339
|
+
const {
|
|
340
|
+
sshRun: _sshRun = sshRun,
|
|
341
|
+
sshKubectl: _sshKubectl = sshKubectl,
|
|
342
|
+
getPostgresPod: _getPod = getPostgresPod,
|
|
343
|
+
getSSHKeyPath: _getKey = getSSHKeyPath,
|
|
344
|
+
timeoutMs = 10_000,
|
|
345
|
+
} = deps;
|
|
346
|
+
|
|
347
|
+
const isHA = !!(envConfig.ha?.enabled || envConfig.ha === true || envConfig.secondaryRegion);
|
|
348
|
+
if (!isHA) return null;
|
|
349
|
+
|
|
350
|
+
const servers = envConfig.servers || [];
|
|
351
|
+
const primary = servers.find((sv) => sv.role === 'primary') || servers[0];
|
|
352
|
+
if (!primary?.ip) return null;
|
|
353
|
+
|
|
354
|
+
const sshKeyPath = _getKey(envName);
|
|
355
|
+
if (!existsSync(sshKeyPath)) return null;
|
|
356
|
+
|
|
357
|
+
const isCompose = envConfig.deployMode === 'compose' || envConfig.deployMode === 'compose-ha';
|
|
358
|
+
// No string literals in the SQL → no shell/psql quoting hazards. `-tA` uses
|
|
359
|
+
// '|' as the field separator, so a connected standby yields `streaming|<lag>`.
|
|
360
|
+
const sql =
|
|
361
|
+
'SELECT state, pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) FROM pg_stat_replication';
|
|
362
|
+
|
|
363
|
+
const probe = (async () => {
|
|
364
|
+
let out;
|
|
365
|
+
if (isCompose) {
|
|
366
|
+
out = await _sshRun(
|
|
367
|
+
primary.ip,
|
|
368
|
+
sshKeyPath,
|
|
369
|
+
[
|
|
370
|
+
'bash',
|
|
371
|
+
'-lc',
|
|
372
|
+
`cd /opt/${projectName} && docker compose exec -T db psql -U supabase_admin -d postgres -tAc "${sql}"`,
|
|
373
|
+
],
|
|
374
|
+
{ timeout: timeoutMs },
|
|
375
|
+
);
|
|
376
|
+
} else {
|
|
377
|
+
const pod = await _getPod(primary.ip, sshKeyPath);
|
|
378
|
+
out = await _sshKubectl(
|
|
379
|
+
primary.ip,
|
|
380
|
+
sshKeyPath,
|
|
381
|
+
[
|
|
382
|
+
'exec',
|
|
383
|
+
'-n',
|
|
384
|
+
'vibecarbon',
|
|
385
|
+
pod,
|
|
386
|
+
'--',
|
|
387
|
+
'psql',
|
|
388
|
+
'-U',
|
|
389
|
+
'supabase_admin',
|
|
390
|
+
'-d',
|
|
391
|
+
'postgres',
|
|
392
|
+
'-tAc',
|
|
393
|
+
sql,
|
|
394
|
+
],
|
|
395
|
+
{ timeout: timeoutMs },
|
|
396
|
+
);
|
|
397
|
+
}
|
|
398
|
+
const line = (typeof out === 'string' ? out : '')
|
|
399
|
+
.split('\n')
|
|
400
|
+
.map((l) => l.trim())
|
|
401
|
+
.filter(Boolean)[0];
|
|
402
|
+
if (!line) return { streaming: false, state: 'no standby connected', lagBytes: null };
|
|
403
|
+
const [state, lagStr] = line.split('|');
|
|
404
|
+
const lag = lagStr ? Number.parseInt(lagStr, 10) : Number.NaN;
|
|
405
|
+
return {
|
|
406
|
+
streaming: state === 'streaming',
|
|
407
|
+
state: state || 'unknown',
|
|
408
|
+
lagBytes: Number.isNaN(lag) ? null : lag,
|
|
409
|
+
};
|
|
410
|
+
})();
|
|
411
|
+
|
|
412
|
+
try {
|
|
413
|
+
return await Promise.race([
|
|
414
|
+
probe,
|
|
415
|
+
new Promise((resolve) => {
|
|
416
|
+
// .unref() so a fast probe win doesn't leave this timer holding the
|
|
417
|
+
// event loop open and delaying `status` exit by the timeout window.
|
|
418
|
+
const t = setTimeout(
|
|
419
|
+
() => resolve({ streaming: false, state: 'unknown', lagBytes: null }),
|
|
420
|
+
timeoutMs + 2_000,
|
|
421
|
+
);
|
|
422
|
+
if (typeof t?.unref === 'function') t.unref();
|
|
423
|
+
}),
|
|
424
|
+
]);
|
|
425
|
+
} catch {
|
|
426
|
+
return { streaming: false, state: 'unknown', lagBytes: null };
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
327
430
|
async function getServerInfo(serverId) {
|
|
328
431
|
const apiToken = process.env.HETZNER_API_TOKEN;
|
|
329
432
|
if (!apiToken) return null;
|
|
@@ -506,6 +609,20 @@ function renderEnvironment(envName, envConfig, checks) {
|
|
|
506
609
|
? `${c.success('Enabled')} ${c.dim(`(failover: ${failoverRegion})`)}`
|
|
507
610
|
: c.success('Enabled');
|
|
508
611
|
lines.push(`${c.dim('HA')} ${haDisplay}`);
|
|
612
|
+
|
|
613
|
+
// Real replication state (finding #4) — never a hardcoded "streaming".
|
|
614
|
+
if (checks.replication) {
|
|
615
|
+
const r = checks.replication;
|
|
616
|
+
let replDisplay;
|
|
617
|
+
if (r.streaming) {
|
|
618
|
+
const lag =
|
|
619
|
+
r.lagBytes != null ? c.dim(` (lag: ${(r.lagBytes / 1024).toFixed(0)} KiB)`) : '';
|
|
620
|
+
replDisplay = `${c.success('streaming')}${lag}`;
|
|
621
|
+
} else {
|
|
622
|
+
replDisplay = `${c.error(r.state || 'not streaming')} ${c.dim('— DR not guaranteed')}`;
|
|
623
|
+
}
|
|
624
|
+
lines.push(`${c.dim('Replication')} ${replDisplay}`);
|
|
625
|
+
}
|
|
509
626
|
}
|
|
510
627
|
|
|
511
628
|
// Servers
|
|
@@ -816,6 +933,10 @@ async function main() {
|
|
|
816
933
|
// Git sync
|
|
817
934
|
checks.gitSync = checkGitSync(envName, envConfig);
|
|
818
935
|
|
|
936
|
+
// Real replication state for HA envs (best-effort, hard-bounded). null
|
|
937
|
+
// for non-HA or when the primary/key isn't locally reachable.
|
|
938
|
+
checks.replication = await checkReplication(envName, envConfig, projectConfig.projectName);
|
|
939
|
+
|
|
819
940
|
return { envName, config: envConfig, checks };
|
|
820
941
|
}),
|
|
821
942
|
);
|
|
@@ -902,6 +1023,7 @@ export {
|
|
|
902
1023
|
checkHttpEndpoint,
|
|
903
1024
|
checkLocalDev,
|
|
904
1025
|
checkRemoteHealth,
|
|
1026
|
+
checkReplication,
|
|
905
1027
|
formatRelativeTime,
|
|
906
1028
|
getBranchName,
|
|
907
1029
|
getServerInfo,
|
package/src/upgrade.js
CHANGED
|
@@ -26,7 +26,6 @@ import { renderHelp } from './lib/cli/help.js';
|
|
|
26
26
|
import { parseFlags } from './lib/cli/parse-flags.js';
|
|
27
27
|
import { c, printBanner } from './lib/colors.js';
|
|
28
28
|
import { runCommandAsync } from './lib/command.js';
|
|
29
|
-
import { requireLicense } from './lib/licensing/index.js';
|
|
30
29
|
import { mergePackageJson } from './lib/merge-package-json.js';
|
|
31
30
|
import {
|
|
32
31
|
adaptDockerfileForPackageManager,
|
|
@@ -294,9 +293,6 @@ async function main(cliArgs) {
|
|
|
294
293
|
// Detect project
|
|
295
294
|
assertInProjectDir();
|
|
296
295
|
|
|
297
|
-
// Require a paid license (Diamond or Fullerene)
|
|
298
|
-
requireLicense('upgrade');
|
|
299
|
-
|
|
300
296
|
const cwd = process.cwd();
|
|
301
297
|
const manifest = loadManifest(cwd);
|
|
302
298
|
const currentTemplateVersion = manifest.templateVersion || '0.0.0';
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
# Promote a standby PostgreSQL to primary and scale up services.
|
|
3
|
-
#
|
|
4
|
-
# This script runs ON the standby master node and uses kubectl to:
|
|
5
|
-
# 1. Promote PostgreSQL from standby to primary
|
|
6
|
-
# 2. Wait for the promotion to complete
|
|
7
|
-
# 3. Scale up Supabase and application services
|
|
8
|
-
#
|
|
9
|
-
# Optional env vars:
|
|
10
|
-
# NAMESPACE — Kubernetes namespace (default: vibecarbon)
|
|
11
|
-
|
|
12
|
-
set -euo pipefail
|
|
13
|
-
|
|
14
|
-
NAMESPACE="${NAMESPACE:-vibecarbon}"
|
|
15
|
-
PGDATA="/var/lib/postgresql/data"
|
|
16
|
-
|
|
17
|
-
# Find the postgres pod
|
|
18
|
-
POD=$(kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/name=supabase-db -o jsonpath='{.items[0].metadata.name}')
|
|
19
|
-
if [ -z "$POD" ]; then
|
|
20
|
-
echo "ERROR: No postgres pod found in namespace $NAMESPACE"
|
|
21
|
-
exit 1
|
|
22
|
-
fi
|
|
23
|
-
|
|
24
|
-
echo "Promoting standby: pod=$POD"
|
|
25
|
-
|
|
26
|
-
# 1. Promote PostgreSQL
|
|
27
|
-
echo "Promoting PostgreSQL standby to primary..."
|
|
28
|
-
kubectl exec -n "$NAMESPACE" "$POD" -- su postgres -c "pg_ctl promote -D $PGDATA" 2>/dev/null || true
|
|
29
|
-
|
|
30
|
-
# 2. Wait for promotion to complete
|
|
31
|
-
echo "Waiting for promotion..."
|
|
32
|
-
for i in $(seq 1 30); do
|
|
33
|
-
result=$(kubectl exec -n "$NAMESPACE" "$POD" -- psql -U supabase_admin -tAc 'SELECT pg_is_in_recovery()' 2>/dev/null || echo "t")
|
|
34
|
-
if [ "$result" = "f" ]; then
|
|
35
|
-
echo "PostgreSQL promoted successfully"
|
|
36
|
-
break
|
|
37
|
-
fi
|
|
38
|
-
if [ "$i" = "30" ]; then
|
|
39
|
-
echo "WARNING: Promotion not confirmed after 60 seconds"
|
|
40
|
-
fi
|
|
41
|
-
sleep 2
|
|
42
|
-
done
|
|
43
|
-
|
|
44
|
-
# 3. Scale up services
|
|
45
|
-
echo "Scaling up services..."
|
|
46
|
-
kubectl scale deployment app -n "$NAMESPACE" --replicas=2
|
|
47
|
-
kubectl scale deployment auth -n "$NAMESPACE" --replicas=2
|
|
48
|
-
kubectl scale deployment rest -n "$NAMESPACE" --replicas=2
|
|
49
|
-
kubectl scale deployment realtime -n "$NAMESPACE" --replicas=2
|
|
50
|
-
kubectl scale deployment storage -n "$NAMESPACE" --replicas=1
|
|
51
|
-
|
|
52
|
-
echo "Standby activation complete"
|