vibecarbon 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/LICENSE +110 -663
  2. package/README.md +4 -4
  3. package/carbon/.env.example +11 -0
  4. package/carbon/backup/compose-backup.sh +8 -0
  5. package/carbon/docker-compose.yml +11 -0
  6. package/carbon/ha/primary-init.sql +10 -2
  7. package/carbon/k8s/base/backup/cronjob.yaml +9 -0
  8. package/carbon/k8s/base/repl-gateway/kustomization.yaml +11 -0
  9. package/carbon/k8s/base/repl-gateway/repl-gateway.yaml +92 -0
  10. package/carbon/k8s/values/supabase.values.yaml +6 -1
  11. package/carbon/src/client/lib/supabase.ts +41 -0
  12. package/carbon/src/client/locales/de.json +27 -12
  13. package/carbon/src/client/locales/en.json +30 -15
  14. package/carbon/src/client/locales/es.json +27 -12
  15. package/carbon/src/client/locales/fr.json +27 -12
  16. package/carbon/src/client/locales/pt.json +27 -12
  17. package/carbon/src/client/pages/Checkout.tsx +1 -1
  18. package/carbon/src/server/billing/providers/paddle.ts +21 -2
  19. package/carbon/src/server/billing/providers/polar.ts +14 -0
  20. package/carbon/src/server/index.ts +12 -9
  21. package/carbon/src/server/lib/client-ip.ts +68 -0
  22. package/carbon/src/server/lib/env.ts +20 -6
  23. package/carbon/src/server/lib/rate-limiter.ts +11 -65
  24. package/carbon/src/server/lib/supabase.ts +10 -4
  25. package/carbon/src/server/middleware/requireOrgRole.ts +76 -0
  26. package/carbon/src/server/middleware/requirePlan.ts +64 -34
  27. package/carbon/src/server/middleware/requireSuperAdmin.ts +26 -0
  28. package/carbon/src/server/routes/v1/admin/contact.ts +5 -20
  29. package/carbon/src/server/routes/v1/admin/jobs.ts +6 -20
  30. package/carbon/src/server/routes/v1/admin/newsletter.ts +26 -23
  31. package/carbon/src/server/routes/v1/auth.ts +5 -2
  32. package/carbon/src/server/routes/v1/billing.ts +24 -64
  33. package/carbon/src/server/routes/v1/index.ts +8 -1
  34. package/carbon/src/server/routes/v1/newsletter.ts +18 -12
  35. package/carbon/src/server/routes/v1/theme.ts +36 -15
  36. package/carbon/supabase/migrations/00003_pg_cron.sql +2 -2
  37. package/carbon/supabase/migrations/00004_contact_newsletter.sql +24 -14
  38. package/carbon/tests/component/supabase-env-guard.test.ts +54 -0
  39. package/carbon/tests/integration/server/middleware/rate-limit-ip-spoof.test.ts +48 -0
  40. package/carbon/tests/integration/server/middleware/require-plan.test.ts +101 -0
  41. package/carbon/tests/integration/server/routes/admin-newsletter.test.ts +93 -0
  42. package/carbon/tests/integration/server/routes/billing-subscription-idor.test.ts +92 -0
  43. package/carbon/tests/integration/server/routes/newsletter-unsubscribe.test.ts +65 -0
  44. package/carbon/tests/integration/server/routes/theme-validation.test.ts +66 -0
  45. package/carbon/tests/unit/server/client-ip.test.ts +76 -0
  46. package/carbon/tests/unit/server/env-bool.test.ts +63 -0
  47. package/carbon/tests/unit/server/paddle-webhook.test.ts +57 -0
  48. package/carbon/tests/unit/server/polar-webhook.test.ts +63 -0
  49. package/carbon/tests/unit/server/supabase-client.test.ts +30 -0
  50. package/carbon/volumes/db/wal-archive.sh +36 -0
  51. package/package.json +2 -2
  52. package/src/backup.js +0 -2
  53. package/src/cli.js +16 -0
  54. package/src/deploy.js +16 -5
  55. package/src/destroy.js +968 -886
  56. package/src/failover.js +118 -237
  57. package/src/lib/command.js +19 -1
  58. package/src/lib/deploy/compose/ha.js +180 -102
  59. package/src/lib/deploy/compose/index.js +106 -57
  60. package/src/lib/deploy/compose/single.js +421 -0
  61. package/src/lib/deploy/image.js +38 -30
  62. package/src/lib/deploy/k8s/ha/index.js +397 -150
  63. package/src/lib/deploy/k8s/k3s.js +411 -279
  64. package/src/lib/deploy/orchestrator.js +274 -453
  65. package/src/lib/deploy/remote-build.js +8 -1
  66. package/src/lib/deploy/replication.js +540 -0
  67. package/src/lib/deploy/state.js +62 -5
  68. package/src/lib/deploy/tier-registry.js +30 -0
  69. package/src/lib/deploy/utils.js +30 -2
  70. package/src/lib/deploy/wireguard.js +146 -0
  71. package/src/lib/host-keys.js +120 -5
  72. package/src/lib/iac/index.js +57 -23
  73. package/src/lib/licensing/gate.js +59 -0
  74. package/src/lib/licensing/index.js +1 -1
  75. package/src/lib/licensing/tiers.js +6 -5
  76. package/src/lib/prod-confirm.js +65 -0
  77. package/src/lib/providers/hetzner-s3.js +85 -0
  78. package/src/lib/retry.js +59 -0
  79. package/src/lib/scale-plan.js +159 -0
  80. package/src/lib/ssh.js +35 -35
  81. package/src/restore.js +102 -14
  82. package/src/scale.js +105 -133
  83. package/src/status.js +122 -0
  84. package/src/upgrade.js +0 -4
  85. package/carbon/ha/activate-standby.sh +0 -52
  86. package/carbon/ha/standby-init.sh +0 -74
  87. package/carbon/src/server/lib/request.ts +0 -34
  88. package/src/lib/pod-backups.js +0 -53
package/src/scale.js CHANGED
@@ -38,14 +38,16 @@ import {
38
38
  waitForSSH,
39
39
  } from './lib/deploy/compose/index.js';
40
40
  import { buildRemote } from './lib/deploy/remote-build.js';
41
+ import { isHATier, resolveTier } from './lib/deploy/tier-registry.js';
41
42
  import { DEFAULT_WORKER_MAX, DEFAULT_WORKER_MIN } from './lib/deploy/utils.js';
42
43
  import { waitForDNSPropagation } from './lib/dns-propagation.js';
43
44
  import { getApiToken, getS3Credentials } from './lib/hetzner-guided-setup.js';
44
- import { requireLicense } from './lib/licensing/index.js';
45
45
  import { ensureOperatorIpAccess } from './lib/operator-ip.js';
46
46
  import { perfAsync } from './lib/perf.js';
47
47
  import { assertInProjectDir } from './lib/project-guard.js';
48
48
  import { HETZNER_PRICING_URL, HetznerProvider } from './lib/providers/hetzner.js';
49
+ import { pollUntil } from './lib/retry.js';
50
+ import { buildProgramConfig, planK8sScaleChanges, updateCaNodesArgs } from './lib/scale-plan.js';
49
51
  import { buildComposeTypeOptions, buildSimpleTypeOptions } from './lib/server-types.js';
50
52
  import { parseDotenv } from './lib/shell.js';
51
53
  import { sshRun, sshRunScript } from './lib/ssh.js';
@@ -99,6 +101,7 @@ const SPEC = {
99
101
  // ============================================================================
100
102
 
101
103
  async function scaleCompose(environment, envConfig, projectConfig, options = {}) {
104
+ const tier = options.tier || resolveTier(envConfig);
102
105
  const servers = envConfig.servers || [];
103
106
  const projectName = projectConfig.projectName;
104
107
 
@@ -132,7 +135,7 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
132
135
 
133
136
  // Ensure operator IP is in the firewall allowlist before any SSH work.
134
137
  try {
135
- const isHA = envConfig.deployMode === 'compose-ha';
138
+ const isHA = isHATier(tier);
136
139
  const result = await ensureOperatorIpAccess({
137
140
  projectConfig,
138
141
  environment,
@@ -175,7 +178,7 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
175
178
 
176
179
  // For compose-ha: select which server(s) to scale
177
180
  let targetServers = resizableServers;
178
- if (envConfig.deployMode === 'compose-ha' && resizableServers.length > 1) {
181
+ if (isHATier(tier) && resizableServers.length > 1) {
179
182
  const primaryServer = resizableServers.find((s) => s.role === 'primary');
180
183
  const standbyServer = resizableServers.find((s) => s.role === 'standby');
181
184
 
@@ -233,7 +236,7 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
233
236
  return;
234
237
  }
235
238
 
236
- const ha = envConfig.deployMode === 'compose-ha';
239
+ const ha = isHATier(tier);
237
240
  const services = projectConfig.services || {};
238
241
  p.note(`Current Hetzner pricing: ${HETZNER_PRICING_URL}`, 'Cost');
239
242
 
@@ -289,8 +292,7 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
289
292
  // exactly 2× single-server scale (711s vs ~365s); parallelizing makes it
290
293
  // ~max(primary, standby) ≈ 6m off the critical path.
291
294
  const scaleSingleServer = async (server) => {
292
- const label =
293
- envConfig.deployMode === 'compose-ha' ? `${server.role} (${server.ip})` : server.ip;
295
+ const label = isHATier(tier) ? `${server.role} (${server.ip})` : server.ip;
294
296
  const s = p.spinner();
295
297
 
296
298
  // 1. Backup database on old server via wal-g (pushes directly to S3)
@@ -740,6 +742,17 @@ async function updateDNS(envConfig, apiToken, domain, newIp) {
740
742
  // MAIN
741
743
  // ============================================================================
742
744
 
745
+ // Tier → scale strategy. scaleCompose / scaleK8s are hoisted function
746
+ // declarations, so referencing them here at module scope is safe. Compose and
747
+ // compose-ha share one strategy (it branches on isHATier internally); likewise
748
+ // k8s / k8s-ha.
749
+ const SCALE_STRATEGIES = {
750
+ compose: scaleCompose,
751
+ 'compose-ha': scaleCompose,
752
+ k8s: scaleK8s,
753
+ 'k8s-ha': scaleK8s,
754
+ };
755
+
743
756
  export async function run(args) {
744
757
  const { values, positional, errors } = parseFlags(args, SPEC);
745
758
 
@@ -764,7 +777,6 @@ export async function run(args) {
764
777
  // scale` from a parent directory emits the canonical message.
765
778
  const projectConfig = assertInProjectDir();
766
779
 
767
- requireLicense('scale');
768
780
  printBanner();
769
781
  p.intro(`${c.bold('vibecarbon scale')} ${c.dim(`v${VERSION}`)}`);
770
782
 
@@ -831,15 +843,18 @@ export async function run(args) {
831
843
  process.exit(1);
832
844
  }
833
845
 
834
- // Compose deployments: vertical scaling via Hetzner API
835
- if (envConfig.deployMode === 'compose' || envConfig.deployMode === 'compose-ha') {
836
- await scaleCompose(environment, envConfig, projectConfig, {
837
- type: parsed.type,
838
- yes: parsed.yes,
839
- });
840
- return;
841
- }
846
+ // Dispatch to the tier's scale strategy. The 4th arg carries the parsed
847
+ // flags plus the resolved `tier` so each strategy dispatches on tier /
848
+ // isHATier(tier) instead of re-deriving deployMode+ha.
849
+ const tier = resolveTier(envConfig);
850
+ await SCALE_STRATEGIES[tier](environment, envConfig, projectConfig, { ...parsed, tier });
851
+ }
852
+
853
+ // ============================================================================
854
+ // KUBERNETES SCALING
855
+ // ============================================================================
842
856
 
857
+ async function scaleK8s(environment, envConfig, projectConfig, parsed) {
843
858
  const region = envConfig.region || 'fsn1';
844
859
 
845
860
  // Pre-fetch live server types (non-blocking, uses saved credentials if available)
@@ -884,60 +899,28 @@ export async function run(args) {
884
899
  const regionTypes = HetznerProvider.getServerTypesForRegion(region);
885
900
  const serverTypeOptions = buildSimpleTypeOptions(regionTypes, { filterSharedCpu: false });
886
901
 
902
+ // Non-interactive plan (pure): `-yes -type` → resize every role (matrix run
903
+ // #10 caught the old behavior of only changing worker pools); `-yes` + bounds
904
+ // → cluster-autoscaler bounds only. `null` means no scripted plan matched —
905
+ // fall through to the interactive multiselect. See src/lib/scale-plan.js for
906
+ // the branch rationale (in-place resize vs. master-replace guard, phase-8
907
+ // bounds TODO). The user-facing logging stays here (side-effectful).
887
908
  let changes;
888
- if (parsed.yes && parsed.type) {
889
- // Non-interactive: `--type X` means "scale every node role to X" for
890
- // parity with compose's `--type X` (which resizes the one VPS).
891
- // Without this, a k8s `--type cx33` only changed worker pools and
892
- // left master + supabase on the previous type — a UX inconsistency
893
- // the e2e verify-scale assertion caught (matrix run #10 k8s
894
- // scale: only the worker IP showed cx33 post-scale, supabase stayed
895
- // cx23).
896
- //
897
- // Hetzner server_type changes are in-place resizes (~2 min reboot
898
- // per node), so the master-replace guard below does NOT trip on
899
- // legitimate type bumps — that guard fires only on userData drift
900
- // (k3sToken, labels, cloud-init template), which the probe-and-replay
901
- // earlier in this function already prevents. Master upgrades cause
902
- // ~2 min of control-plane downtime; running under -y means the
903
- // operator accepted the cost.
904
- //
905
- // Per-role overrides remain available through the interactive
906
- // multiselect when finer control is needed.
907
- changes = ['masterType', 'supabaseType', 'workerType'];
908
- newValues.masterType = parsed.type;
909
- newValues.supabaseType = parsed.type;
910
- newValues.workerType = parsed.type;
911
- // Allow --type and bounds to be combined in one invocation. CA
912
- // re-patch happens regardless once changes include 'workerBounds'.
913
- if (parsed.minWorkers != null || parsed.maxWorkers != null) {
914
- changes.push('workerBounds');
915
- if (parsed.minWorkers != null) newValues.minWorkers = parsed.minWorkers;
916
- if (parsed.maxWorkers != null) newValues.maxWorkers = parsed.maxWorkers;
909
+ const plan = planK8sScaleChanges(parsed, envConfig);
910
+ if (plan) {
911
+ changes = plan.changes;
912
+ Object.assign(newValues, plan.newValues);
913
+ if (changes.includes('masterType')) {
914
+ p.log.info(
915
+ `Scaling all node roles (master, supabase, worker) to ${c.bold(parsed.type)} ${c.dim('(in-place resize, ~2 min downtime per node)')}`,
916
+ );
917
+ } else {
918
+ p.log.info(
919
+ `Updating cluster-autoscaler bounds: ${c.bold(
920
+ `min=${newValues.minWorkers ?? envConfig.minWorkers ?? DEFAULT_WORKER_MIN}`,
921
+ )} ${c.bold(`max=${newValues.maxWorkers ?? envConfig.maxWorkers ?? DEFAULT_WORKER_MAX}`)}`,
922
+ );
917
923
  }
918
- p.log.info(
919
- `Scaling all node roles (master, supabase, worker) to ${c.bold(parsed.type)} ${c.dim('(in-place resize, ~2 min downtime per node)')}`,
920
- );
921
- } else if (parsed.yes && (parsed.minWorkers != null || parsed.maxWorkers != null)) {
922
- // Non-interactive bounds-only path: --min-workers and/or --max-workers
923
- // without --type. We do not run Pulumi here unless minWorkers rose above
924
- // the previously persisted floor (which would require new static workers
925
- // to be provisioned).
926
- //
927
- // TODO(phase 8 follow-up): bounds-driven Pulumi changes — when
928
- // minWorkers rises above the persisted floor, run Pulumi to add static
929
- // workers; when minWorkers drops, leave Pulumi alone (don't reap static
930
- // workers on bounds relax). For now we patch CA only; bumping min above
931
- // the persisted floor will not provision new static workers until the
932
- // next deploy.
933
- changes = ['workerBounds'];
934
- if (parsed.minWorkers != null) newValues.minWorkers = parsed.minWorkers;
935
- if (parsed.maxWorkers != null) newValues.maxWorkers = parsed.maxWorkers;
936
- p.log.info(
937
- `Updating cluster-autoscaler bounds: ${c.bold(
938
- `min=${newValues.minWorkers ?? envConfig.minWorkers ?? DEFAULT_WORKER_MIN}`,
939
- )} ${c.bold(`max=${newValues.maxWorkers ?? envConfig.maxWorkers ?? DEFAULT_WORKER_MAX}`)}`,
940
- );
941
924
  } else {
942
925
  changes = await p.multiselect({
943
926
  message: 'What would you like to change?',
@@ -1005,7 +988,11 @@ export async function run(args) {
1005
988
  (c) => c === 'workerType' || c === 'masterType' || c === 'supabaseType',
1006
989
  );
1007
990
 
1008
- const isHA = envConfig.ha?.enabled && envConfig.secondaryRegion;
991
+ // isHATier(tier) is behavior-identical to `envConfig.ha?.enabled` for the k8s
992
+ // tier; the `&& secondaryRegion` conjunct is kept because HA cluster fan-out
993
+ // below also requires a configured standby region (registry predicate alone
994
+ // is NOT equivalent here).
995
+ const isHA = isHATier(parsed.tier) && envConfig.secondaryRegion;
1009
996
  p.note(`Current Hetzner pricing: ${HETZNER_PRICING_URL}`, 'Cost');
1010
997
 
1011
998
  // 6. Require API token if infra changes (server type swap) needed
@@ -1023,11 +1010,12 @@ export async function run(args) {
1023
1010
  // a no-op scale that just returns) since both paths below require token.
1024
1011
  if (apiToken) {
1025
1012
  try {
1026
- const isHA = !!envConfig.ha?.enabled;
1013
+ // Firewall allowlist gate uses ha-vs-not only (no secondaryRegion
1014
+ // conjunct) — isHATier(tier) === `!!envConfig.ha?.enabled` for k8s.
1027
1015
  const result = await ensureOperatorIpAccess({
1028
1016
  projectConfig,
1029
1017
  environment,
1030
- isHA,
1018
+ isHA: isHATier(parsed.tier),
1031
1019
  apiToken,
1032
1020
  yes: !!parsed.yes,
1033
1021
  onMessage: (msg) => p.log.info(msg),
@@ -1056,14 +1044,6 @@ export async function run(args) {
1056
1044
  );
1057
1045
  const { buildHetznerK8sProgram } = await import('./lib/iac/programs/hetzner-k8s.js');
1058
1046
  const { K3S_VERSION } = await import('./lib/deploy/k8s/k3s.js');
1059
- const REGION_TO_NETWORK_ZONE = {
1060
- nbg1: 'eu-central',
1061
- hel1: 'eu-central',
1062
- fsn1: 'eu-central',
1063
- hil: 'us-west',
1064
- ash: 'us-east',
1065
- sin: 'ap-southeast',
1066
- };
1067
1047
  // SSH key location depends on deploy topology:
1068
1048
  // - Single-cluster k8s: deployK3s' ensureSshKey writes
1069
1049
  // `.vibecarbon/ssh-<env>` and lets Pulumi manage the SshKey resource.
@@ -1140,6 +1120,10 @@ export async function run(args) {
1140
1120
  bucket: envConfig.s3.bucket,
1141
1121
  region: envConfig.s3.region,
1142
1122
  endpoint: envConfig.s3.endpoint,
1123
+ // Dedicated Pulumi-state bucket. Undefined for envs deployed before
1124
+ // the state-bucket split — resolveBackendUrl then falls back to the
1125
+ // app bucket, matching where their state actually still lives.
1126
+ stateBucket: envConfig.s3.stateBucket,
1143
1127
  accessKey: s3Creds.accessKey,
1144
1128
  secretKey: s3Creds.secretKey,
1145
1129
  };
@@ -1152,45 +1136,30 @@ export async function run(args) {
1152
1136
  // destructive replace. The k3sToken probe-and-replay (further below)
1153
1137
  // keeps userData stable; HA needed `existingSshKeyId` to also be
1154
1138
  // mirrored or master.sshKeys would still drift and replace etcd.
1139
+ // Byte-identical to deployK3s' program inputs — every field that flows
1140
+ // into a Server's sshKeys/firewallIds/labels/userData is a destructive-
1141
+ // replace tripwire (replacing master = etcd wipe). buildProgramConfig
1142
+ // (src/lib/scale-plan.js) owns the assembly + the region→zone map + the
1143
+ // newValues/persisted/default `??` resolution. k3sVersion and labels MUST
1144
+ // match deployK3s (both are interpolated into userData / resource labels).
1155
1145
  const allowedCidrs = (projectConfig.operatorCidrs ?? []).map((e) => e.cidr);
1156
- const programConfig = {
1146
+ const programConfig = buildProgramConfig({
1157
1147
  projectName: projectConfig.projectName,
1158
1148
  environment: clusterEnv,
1159
1149
  sshPublicKey,
1160
- allowedSshIps: allowedCidrs,
1161
- allowedK8sApiIps: allowedCidrs,
1162
- // HA deploys share one Hetzner SshKey across primary + standby and
1163
- // pass `existingSshKeyId` so neither stack manages the resource.
1164
- // Single-cluster lets Pulumi own the SshKey (existingSshKeyId =
1165
- // undefined). Mirror whichever the deploy did or master.sshKeys
1166
- // changes and Pulumi replaces every node.
1150
+ allowedCidrs,
1167
1151
  existingSshKeyId,
1168
1152
  location: clusterRegion,
1169
- networkZone: REGION_TO_NETWORK_ZONE[clusterRegion] || 'eu-central',
1170
- masterServerType: newValues.masterType ?? currentMasterType,
1171
- supabaseServerType: newValues.supabaseType ?? currentSupabaseType,
1172
- workerServerType: newValues.workerType ?? currentWorkerType,
1173
- // Phase 8: minWorkers/maxWorkers replace workerCount. minWorkers is
1174
- // what Pulumi consumes (static floor of worker servers it owns).
1175
- // maxWorkers is not consumed by Pulumi; it flows to the CA Deployment
1176
- // patch below. If the operator passed --min-workers/--max-workers
1177
- // we honor them; otherwise replay the previously persisted values
1178
- // (which are needed verbatim to keep Pulumi's plan a no-op for the
1179
- // worker pool when the operator is just resizing types).
1180
- minWorkers: newValues.minWorkers ?? envConfig.minWorkers ?? DEFAULT_WORKER_MIN,
1181
- maxWorkers: newValues.maxWorkers ?? envConfig.maxWorkers ?? DEFAULT_WORKER_MAX,
1182
- // MUST match the K3S_VERSION used by deployK3s — k3sVersion is
1183
- // interpolated into each server's userData (cloud-init script), so a
1184
- // mismatch makes Pulumi see ~userData on master/supabase/workers and
1185
- // tries to replace them. Replacing master = etcd wipe.
1153
+ newValues,
1154
+ currentMasterType,
1155
+ currentSupabaseType,
1156
+ currentWorkerType,
1157
+ persistedMinWorkers: envConfig.minWorkers,
1158
+ persistedMaxWorkers: envConfig.maxWorkers,
1186
1159
  k3sVersion: K3S_VERSION,
1187
- // MUST match the labels passed by deployK3s (src/lib/deploy/k8s/k3s.js).
1188
- // Mismatched labels make Pulumi mark every resource as ~labels and
1189
- // attempt a replace — for hcloud.SshKey, replace = delete+create with
1190
- // the same public-key bytes, which Hetzner rejects with HTTP 409.
1191
1160
  labels: { 'managed-by': 'vibecarbon', 'os-flavor': 'k3s' },
1192
1161
  apiToken,
1193
- };
1162
+ });
1194
1163
 
1195
1164
  // Probe prior stack outputs for the k3sToken so we can replay it into
1196
1165
  // the real program. classifyK3sTokenProbe normalizes recovered/empty/
@@ -1386,13 +1355,7 @@ export async function run(args) {
1386
1355
  } catch {
1387
1356
  argList = [];
1388
1357
  }
1389
- const updated = argList.map((a) =>
1390
- typeof a === 'string' && a.startsWith('--nodes=') ? `--nodes=${caNodesSpec}` : a,
1391
- );
1392
- // If for some reason --nodes wasn't there, append it.
1393
- if (!updated.some((a) => typeof a === 'string' && a.startsWith('--nodes='))) {
1394
- updated.push(`--nodes=${caNodesSpec}`);
1395
- }
1358
+ const updated = updateCaNodesArgs(argList, caNodesSpec);
1396
1359
  const patch = JSON.stringify({
1397
1360
  spec: {
1398
1361
  template: { spec: { containers: [{ name: 'cluster-autoscaler', args: updated }] } },
@@ -1515,31 +1478,40 @@ export async function run(args) {
1515
1478
  // plane has truly stabilized post-resize.
1516
1479
  const apiSpinner = p.spinner();
1517
1480
  apiSpinner.start(`Waiting for API server to be reachable (${label})`);
1518
- const apiStart = Date.now();
1519
1481
  const API_READY_BUDGET_MS = 300_000;
1520
- let apiReady = false;
1521
- let lastApiErr = '';
1522
- while (Date.now() - apiStart < API_READY_BUDGET_MS) {
1523
- try {
1524
- runCommand(['kubectl', '--kubeconfig', kubeconfigPath, 'get', '--raw=/healthz'], {
1525
- silent: true,
1526
- timeout: 10_000,
1527
- });
1528
- apiReady = true;
1529
- break;
1530
- } catch (err) {
1531
- lastApiErr = err.message?.split('\n')[0]?.slice(0, 120) || String(err).slice(0, 120);
1532
- await new Promise((r) => setTimeout(r, 5_000));
1533
- }
1534
- }
1535
- if (!apiReady) {
1482
+ // Fixed 5s interval (backoffFactor: 1) under a 300s budget — the probe
1483
+ // throws while the floating IP is re-attaching (runCommand rejects on
1484
+ // nonzero exit when silent), and pollUntil retries a throwing probe.
1485
+ // Timeout still throws (preserving the old fail-fast); we re-wrap so the
1486
+ // operator-facing message and last-error detail are unchanged.
1487
+ try {
1488
+ await pollUntil(
1489
+ () => {
1490
+ runCommand(['kubectl', '--kubeconfig', kubeconfigPath, 'get', '--raw=/healthz'], {
1491
+ silent: true,
1492
+ timeout: 10_000,
1493
+ });
1494
+ return true;
1495
+ },
1496
+ {
1497
+ budgetMs: API_READY_BUDGET_MS,
1498
+ initialDelayMs: 5000,
1499
+ backoffFactor: 1,
1500
+ description: 'k8s API healthy after resize',
1501
+ },
1502
+ );
1503
+ apiSpinner.stop(`API server reachable (${label})`);
1504
+ } catch (err) {
1536
1505
  apiSpinner.stop(`API server (${label}) not reachable within 5 min`);
1506
+ const cause = err.cause;
1507
+ const lastApiErr = cause
1508
+ ? cause.message?.split('\n')[0]?.slice(0, 120) || String(cause).slice(0, 120)
1509
+ : '';
1537
1510
  throw new Error(
1538
1511
  `Kube API server for ${label} never returned /healthz=ok within 5 min after Pulumi resize. ` +
1539
1512
  `Last error: ${lastApiErr}`,
1540
1513
  );
1541
1514
  }
1542
- apiSpinner.stop(`API server reachable (${label})`);
1543
1515
  // Wait for the Hetzner CSI driver DaemonSet to register on every
1544
1516
  // node BEFORE checking pod-Ready. RCA from k8s-ha standby failure
1545
1517
  // 2026-04-28 (live cluster repro under triple-node-reboot):
package/src/status.js CHANGED
@@ -20,6 +20,7 @@ import { c, printBanner } from './lib/colors.js';
20
20
  import { runCommand } from './lib/command.js';
21
21
  import { cleanStaleProjects, loadGlobalRegistry, loadProjectConfig } from './lib/config.js';
22
22
  import { HetznerProvider } from './lib/providers/hetzner.js';
23
+ import { getPostgresPod, getSSHKeyPath, sshKubectl, sshRun } from './lib/ssh.js';
23
24
  import { VERSION } from './lib/version.js';
24
25
 
25
26
  /** @type {import('./lib/cli/parse-flags.js').CommandSpec & { summary?: string, description?: string, examples?: Array<{ command: string, description?: string }> }} */
@@ -324,6 +325,108 @@ async function checkRemoteHealth(domain) {
324
325
  };
325
326
  }
326
327
 
328
+ /**
329
+ * Read the REAL streaming-replication state for an HA environment by querying
330
+ * `pg_stat_replication` on the CURRENT primary. Surfaces honest DR state instead
331
+ * of a hardcoded "streaming" string (finding #4). Best-effort and hard-bounded:
332
+ * any failure (unreachable, db down, non-HA) resolves to a value the renderer
333
+ * can degrade on, and the whole probe is raced against a short timeout so
334
+ * `status` never hangs.
335
+ *
336
+ * @returns {Promise<{ streaming: boolean, state: string, lagBytes: number|null } | null>}
337
+ */
338
+ async function checkReplication(envName, envConfig, projectName, deps = {}) {
339
+ const {
340
+ sshRun: _sshRun = sshRun,
341
+ sshKubectl: _sshKubectl = sshKubectl,
342
+ getPostgresPod: _getPod = getPostgresPod,
343
+ getSSHKeyPath: _getKey = getSSHKeyPath,
344
+ timeoutMs = 10_000,
345
+ } = deps;
346
+
347
+ const isHA = !!(envConfig.ha?.enabled || envConfig.ha === true || envConfig.secondaryRegion);
348
+ if (!isHA) return null;
349
+
350
+ const servers = envConfig.servers || [];
351
+ const primary = servers.find((sv) => sv.role === 'primary') || servers[0];
352
+ if (!primary?.ip) return null;
353
+
354
+ const sshKeyPath = _getKey(envName);
355
+ if (!existsSync(sshKeyPath)) return null;
356
+
357
+ const isCompose = envConfig.deployMode === 'compose' || envConfig.deployMode === 'compose-ha';
358
+ // No string literals in the SQL → no shell/psql quoting hazards. `-tA` uses
359
+ // '|' as the field separator, so a connected standby yields `streaming|<lag>`.
360
+ const sql =
361
+ 'SELECT state, pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) FROM pg_stat_replication';
362
+
363
+ const probe = (async () => {
364
+ let out;
365
+ if (isCompose) {
366
+ out = await _sshRun(
367
+ primary.ip,
368
+ sshKeyPath,
369
+ [
370
+ 'bash',
371
+ '-lc',
372
+ `cd /opt/${projectName} && docker compose exec -T db psql -U supabase_admin -d postgres -tAc "${sql}"`,
373
+ ],
374
+ { timeout: timeoutMs },
375
+ );
376
+ } else {
377
+ const pod = await _getPod(primary.ip, sshKeyPath);
378
+ out = await _sshKubectl(
379
+ primary.ip,
380
+ sshKeyPath,
381
+ [
382
+ 'exec',
383
+ '-n',
384
+ 'vibecarbon',
385
+ pod,
386
+ '--',
387
+ 'psql',
388
+ '-U',
389
+ 'supabase_admin',
390
+ '-d',
391
+ 'postgres',
392
+ '-tAc',
393
+ sql,
394
+ ],
395
+ { timeout: timeoutMs },
396
+ );
397
+ }
398
+ const line = (typeof out === 'string' ? out : '')
399
+ .split('\n')
400
+ .map((l) => l.trim())
401
+ .filter(Boolean)[0];
402
+ if (!line) return { streaming: false, state: 'no standby connected', lagBytes: null };
403
+ const [state, lagStr] = line.split('|');
404
+ const lag = lagStr ? Number.parseInt(lagStr, 10) : Number.NaN;
405
+ return {
406
+ streaming: state === 'streaming',
407
+ state: state || 'unknown',
408
+ lagBytes: Number.isNaN(lag) ? null : lag,
409
+ };
410
+ })();
411
+
412
+ try {
413
+ return await Promise.race([
414
+ probe,
415
+ new Promise((resolve) => {
416
+ // .unref() so a fast probe win doesn't leave this timer holding the
417
+ // event loop open and delaying `status` exit by the timeout window.
418
+ const t = setTimeout(
419
+ () => resolve({ streaming: false, state: 'unknown', lagBytes: null }),
420
+ timeoutMs + 2_000,
421
+ );
422
+ if (typeof t?.unref === 'function') t.unref();
423
+ }),
424
+ ]);
425
+ } catch {
426
+ return { streaming: false, state: 'unknown', lagBytes: null };
427
+ }
428
+ }
429
+
327
430
  async function getServerInfo(serverId) {
328
431
  const apiToken = process.env.HETZNER_API_TOKEN;
329
432
  if (!apiToken) return null;
@@ -506,6 +609,20 @@ function renderEnvironment(envName, envConfig, checks) {
506
609
  ? `${c.success('Enabled')} ${c.dim(`(failover: ${failoverRegion})`)}`
507
610
  : c.success('Enabled');
508
611
  lines.push(`${c.dim('HA')} ${haDisplay}`);
612
+
613
+ // Real replication state (finding #4) — never a hardcoded "streaming".
614
+ if (checks.replication) {
615
+ const r = checks.replication;
616
+ let replDisplay;
617
+ if (r.streaming) {
618
+ const lag =
619
+ r.lagBytes != null ? c.dim(` (lag: ${(r.lagBytes / 1024).toFixed(0)} KiB)`) : '';
620
+ replDisplay = `${c.success('streaming')}${lag}`;
621
+ } else {
622
+ replDisplay = `${c.error(r.state || 'not streaming')} ${c.dim('— DR not guaranteed')}`;
623
+ }
624
+ lines.push(`${c.dim('Replication')} ${replDisplay}`);
625
+ }
509
626
  }
510
627
 
511
628
  // Servers
@@ -816,6 +933,10 @@ async function main() {
816
933
  // Git sync
817
934
  checks.gitSync = checkGitSync(envName, envConfig);
818
935
 
936
+ // Real replication state for HA envs (best-effort, hard-bounded). null
937
+ // for non-HA or when the primary/key isn't locally reachable.
938
+ checks.replication = await checkReplication(envName, envConfig, projectConfig.projectName);
939
+
819
940
  return { envName, config: envConfig, checks };
820
941
  }),
821
942
  );
@@ -902,6 +1023,7 @@ export {
902
1023
  checkHttpEndpoint,
903
1024
  checkLocalDev,
904
1025
  checkRemoteHealth,
1026
+ checkReplication,
905
1027
  formatRelativeTime,
906
1028
  getBranchName,
907
1029
  getServerInfo,
package/src/upgrade.js CHANGED
@@ -26,7 +26,6 @@ import { renderHelp } from './lib/cli/help.js';
26
26
  import { parseFlags } from './lib/cli/parse-flags.js';
27
27
  import { c, printBanner } from './lib/colors.js';
28
28
  import { runCommandAsync } from './lib/command.js';
29
- import { requireLicense } from './lib/licensing/index.js';
30
29
  import { mergePackageJson } from './lib/merge-package-json.js';
31
30
  import {
32
31
  adaptDockerfileForPackageManager,
@@ -294,9 +293,6 @@ async function main(cliArgs) {
294
293
  // Detect project
295
294
  assertInProjectDir();
296
295
 
297
- // Require a paid license (Diamond or Fullerene)
298
- requireLicense('upgrade');
299
-
300
296
  const cwd = process.cwd();
301
297
  const manifest = loadManifest(cwd);
302
298
  const currentTemplateVersion = manifest.templateVersion || '0.0.0';
@@ -1,52 +0,0 @@
1
- #!/bin/bash
2
- # Promote a standby PostgreSQL to primary and scale up services.
3
- #
4
- # This script runs ON the standby master node and uses kubectl to:
5
- # 1. Promote PostgreSQL from standby to primary
6
- # 2. Wait for the promotion to complete
7
- # 3. Scale up Supabase and application services
8
- #
9
- # Optional env vars:
10
- # NAMESPACE — Kubernetes namespace (default: vibecarbon)
11
-
12
- set -euo pipefail
13
-
14
- NAMESPACE="${NAMESPACE:-vibecarbon}"
15
- PGDATA="/var/lib/postgresql/data"
16
-
17
- # Find the postgres pod
18
- POD=$(kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/name=supabase-db -o jsonpath='{.items[0].metadata.name}')
19
- if [ -z "$POD" ]; then
20
- echo "ERROR: No postgres pod found in namespace $NAMESPACE"
21
- exit 1
22
- fi
23
-
24
- echo "Promoting standby: pod=$POD"
25
-
26
- # 1. Promote PostgreSQL
27
- echo "Promoting PostgreSQL standby to primary..."
28
- kubectl exec -n "$NAMESPACE" "$POD" -- su postgres -c "pg_ctl promote -D $PGDATA" 2>/dev/null || true
29
-
30
- # 2. Wait for promotion to complete
31
- echo "Waiting for promotion..."
32
- for i in $(seq 1 30); do
33
- result=$(kubectl exec -n "$NAMESPACE" "$POD" -- psql -U supabase_admin -tAc 'SELECT pg_is_in_recovery()' 2>/dev/null || echo "t")
34
- if [ "$result" = "f" ]; then
35
- echo "PostgreSQL promoted successfully"
36
- break
37
- fi
38
- if [ "$i" = "30" ]; then
39
- echo "WARNING: Promotion not confirmed after 60 seconds"
40
- fi
41
- sleep 2
42
- done
43
-
44
- # 3. Scale up services
45
- echo "Scaling up services..."
46
- kubectl scale deployment app -n "$NAMESPACE" --replicas=2
47
- kubectl scale deployment auth -n "$NAMESPACE" --replicas=2
48
- kubectl scale deployment rest -n "$NAMESPACE" --replicas=2
49
- kubectl scale deployment realtime -n "$NAMESPACE" --replicas=2
50
- kubectl scale deployment storage -n "$NAMESPACE" --replicas=1
51
-
52
- echo "Standby activation complete"