vibecarbon 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/LICENSE +110 -663
  2. package/README.md +4 -4
  3. package/carbon/.env.example +11 -0
  4. package/carbon/backup/compose-backup.sh +8 -0
  5. package/carbon/docker-compose.yml +11 -0
  6. package/carbon/ha/primary-init.sql +10 -2
  7. package/carbon/k8s/base/backup/cronjob.yaml +9 -0
  8. package/carbon/k8s/base/repl-gateway/kustomization.yaml +11 -0
  9. package/carbon/k8s/base/repl-gateway/repl-gateway.yaml +92 -0
  10. package/carbon/k8s/values/supabase.values.yaml +6 -1
  11. package/carbon/src/client/lib/supabase.ts +41 -0
  12. package/carbon/src/client/locales/de.json +27 -12
  13. package/carbon/src/client/locales/en.json +30 -15
  14. package/carbon/src/client/locales/es.json +27 -12
  15. package/carbon/src/client/locales/fr.json +27 -12
  16. package/carbon/src/client/locales/pt.json +27 -12
  17. package/carbon/src/client/pages/Checkout.tsx +1 -1
  18. package/carbon/src/server/billing/providers/paddle.ts +21 -2
  19. package/carbon/src/server/billing/providers/polar.ts +14 -0
  20. package/carbon/src/server/index.ts +12 -9
  21. package/carbon/src/server/lib/client-ip.ts +68 -0
  22. package/carbon/src/server/lib/env.ts +20 -6
  23. package/carbon/src/server/lib/rate-limiter.ts +11 -65
  24. package/carbon/src/server/lib/supabase.ts +10 -4
  25. package/carbon/src/server/middleware/requireOrgRole.ts +76 -0
  26. package/carbon/src/server/middleware/requirePlan.ts +64 -34
  27. package/carbon/src/server/middleware/requireSuperAdmin.ts +26 -0
  28. package/carbon/src/server/routes/v1/admin/contact.ts +5 -20
  29. package/carbon/src/server/routes/v1/admin/jobs.ts +6 -20
  30. package/carbon/src/server/routes/v1/admin/newsletter.ts +26 -23
  31. package/carbon/src/server/routes/v1/auth.ts +5 -2
  32. package/carbon/src/server/routes/v1/billing.ts +24 -64
  33. package/carbon/src/server/routes/v1/index.ts +8 -1
  34. package/carbon/src/server/routes/v1/newsletter.ts +18 -12
  35. package/carbon/src/server/routes/v1/theme.ts +36 -15
  36. package/carbon/supabase/migrations/00003_pg_cron.sql +2 -2
  37. package/carbon/supabase/migrations/00004_contact_newsletter.sql +24 -14
  38. package/carbon/tests/component/supabase-env-guard.test.ts +54 -0
  39. package/carbon/tests/integration/server/middleware/rate-limit-ip-spoof.test.ts +48 -0
  40. package/carbon/tests/integration/server/middleware/require-plan.test.ts +101 -0
  41. package/carbon/tests/integration/server/routes/admin-newsletter.test.ts +93 -0
  42. package/carbon/tests/integration/server/routes/billing-subscription-idor.test.ts +92 -0
  43. package/carbon/tests/integration/server/routes/newsletter-unsubscribe.test.ts +65 -0
  44. package/carbon/tests/integration/server/routes/theme-validation.test.ts +66 -0
  45. package/carbon/tests/unit/server/client-ip.test.ts +76 -0
  46. package/carbon/tests/unit/server/env-bool.test.ts +63 -0
  47. package/carbon/tests/unit/server/paddle-webhook.test.ts +57 -0
  48. package/carbon/tests/unit/server/polar-webhook.test.ts +63 -0
  49. package/carbon/tests/unit/server/supabase-client.test.ts +30 -0
  50. package/carbon/volumes/db/wal-archive.sh +36 -0
  51. package/package.json +2 -2
  52. package/src/backup.js +0 -2
  53. package/src/cli.js +16 -0
  54. package/src/deploy.js +16 -5
  55. package/src/destroy.js +968 -886
  56. package/src/failover.js +118 -237
  57. package/src/lib/command.js +19 -1
  58. package/src/lib/deploy/compose/ha.js +180 -102
  59. package/src/lib/deploy/compose/index.js +106 -57
  60. package/src/lib/deploy/compose/single.js +421 -0
  61. package/src/lib/deploy/image.js +38 -30
  62. package/src/lib/deploy/k8s/ha/index.js +397 -150
  63. package/src/lib/deploy/k8s/k3s.js +411 -279
  64. package/src/lib/deploy/orchestrator.js +274 -453
  65. package/src/lib/deploy/remote-build.js +8 -1
  66. package/src/lib/deploy/replication.js +540 -0
  67. package/src/lib/deploy/state.js +62 -5
  68. package/src/lib/deploy/tier-registry.js +30 -0
  69. package/src/lib/deploy/utils.js +30 -2
  70. package/src/lib/deploy/wireguard.js +146 -0
  71. package/src/lib/host-keys.js +120 -5
  72. package/src/lib/iac/index.js +57 -23
  73. package/src/lib/licensing/gate.js +59 -0
  74. package/src/lib/licensing/index.js +1 -1
  75. package/src/lib/licensing/tiers.js +6 -5
  76. package/src/lib/prod-confirm.js +65 -0
  77. package/src/lib/providers/hetzner-s3.js +85 -0
  78. package/src/lib/retry.js +59 -0
  79. package/src/lib/scale-plan.js +159 -0
  80. package/src/lib/ssh.js +35 -35
  81. package/src/restore.js +102 -14
  82. package/src/scale.js +105 -133
  83. package/src/status.js +122 -0
  84. package/src/upgrade.js +0 -4
  85. package/carbon/ha/activate-standby.sh +0 -52
  86. package/carbon/ha/standby-init.sh +0 -74
  87. package/carbon/src/server/lib/request.ts +0 -34
  88. package/src/lib/pod-backups.js +0 -53
@@ -0,0 +1,65 @@
1
+ /**
2
+ * Shared production type-to-confirm guard.
3
+ *
4
+ * Destructive commands (destroy, restore, failover) must NOT let a `-y` /
5
+ * `--yes` flag silently blow away a production environment. This module is the
6
+ * single source of truth for "which envs are production" and for the
7
+ * type-to-confirm prompt that runs UNCONDITIONALLY (independent of -y) on those
8
+ * envs — matching `destroy`'s long-standing behavior.
9
+ *
10
+ * `src/destroy.js` still carries its own copy of `requiresProdTypeToConfirm`
11
+ * for its bespoke `projectName-envName` slug; new callers (restore, and later
12
+ * failover) should import from here.
13
+ */
14
+
15
+ import * as p from '@clack/prompts';
16
+
17
+ /**
18
+ * Returns true for environment names that require a type-to-confirm prompt even
19
+ * when -y is passed. Currently: `prod` and `production` (case-insensitive).
20
+ * A trailing/leading qualifier (e.g. `prod-backup`, `production-us`) is NOT
21
+ * treated as production — it is a distinct environment.
22
+ *
23
+ * @param {string | null | undefined} envName
24
+ * @returns {boolean}
25
+ */
26
+ export function requiresProdTypeToConfirm(envName) {
27
+ if (!envName) return false;
28
+ return /^(prod|production)$/i.test(envName);
29
+ }
30
+
31
+ /**
32
+ * If `envName` is a production environment, require the operator to type a
33
+ * confirmation string before continuing — even under -y. On cancel or a
34
+ * non-production env with no confirmation needed, behaves correctly:
35
+ * - non-prod env: returns immediately (no prompt).
36
+ * - prod env: prompts; a cancel exits(0); a correct entry returns.
37
+ *
38
+ * Exits the process (code 0) on cancel — callers do not need to handle it.
39
+ *
40
+ * @param {string} envName
41
+ * @param {object} [opts]
42
+ * @param {string} [opts.confirmValue=envName] - the exact string to type
43
+ * @param {string} [opts.actionLabel='this operation'] - verb shown in the prompt (e.g. 'restore')
44
+ * @param {boolean} [opts.yes=false] - whether -y was passed (only affects the warning copy)
45
+ * @returns {Promise<void>}
46
+ */
47
+ export async function confirmProdOrExit(envName, opts = {}) {
48
+ const { confirmValue = envName, actionLabel = 'this operation', yes = false } = opts;
49
+ if (!requiresProdTypeToConfirm(envName)) return;
50
+
51
+ if (yes) {
52
+ p.log.warn(
53
+ `A ${actionLabel} against a production environment still requires type-to-confirm, even with -y.`,
54
+ );
55
+ }
56
+
57
+ const doubleConfirm = await p.text({
58
+ message: `Type "${confirmValue}" to confirm ${actionLabel} of the production environment:`,
59
+ validate: (v) => (v !== confirmValue ? `Please type "${confirmValue}" to confirm` : undefined),
60
+ });
61
+ if (p.isCancel(doubleConfirm)) {
62
+ p.cancel('Operation cancelled.');
63
+ process.exit(0);
64
+ }
65
+ }
@@ -17,6 +17,7 @@
17
17
 
18
18
  import {
19
19
  AbortMultipartUploadCommand,
20
+ CopyObjectCommand,
20
21
  CreateBucketCommand,
21
22
  DeleteBucketCommand,
22
23
  DeleteObjectsCommand,
@@ -251,6 +252,74 @@ export class HetznerS3Provider {
251
252
  }
252
253
  }
253
254
 
255
+ /**
256
+ * Return true if the bucket contains at least one object under `prefix`.
257
+ * Tolerant of NoSuchBucket (returns false) so callers can probe a bucket
258
+ * that may not exist yet. Used by the deploy-time Pulumi-state migration to
259
+ * decide whether the dedicated state bucket has already been seeded and
260
+ * whether the legacy app bucket still holds state to copy over.
261
+ * @param {string} bucketName
262
+ * @param {string} prefix
263
+ * @returns {Promise<boolean>}
264
+ */
265
+ async hasObjectsWithPrefix(bucketName, prefix) {
266
+ try {
267
+ const res = await this._send(
268
+ new ListObjectsV2Command({ Bucket: bucketName, Prefix: prefix, MaxKeys: 1 }),
269
+ {
270
+ terminal: (err) => {
271
+ const status = err.$metadata?.httpStatusCode;
272
+ return err.name === 'NoSuchBucket' || status === 404;
273
+ },
274
+ },
275
+ );
276
+ return (res.KeyCount ?? res.Contents?.length ?? 0) > 0;
277
+ } catch (err) {
278
+ const status = err.$metadata?.httpStatusCode;
279
+ if (err.name === 'NoSuchBucket' || status === 404) return false;
280
+ throw err;
281
+ }
282
+ }
283
+
284
+ /**
285
+ * Copy every object under `prefix` from `srcBucket` to `destBucket`,
286
+ * preserving keys. Server-side copy (CopyObjectCommand) — no bytes transit
287
+ * the operator. Used to migrate legacy Pulumi state (the `.pulumi/` prefix)
288
+ * out of the app storage bucket into the dedicated state bucket. Both
289
+ * buckets must be in this provider's region.
290
+ * @param {string} srcBucket
291
+ * @param {string} destBucket
292
+ * @param {string} prefix
293
+ * @returns {Promise<{copied: number}>}
294
+ */
295
+ async copyPrefix(srcBucket, destBucket, prefix) {
296
+ let continuationToken;
297
+ let copied = 0;
298
+ do {
299
+ const listResult = await this._send(
300
+ new ListObjectsV2Command({
301
+ Bucket: srcBucket,
302
+ Prefix: prefix,
303
+ ContinuationToken: continuationToken,
304
+ }),
305
+ );
306
+ for (const obj of listResult.Contents || []) {
307
+ await this._send(
308
+ new CopyObjectCommand({
309
+ Bucket: destBucket,
310
+ Key: obj.Key,
311
+ // CopySource must be URL-encoded; keys contain `/` path separators
312
+ // (`.pulumi/stacks/...`) which encodeURI preserves.
313
+ CopySource: encodeURI(`${srcBucket}/${obj.Key}`),
314
+ }),
315
+ );
316
+ copied += 1;
317
+ }
318
+ continuationToken = listResult.IsTruncated ? listResult.NextContinuationToken : undefined;
319
+ } while (continuationToken);
320
+ return { copied };
321
+ }
322
+
254
323
  /**
255
324
  * Search all S3 regions for a bucket we own.
256
325
  * Each region's HeadBucket failure is intentionally swallowed — finding
@@ -674,3 +743,19 @@ export function sanitizeBucketName(projectName, suffix = 'storage') {
674
743
  .replace(/^-|-$/g, '') // Remove leading/trailing hyphens
675
744
  .slice(0, 63); // Truncate to max length
676
745
  }
746
+
747
+ /**
748
+ * Derive the dedicated Pulumi-state bucket name from the app storage bucket
749
+ * name. Pulumi state used to live in the app storage bucket, which created a
750
+ * destroy-time circular hazard (deleting the storage bucket nuked the state
751
+ * backend mid-destroy). State now lives in a separate bucket derived once,
752
+ * deterministically, from the app bucket — so the derivation is stable across
753
+ * deploy / scale / destroy and identical for both HA stacks (they share one
754
+ * state bucket).
755
+ *
756
+ * @param {string} appBucketName - The app storage bucket (e.g. `myapp-storage`)
757
+ * @returns {string} - Valid state bucket name (e.g. `myapp-storage-pulumi-state`)
758
+ */
759
+ export function deriveStateBucketName(appBucketName) {
760
+ return sanitizeBucketName(appBucketName, 'pulumi-state');
761
+ }
@@ -0,0 +1,59 @@
1
+ /**
2
+ * Shared retry/polling primitives for the exec layer.
3
+ *
4
+ * Two shapes cover every bespoke loop in the codebase:
5
+ * - runWithRetry: N attempts with explicit inter-attempt delays
6
+ * (kubectl transient retries, ssh transient retries, sideload/push settles).
7
+ * - pollUntil: probe under a time budget with capped exponential backoff
8
+ * (wait-for-k3s, wait-for-schema, wait-for-API-healthy, delete-until-404).
9
+ */
10
+ import { setTimeout as sleep } from 'node:timers/promises';
11
+
12
+ export async function runWithRetry(
13
+ fn,
14
+ { delaysMs = [5000, 5000], isTransient = () => true, onRetry } = {},
15
+ ) {
16
+ let attempt = 0;
17
+ // attempts = delaysMs.length + 1
18
+ for (;;) {
19
+ try {
20
+ return await fn(attempt);
21
+ } catch (err) {
22
+ if (attempt >= delaysMs.length || !isTransient(err)) throw err;
23
+ attempt += 1;
24
+ if (onRetry) onRetry(err, attempt);
25
+ await sleep(delaysMs[attempt - 1]);
26
+ }
27
+ }
28
+ }
29
+
30
+ export async function pollUntil(
31
+ probe,
32
+ {
33
+ budgetMs,
34
+ initialDelayMs = 2000,
35
+ maxDelayMs = 15_000,
36
+ backoffFactor = 2,
37
+ description = 'condition',
38
+ } = {},
39
+ ) {
40
+ const deadline = Date.now() + budgetMs;
41
+ let delay = initialDelayMs;
42
+ let lastErr;
43
+ for (;;) {
44
+ try {
45
+ const result = await probe();
46
+ if (result) return result;
47
+ lastErr = undefined;
48
+ } catch (err) {
49
+ lastErr = err;
50
+ }
51
+ if (Date.now() + delay > deadline) break;
52
+ await sleep(delay);
53
+ delay = Math.min(delay * backoffFactor, maxDelayMs);
54
+ }
55
+ const cause = lastErr ? `: ${lastErr.message}` : '';
56
+ const err = new Error(`Timed out after ${budgetMs}ms waiting for ${description}${cause}`);
57
+ err.cause = lastErr;
58
+ throw err;
59
+ }
@@ -0,0 +1,159 @@
1
+ /**
2
+ * Pure planning helpers for `vibecarbon scale` (k8s / k8s-ha).
3
+ *
4
+ * These functions carry NO side effects — no clack prompts, no SSH, no fetch,
5
+ * no Pulumi. They translate parsed flags and resolved config into the plain
6
+ * data shapes the orchestration code in src/scale.js consumes, which keeps the
7
+ * two behaviors that matter most testable in isolation:
8
+ *
9
+ * - the non-interactive `-yes`/`-type`/bounds flag matrix, and
10
+ * - the Pulumi programConfig assembly, whose output MUST stay byte-identical
11
+ * to what `deploy` produced or Pulumi plans a destructive replace of every
12
+ * node (etcd loss). See the extensive comments at the call site.
13
+ */
14
+ import { DEFAULT_WORKER_MAX, DEFAULT_WORKER_MIN } from './deploy/utils.js';
15
+
16
+ const REGION_TO_NETWORK_ZONE = {
17
+ nbg1: 'eu-central',
18
+ hel1: 'eu-central',
19
+ fsn1: 'eu-central',
20
+ hil: 'us-west',
21
+ ash: 'us-east',
22
+ sin: 'ap-southeast',
23
+ };
24
+
25
+ /**
26
+ * Resolve the non-interactive scale plan from parsed flags.
27
+ *
28
+ * Returns `{ changes, newValues }` for the two scripted branches:
29
+ * - `-yes -type X` → resize every node role to X (plus workerBounds when
30
+ * --min/--max are also present).
31
+ * - `-yes` + bounds → cluster-autoscaler bounds only.
32
+ * Returns `null` when neither branch matches, signalling the caller to fall
33
+ * back to the interactive multiselect prompt.
34
+ *
35
+ * `changes[]` elements are the string ids the caller dispatches on:
36
+ * 'masterType' | 'supabaseType' | 'workerType' | 'workerBounds'.
37
+ *
38
+ * envConfig is accepted for signature parity with the interactive caller; the
39
+ * non-interactive branches are a pure function of `parsed`.
40
+ *
41
+ * @param {{ yes?: boolean, type?: string|null, minWorkers?: number|null, maxWorkers?: number|null }} parsed
42
+ * @param {object} envConfig
43
+ * @returns {{ changes: string[], newValues: Record<string, unknown> } | null}
44
+ */
45
+ export function planK8sScaleChanges(parsed, _envConfig) {
46
+ const newValues = {};
47
+
48
+ if (parsed.yes && parsed.type) {
49
+ // `--type X` means "scale every node role to X" for parity with compose's
50
+ // `--type X` (which resizes the one VPS).
51
+ const changes = ['masterType', 'supabaseType', 'workerType'];
52
+ newValues.masterType = parsed.type;
53
+ newValues.supabaseType = parsed.type;
54
+ newValues.workerType = parsed.type;
55
+ // Allow --type and bounds to be combined in one invocation.
56
+ if (parsed.minWorkers != null || parsed.maxWorkers != null) {
57
+ changes.push('workerBounds');
58
+ if (parsed.minWorkers != null) newValues.minWorkers = parsed.minWorkers;
59
+ if (parsed.maxWorkers != null) newValues.maxWorkers = parsed.maxWorkers;
60
+ }
61
+ return { changes, newValues };
62
+ }
63
+
64
+ if (parsed.yes && (parsed.minWorkers != null || parsed.maxWorkers != null)) {
65
+ // Bounds-only path: --min-workers and/or --max-workers without --type.
66
+ const changes = ['workerBounds'];
67
+ if (parsed.minWorkers != null) newValues.minWorkers = parsed.minWorkers;
68
+ if (parsed.maxWorkers != null) newValues.maxWorkers = parsed.maxWorkers;
69
+ return { changes, newValues };
70
+ }
71
+
72
+ return null;
73
+ }
74
+
75
+ /**
76
+ * Assemble the Hetzner-k8s Pulumi programConfig. Its output MUST be
77
+ * byte-identical to what deployK3s passed at deploy time — every field that
78
+ * flows into a Server's `sshKeys`, `firewallIds`, `labels`, or `userData` is a
79
+ * tripwire for a destructive replace, and replacing master wipes etcd.
80
+ *
81
+ * @param {object} inputs
82
+ * @param {string} inputs.projectName
83
+ * @param {string} inputs.environment Cluster env (e.g. `prod`, `prod-primary`).
84
+ * @param {string} inputs.sshPublicKey
85
+ * @param {string[]} inputs.allowedCidrs Operator CIDRs (SSH + k8s API).
86
+ * @param {string|undefined} inputs.existingSshKeyId
87
+ * @param {string} inputs.location Cluster region.
88
+ * @param {Record<string, unknown>} inputs.newValues Plan output; may set *Type/*Workers.
89
+ * @param {string} inputs.currentMasterType
90
+ * @param {string} inputs.currentSupabaseType
91
+ * @param {string} inputs.currentWorkerType
92
+ * @param {number|undefined} inputs.persistedMinWorkers envConfig.minWorkers
93
+ * @param {number|undefined} inputs.persistedMaxWorkers envConfig.maxWorkers
94
+ * @param {string} inputs.k3sVersion
95
+ * @param {Record<string, string>} inputs.labels
96
+ * @param {string} inputs.apiToken
97
+ * @returns {object}
98
+ */
99
+ export function buildProgramConfig({
100
+ projectName,
101
+ environment,
102
+ sshPublicKey,
103
+ allowedCidrs,
104
+ existingSshKeyId,
105
+ location,
106
+ newValues,
107
+ currentMasterType,
108
+ currentSupabaseType,
109
+ currentWorkerType,
110
+ persistedMinWorkers,
111
+ persistedMaxWorkers,
112
+ k3sVersion,
113
+ labels,
114
+ apiToken,
115
+ }) {
116
+ return {
117
+ projectName,
118
+ environment,
119
+ sshPublicKey,
120
+ allowedSshIps: allowedCidrs,
121
+ allowedK8sApiIps: allowedCidrs,
122
+ // HA deploys share one Hetzner SshKey across primary + standby and pass
123
+ // `existingSshKeyId` so neither stack manages the resource. Single-cluster
124
+ // lets Pulumi own the SshKey (existingSshKeyId = undefined).
125
+ existingSshKeyId,
126
+ location,
127
+ networkZone: REGION_TO_NETWORK_ZONE[location] || 'eu-central',
128
+ masterServerType: newValues.masterType ?? currentMasterType,
129
+ supabaseServerType: newValues.supabaseType ?? currentSupabaseType,
130
+ workerServerType: newValues.workerType ?? currentWorkerType,
131
+ // minWorkers is Pulumi's static worker floor; maxWorkers flows to the CA
132
+ // patch (not consumed by Pulumi). Replay persisted values when the operator
133
+ // didn't override, keeping Pulumi's worker-pool plan a no-op.
134
+ minWorkers: newValues.minWorkers ?? persistedMinWorkers ?? DEFAULT_WORKER_MIN,
135
+ maxWorkers: newValues.maxWorkers ?? persistedMaxWorkers ?? DEFAULT_WORKER_MAX,
136
+ k3sVersion,
137
+ labels,
138
+ apiToken,
139
+ };
140
+ }
141
+
142
+ /**
143
+ * Rewrite the cluster-autoscaler container's `--nodes=` arg to a freshly
144
+ * formatted spec, preserving every other arg. Appends the flag if the deploy
145
+ * container somehow shipped without it. Pure list transform.
146
+ *
147
+ * @param {string[]} argList Current container args (from kubectl jsonpath).
148
+ * @param {string} caNodesSpec New `<min>:<max>:<type>:<region>:<pool>` spec.
149
+ * @returns {string[]}
150
+ */
151
+ export function updateCaNodesArgs(argList, caNodesSpec) {
152
+ const updated = (Array.isArray(argList) ? argList : []).map((a) =>
153
+ typeof a === 'string' && a.startsWith('--nodes=') ? `--nodes=${caNodesSpec}` : a,
154
+ );
155
+ if (!updated.some((a) => typeof a === 'string' && a.startsWith('--nodes='))) {
156
+ updated.push(`--nodes=${caNodesSpec}`);
157
+ }
158
+ return updated;
159
+ }
package/src/lib/ssh.js CHANGED
@@ -8,16 +8,22 @@
8
8
  * of dynamic tokens), or
9
9
  * 2. Use sshRunScript to execute a bash script SCP'd as a file.
10
10
  *
11
- * Host-key checking: first-provision callers pass { firstConnect: true }
12
- * which switches StrictHostKeyChecking to accept-new (TOFU). All other
13
- * calls use strict checking against a per-environment known_hosts file.
11
+ * Host-key checking: EVERY call pins against a per-environment known_hosts
12
+ * file there is no /dev/null + StrictHostKeyChecking=no bypass. When a
13
+ * caller passes an explicit `env`, first-provision callers pass
14
+ * { firstConnect: true } to accept-new (TOFU) once and strict-check ('yes')
15
+ * thereafter. Callers that only have `(ip, sshKeyPath)` get the per-env
16
+ * known_hosts derived from the key path (knownHostsPathForKey) with
17
+ * StrictHostKeyChecking=accept-new — which still REJECTS a changed key for an
18
+ * already-pinned host (MITM on an established env fails) while TOFU-ing a fresh
19
+ * or recycled IP. The operator's ~/.ssh/known_hosts is never touched.
14
20
  */
15
21
 
16
22
  import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
17
23
  import { tmpdir } from 'node:os';
18
24
  import { join } from 'node:path';
19
25
  import { runCommandAsync } from './command.js';
20
- import { knownHostsPath } from './host-keys.js';
26
+ import { knownHostsPath, knownHostsPathForKey } from './host-keys.js';
21
27
  import { shEscape } from './shell.js';
22
28
 
23
29
  // k3s writes its kubeconfig here; kubectl doesn't find it by default in SSH sessions.
@@ -53,30 +59,36 @@ export function getSSHKeyPath(environment) {
53
59
  }
54
60
 
55
61
  /**
56
- * Build the SSH `-o` option args for host-key checking.
62
+ * Build the SSH `-o` option args for host-key checking. Always pins against a
63
+ * per-environment known_hosts file under .vibecarbon/ — never /dev/null.
64
+ * GlobalKnownHostsFile=/dev/null ignores the system-wide file; the operator's
65
+ * ~/.ssh/known_hosts is never read or polluted (Hetzner recycles public IPs,
66
+ * and a stale entry there would reject an unrelated reconnect).
57
67
  *
58
- * - With `env`: StrictHostKeyChecking=yes against .vibecarbon/known_hosts_<env>
59
- * (or accept-new if firstConnect=true used during first provision).
68
+ * - With explicit `env`: pin to .vibecarbon/known_hosts_<env>,
69
+ * StrictHostKeyChecking=yes (or accept-new when firstConnect=true, used
70
+ * during first provision).
60
71
  *
61
- * - Without `env`: bypass the user's known_hosts with UserKnownHostsFile=
62
- * /dev/null and StrictHostKeyChecking=no, matching the deploy path's long-
63
- * standing SSH_OPTS in compose/index.js. This keeps backup/restore/scale/
64
- * failover consistent with the deploy flow and avoids the Hetzner recycled-
65
- * IP footgun: destroy-then-redeploy gives you the same public IP with a new
66
- * host key, and a stale TOFU entry in ~/.ssh/known_hosts would refuse the
67
- * reconnect. H-1 (tracked in docs/superpowers/specs/2026-04-17-security-
68
- * remediation-design.md) replaces this with env-scoped pinned known_hosts
69
- * across every CLI command — callers pass `env` once the deploy path
70
- * populates .vibecarbon/known_hosts_<env>.
72
+ * - With only `sshKeyPath` (backup/restore/scale/failover only receive the key
73
+ * path, not an env): derive .vibecarbon/known_hosts_<env> from the key path
74
+ * (knownHostsPathForKey) and use StrictHostKeyChecking=accept-new. accept-new
75
+ * bootstraps an empty pin and TOFU's a fresh/recycled IP, but REJECTS a
76
+ * changed key for an already-pinned host so a MITM against an established
77
+ * env fails, while destroy→redeploy (same IP, new key, re-seeded on
78
+ * provision) does not spuriously hard-fail.
71
79
  */
72
- function sshHostKeyOpts(env, { firstConnect = false } = {}) {
80
+ function sshHostKeyOpts(env, { firstConnect = false, sshKeyPath } = {}) {
73
81
  const opts = [];
74
82
  if (env) {
75
83
  opts.push('-o', `UserKnownHostsFile=${knownHostsPath(env)}`);
84
+ opts.push('-o', 'GlobalKnownHostsFile=/dev/null');
76
85
  opts.push('-o', `StrictHostKeyChecking=${firstConnect ? 'accept-new' : 'yes'}`);
86
+ } else if (sshKeyPath) {
87
+ opts.push('-o', `UserKnownHostsFile=${knownHostsPathForKey(sshKeyPath)}`);
88
+ opts.push('-o', 'GlobalKnownHostsFile=/dev/null');
89
+ opts.push('-o', 'StrictHostKeyChecking=accept-new');
77
90
  } else {
78
- opts.push('-o', 'UserKnownHostsFile=/dev/null');
79
- opts.push('-o', 'StrictHostKeyChecking=no');
91
+ throw new Error('sshHostKeyOpts requires either `env` or `sshKeyPath` for host-key pinning');
80
92
  }
81
93
  opts.push('-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10');
82
94
  // ConnectTimeout only covers TCP connect — once the socket is open, ssh
@@ -94,18 +106,6 @@ function sshHostKeyOpts(env, { firstConnect = false } = {}) {
94
106
  return opts;
95
107
  }
96
108
 
97
- /**
98
- * The no-host-key-pinning SSH option set as a single space-joined string,
99
- * for the compose deploy path's string-command runners (sshRun/sshRunAsync
100
- * in compose/index.js) and raw `ssh ${SSH_OPTS}` interpolations.
101
- *
102
- * Derived from sshHostKeyOpts() so the string form can never drift from the
103
- * argv form — the historical footgun was compose/ha.js hand-copying this
104
- * string and silently dropping the ServerAlive keepalives, letting a
105
- * banner-exchange hang stall an HA deploy for ~600s.
106
- */
107
- export const NO_PIN_SSH_OPTS = sshHostKeyOpts().join(' ');
108
-
109
109
  /**
110
110
  * Execute a command on a remote server via SSH using argv form.
111
111
  *
@@ -143,7 +143,7 @@ export async function sshRun(ip, sshKeyPath, argv, options = {}) {
143
143
  'ssh',
144
144
  '-i',
145
145
  sshKeyPath,
146
- ...sshHostKeyOpts(env, { firstConnect }),
146
+ ...sshHostKeyOpts(env, { firstConnect, sshKeyPath }),
147
147
  '--',
148
148
  `root@${ip}`,
149
149
  remoteCmd,
@@ -189,7 +189,7 @@ export async function scpDownload(ip, sshKeyPath, remotePath, localPath, options
189
189
  'scp',
190
190
  '-i',
191
191
  sshKeyPath,
192
- ...sshHostKeyOpts(env, { firstConnect }),
192
+ ...sshHostKeyOpts(env, { firstConnect, sshKeyPath }),
193
193
  '--',
194
194
  `root@${ip}:${remotePath}`,
195
195
  localPath,
@@ -206,7 +206,7 @@ export async function scpUpload(ip, sshKeyPath, localPath, remotePath, options =
206
206
  'scp',
207
207
  '-i',
208
208
  sshKeyPath,
209
- ...sshHostKeyOpts(env, { firstConnect }),
209
+ ...sshHostKeyOpts(env, { firstConnect, sshKeyPath }),
210
210
  '--',
211
211
  localPath,
212
212
  `root@${ip}:${remotePath}`,
package/src/restore.js CHANGED
@@ -19,6 +19,7 @@
19
19
  import { existsSync } from 'node:fs';
20
20
  import { basename } from 'node:path';
21
21
  import * as p from '@clack/prompts';
22
+ import { identifyServers } from './failover.js';
22
23
  import { formatInstant } from './lib/backup-format.js';
23
24
  import { renderHelp } from './lib/cli/help.js';
24
25
  import { parseFlags } from './lib/cli/parse-flags.js';
@@ -31,9 +32,14 @@ import { loadCredentials, loadProjectConfig } from './lib/config.js';
31
32
  // while still accepting injected mocks for unit tests.
32
33
  import { configureStandbyReplication as realConfigureStandbyReplication } from './lib/deploy/compose/ha.js';
33
34
  import { restoreCompose as realRestoreCompose } from './lib/deploy/compose/index.js';
34
- import { requireLicense } from './lib/licensing/index.js';
35
+ import {
36
+ readK8sReplicationState as realReadK8sReplicationState,
37
+ reseedStandbyFromPrimary as realReseedStandbyFromPrimary,
38
+ verifyStreaming,
39
+ } from './lib/deploy/replication.js';
35
40
  import { ensureOperatorIpAccess } from './lib/operator-ip.js';
36
41
  import { perfAsync } from './lib/perf.js';
42
+ import { confirmProdOrExit } from './lib/prod-confirm.js';
37
43
  import { assertInProjectDir } from './lib/project-guard.js';
38
44
  import { getPostgresPod, getSSHKeyPath, sshKubectl } from './lib/ssh.js';
39
45
  import { createTracker } from './lib/tracker.js';
@@ -245,7 +251,6 @@ export async function run(args) {
245
251
  // restore` from a parent directory emits the canonical message.
246
252
  assertInProjectDir();
247
253
 
248
- requireLicense('restore');
249
254
  printBanner();
250
255
  p.intro(`${c.bold('vibecarbon restore')} ${c.dim(`v${VERSION}`)}`);
251
256
 
@@ -392,19 +397,15 @@ export async function run(args) {
392
397
  p.cancel('Operation cancelled.');
393
398
  process.exit(0);
394
399
  }
395
-
396
- if (envName === 'prod' || envName === 'production') {
397
- const doubleConfirm = await p.text({
398
- message: `Type "${envName}" to confirm restoration of production database:`,
399
- validate: (v) => (v !== envName ? `Please type "${envName}" to confirm` : undefined),
400
- });
401
- if (p.isCancel(doubleConfirm)) {
402
- p.cancel('Operation cancelled.');
403
- process.exit(0);
404
- }
405
- }
406
400
  }
407
401
 
402
+ // SECURITY: the production type-to-confirm runs UNCONDITIONALLY — even with
403
+ // -y — so `restore -y prod` can never silently overwrite the production
404
+ // database. This is deliberately OUTSIDE the `!values.y` block above (the
405
+ // soft y/N confirm is skippable with -y; this hard gate is not). Mirrors
406
+ // destroy's prod guard; shared helper so failover can reuse it.
407
+ await confirmProdOrExit(envName, { actionLabel: 'restore', yes: !!values.y });
408
+
408
409
  // Dispatch by mode
409
410
  const tracker = createTracker('restore', { environment: envName });
410
411
  const s = tracker.spinner();
@@ -427,6 +428,7 @@ export async function run(args) {
427
428
  envName,
428
429
  serverIp,
429
430
  sshKeyPath,
431
+ envConfig,
430
432
  });
431
433
  }
432
434
  } catch (error) {
@@ -645,7 +647,21 @@ async function setRestoreMarker(ip, sshKeyPath, value) {
645
647
  ]);
646
648
  }
647
649
 
648
- async function runK8sRestore({ s, chosenSource, envName, serverIp, sshKeyPath }) {
650
+ export async function runK8sRestore({
651
+ s,
652
+ chosenSource,
653
+ envName,
654
+ serverIp,
655
+ sshKeyPath,
656
+ envConfig = {},
657
+ // Injected so the k8s-HA standby re-seed dispatch is unit-testable without a
658
+ // real cluster (mirrors runComposeRestore). Production passes none.
659
+ reseedStandbyFromPrimary = realReseedStandbyFromPrimary,
660
+ readReplicationState = realReadK8sReplicationState,
661
+ // Pacing for the post-reseed streaming verify — injectable so tests don't wait
662
+ // on real timers. Production uses the default setTimeout-backed sleep.
663
+ verifySleep = (ms) => new Promise((r) => setTimeout(r, ms)),
664
+ }) {
649
665
  if (chosenSource.kind === 'local') {
650
666
  throw new Error(
651
667
  'k8s restore is wal-g-based and pulls from S3 — local backup files are not supported. Use `-source latest` (or a timestamp for PITR).',
@@ -695,6 +711,78 @@ async function runK8sRestore({ s, chosenSource, envName, serverIp, sshKeyPath })
695
711
 
696
712
  p.log.success('Restore completed successfully');
697
713
  p.log.info('The application has been scaled back up.');
714
+
715
+ // k8s-HA: the standby is a SEPARATE cluster deployed with `restore: null` — a
716
+ // streaming replica, never wal-g-restored (a wal-g fetch on the standby would
717
+ // create a divergent timeline; see k8s/ha/index.js). This restore rewound the
718
+ // PRIMARY to an earlier LSN, so the standby now has WAL AHEAD of the primary
719
+ // and CANNOT resume streaming (timeline divergence).
720
+ //
721
+ // Finding #2: instead of the old warn-and-point-to-`deploy`, perform a
722
+ // first-class VERIFIED re-seed of the standby using the SHARED reseed
723
+ // primitive (the same hardened staged-basebackup+atomic-swap the failover path
724
+ // uses), then confirm streaming from the restored primary. Fail loudly if it
725
+ // can't complete — never silently leave a diverged standby.
726
+ const isK8sHA = !!(envConfig.ha?.enabled || envConfig.ha === true || envConfig.secondaryRegion);
727
+ if (isK8sHA) {
728
+ const servers = identifyServers(envName, envConfig, {});
729
+ const standbyIp = servers?.standby?.ip;
730
+ const primarySupabaseIp = servers?.primary?.supabaseIp || servers?.primary?.ip;
731
+ if (!standbyIp || !primarySupabaseIp) {
732
+ // We can't locate the standby / primary source — fail loudly rather than
733
+ // report a clean restore while the standby silently diverges.
734
+ throw new Error(
735
+ 'HA restore: the primary was restored, but the standby cluster or the primary ' +
736
+ 'supabase IP could not be resolved from config, so the standby could NOT be ' +
737
+ `re-seeded and now has a DIVERGED timeline. DR is not guaranteed. Resync with ` +
738
+ `\`vibecarbon deploy ${envName}\` (re-runs the verified replication setup).`,
739
+ );
740
+ }
741
+
742
+ s.start('Re-seeding standby cluster from restored primary (pg_basebackup)');
743
+ let reseedResult;
744
+ try {
745
+ reseedResult = await reseedStandbyFromPrimary(standbyIp, sshKeyPath, primarySupabaseIp);
746
+ } catch (err) {
747
+ s.stop('Standby re-seed failed');
748
+ throw new Error(
749
+ `HA restore: the primary was restored and is serving, but re-seeding the standby ` +
750
+ `FAILED: ${err.message}\nThe standby has a DIVERGED timeline and is NOT replicating — ` +
751
+ `DR is not guaranteed. Resync with \`vibecarbon deploy ${envName}\`.`,
752
+ );
753
+ }
754
+ if (reseedResult === 'skipped') {
755
+ s.stop('Standby re-seed skipped — primary unreachable from standby');
756
+ throw new Error(
757
+ 'HA restore: the standby could NOT reach the restored primary to re-seed ' +
758
+ '(cross-cluster connectivity). The standby has a DIVERGED timeline and is NOT ' +
759
+ `replicating — DR is not guaranteed. Verify the replication firewall/network, then ` +
760
+ `resync with \`vibecarbon deploy ${envName}\`.`,
761
+ );
762
+ }
763
+ s.stop('Standby re-seeded from restored primary');
764
+
765
+ // Confirm streaming from the restored primary — a re-seed that boots but
766
+ // never streams is still a diverged standby.
767
+ s.start('Verifying standby is streaming from restored primary');
768
+ const { streaming, lastState } = await verifyStreaming({
769
+ readState: async () => readReplicationState(servers.primary.ip, sshKeyPath),
770
+ attempts: 15,
771
+ delaysMs: [1000, 2000, 3000],
772
+ sleep: verifySleep,
773
+ });
774
+ if (!streaming) {
775
+ s.stop('Standby streaming NOT confirmed');
776
+ throw new Error(
777
+ `HA restore: the standby was re-seeded but streaming replication is NOT confirmed ` +
778
+ `(last pg_stat_replication.state=${
779
+ lastState ? JSON.stringify(lastState) : 'no replica connected'
780
+ }). DR is not guaranteed. Resync with \`vibecarbon deploy ${envName}\`.`,
781
+ );
782
+ }
783
+ s.stop('Standby streaming from restored primary — DR restored');
784
+ }
785
+
698
786
  p.log.info(`Verify with: ${c.info(`vibecarbon status ${envName}`)}`);
699
787
  }
700
788