vibecarbon 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +110 -663
- package/README.md +4 -4
- package/carbon/.env.example +11 -0
- package/carbon/backup/compose-backup.sh +8 -0
- package/carbon/docker-compose.yml +11 -0
- package/carbon/ha/primary-init.sql +10 -2
- package/carbon/k8s/base/backup/cronjob.yaml +9 -0
- package/carbon/k8s/base/repl-gateway/kustomization.yaml +11 -0
- package/carbon/k8s/base/repl-gateway/repl-gateway.yaml +92 -0
- package/carbon/k8s/values/supabase.values.yaml +6 -1
- package/carbon/src/client/lib/supabase.ts +41 -0
- package/carbon/src/client/locales/de.json +27 -12
- package/carbon/src/client/locales/en.json +30 -15
- package/carbon/src/client/locales/es.json +27 -12
- package/carbon/src/client/locales/fr.json +27 -12
- package/carbon/src/client/locales/pt.json +27 -12
- package/carbon/src/client/pages/Checkout.tsx +1 -1
- package/carbon/src/server/billing/providers/paddle.ts +21 -2
- package/carbon/src/server/billing/providers/polar.ts +14 -0
- package/carbon/src/server/index.ts +12 -9
- package/carbon/src/server/lib/client-ip.ts +68 -0
- package/carbon/src/server/lib/env.ts +20 -6
- package/carbon/src/server/lib/rate-limiter.ts +11 -65
- package/carbon/src/server/lib/supabase.ts +10 -4
- package/carbon/src/server/middleware/requireOrgRole.ts +76 -0
- package/carbon/src/server/middleware/requirePlan.ts +64 -34
- package/carbon/src/server/middleware/requireSuperAdmin.ts +26 -0
- package/carbon/src/server/routes/v1/admin/contact.ts +5 -20
- package/carbon/src/server/routes/v1/admin/jobs.ts +6 -20
- package/carbon/src/server/routes/v1/admin/newsletter.ts +26 -23
- package/carbon/src/server/routes/v1/auth.ts +5 -2
- package/carbon/src/server/routes/v1/billing.ts +24 -64
- package/carbon/src/server/routes/v1/index.ts +8 -1
- package/carbon/src/server/routes/v1/newsletter.ts +18 -12
- package/carbon/src/server/routes/v1/theme.ts +36 -15
- package/carbon/supabase/migrations/00003_pg_cron.sql +2 -2
- package/carbon/supabase/migrations/00004_contact_newsletter.sql +24 -14
- package/carbon/tests/component/supabase-env-guard.test.ts +54 -0
- package/carbon/tests/integration/server/middleware/rate-limit-ip-spoof.test.ts +48 -0
- package/carbon/tests/integration/server/middleware/require-plan.test.ts +101 -0
- package/carbon/tests/integration/server/routes/admin-newsletter.test.ts +93 -0
- package/carbon/tests/integration/server/routes/billing-subscription-idor.test.ts +92 -0
- package/carbon/tests/integration/server/routes/newsletter-unsubscribe.test.ts +65 -0
- package/carbon/tests/integration/server/routes/theme-validation.test.ts +66 -0
- package/carbon/tests/unit/server/client-ip.test.ts +76 -0
- package/carbon/tests/unit/server/env-bool.test.ts +63 -0
- package/carbon/tests/unit/server/paddle-webhook.test.ts +57 -0
- package/carbon/tests/unit/server/polar-webhook.test.ts +63 -0
- package/carbon/tests/unit/server/supabase-client.test.ts +30 -0
- package/carbon/volumes/db/wal-archive.sh +36 -0
- package/package.json +2 -2
- package/src/backup.js +0 -2
- package/src/cli.js +16 -0
- package/src/deploy.js +16 -5
- package/src/destroy.js +968 -886
- package/src/failover.js +118 -237
- package/src/lib/command.js +19 -1
- package/src/lib/deploy/compose/ha.js +180 -102
- package/src/lib/deploy/compose/index.js +106 -57
- package/src/lib/deploy/compose/single.js +421 -0
- package/src/lib/deploy/image.js +38 -30
- package/src/lib/deploy/k8s/ha/index.js +397 -150
- package/src/lib/deploy/k8s/k3s.js +411 -279
- package/src/lib/deploy/orchestrator.js +274 -453
- package/src/lib/deploy/remote-build.js +8 -1
- package/src/lib/deploy/replication.js +540 -0
- package/src/lib/deploy/state.js +62 -5
- package/src/lib/deploy/tier-registry.js +30 -0
- package/src/lib/deploy/utils.js +30 -2
- package/src/lib/deploy/wireguard.js +146 -0
- package/src/lib/host-keys.js +120 -5
- package/src/lib/iac/index.js +57 -23
- package/src/lib/licensing/gate.js +59 -0
- package/src/lib/licensing/index.js +1 -1
- package/src/lib/licensing/tiers.js +6 -5
- package/src/lib/prod-confirm.js +65 -0
- package/src/lib/providers/hetzner-s3.js +85 -0
- package/src/lib/retry.js +59 -0
- package/src/lib/scale-plan.js +159 -0
- package/src/lib/ssh.js +35 -35
- package/src/restore.js +102 -14
- package/src/scale.js +105 -133
- package/src/status.js +122 -0
- package/src/upgrade.js +0 -4
- package/carbon/ha/activate-standby.sh +0 -52
- package/carbon/ha/standby-init.sh +0 -74
- package/carbon/src/server/lib/request.ts +0 -34
- package/src/lib/pod-backups.js +0 -53
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared production type-to-confirm guard.
|
|
3
|
+
*
|
|
4
|
+
* Destructive commands (destroy, restore, failover) must NOT let a `-y` /
|
|
5
|
+
* `--yes` flag silently blow away a production environment. This module is the
|
|
6
|
+
* single source of truth for "which envs are production" and for the
|
|
7
|
+
* type-to-confirm prompt that runs UNCONDITIONALLY (independent of -y) on those
|
|
8
|
+
* envs — matching `destroy`'s long-standing behavior.
|
|
9
|
+
*
|
|
10
|
+
* `src/destroy.js` still carries its own copy of `requiresProdTypeToConfirm`
|
|
11
|
+
* for its bespoke `projectName-envName` slug; new callers (restore, and later
|
|
12
|
+
* failover) should import from here.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import * as p from '@clack/prompts';
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Returns true for environment names that require a type-to-confirm prompt even
|
|
19
|
+
* when -y is passed. Currently: `prod` and `production` (case-insensitive).
|
|
20
|
+
* A trailing/leading qualifier (e.g. `prod-backup`, `production-us`) is NOT
|
|
21
|
+
* treated as production — it is a distinct environment.
|
|
22
|
+
*
|
|
23
|
+
* @param {string | null | undefined} envName
|
|
24
|
+
* @returns {boolean}
|
|
25
|
+
*/
|
|
26
|
+
export function requiresProdTypeToConfirm(envName) {
|
|
27
|
+
if (!envName) return false;
|
|
28
|
+
return /^(prod|production)$/i.test(envName);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* If `envName` is a production environment, require the operator to type a
|
|
33
|
+
* confirmation string before continuing — even under -y. On cancel or a
|
|
34
|
+
* non-production env with no confirmation needed, behaves correctly:
|
|
35
|
+
* - non-prod env: returns immediately (no prompt).
|
|
36
|
+
* - prod env: prompts; a cancel exits(0); a correct entry returns.
|
|
37
|
+
*
|
|
38
|
+
* Exits the process (code 0) on cancel — callers do not need to handle it.
|
|
39
|
+
*
|
|
40
|
+
* @param {string} envName
|
|
41
|
+
* @param {object} [opts]
|
|
42
|
+
* @param {string} [opts.confirmValue=envName] - the exact string to type
|
|
43
|
+
* @param {string} [opts.actionLabel='this operation'] - verb shown in the prompt (e.g. 'restore')
|
|
44
|
+
* @param {boolean} [opts.yes=false] - whether -y was passed (only affects the warning copy)
|
|
45
|
+
* @returns {Promise<void>}
|
|
46
|
+
*/
|
|
47
|
+
export async function confirmProdOrExit(envName, opts = {}) {
|
|
48
|
+
const { confirmValue = envName, actionLabel = 'this operation', yes = false } = opts;
|
|
49
|
+
if (!requiresProdTypeToConfirm(envName)) return;
|
|
50
|
+
|
|
51
|
+
if (yes) {
|
|
52
|
+
p.log.warn(
|
|
53
|
+
`A ${actionLabel} against a production environment still requires type-to-confirm, even with -y.`,
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const doubleConfirm = await p.text({
|
|
58
|
+
message: `Type "${confirmValue}" to confirm ${actionLabel} of the production environment:`,
|
|
59
|
+
validate: (v) => (v !== confirmValue ? `Please type "${confirmValue}" to confirm` : undefined),
|
|
60
|
+
});
|
|
61
|
+
if (p.isCancel(doubleConfirm)) {
|
|
62
|
+
p.cancel('Operation cancelled.');
|
|
63
|
+
process.exit(0);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
import {
|
|
19
19
|
AbortMultipartUploadCommand,
|
|
20
|
+
CopyObjectCommand,
|
|
20
21
|
CreateBucketCommand,
|
|
21
22
|
DeleteBucketCommand,
|
|
22
23
|
DeleteObjectsCommand,
|
|
@@ -251,6 +252,74 @@ export class HetznerS3Provider {
|
|
|
251
252
|
}
|
|
252
253
|
}
|
|
253
254
|
|
|
255
|
+
/**
|
|
256
|
+
* Return true if the bucket contains at least one object under `prefix`.
|
|
257
|
+
* Tolerant of NoSuchBucket (returns false) so callers can probe a bucket
|
|
258
|
+
* that may not exist yet. Used by the deploy-time Pulumi-state migration to
|
|
259
|
+
* decide whether the dedicated state bucket has already been seeded and
|
|
260
|
+
* whether the legacy app bucket still holds state to copy over.
|
|
261
|
+
* @param {string} bucketName
|
|
262
|
+
* @param {string} prefix
|
|
263
|
+
* @returns {Promise<boolean>}
|
|
264
|
+
*/
|
|
265
|
+
async hasObjectsWithPrefix(bucketName, prefix) {
|
|
266
|
+
try {
|
|
267
|
+
const res = await this._send(
|
|
268
|
+
new ListObjectsV2Command({ Bucket: bucketName, Prefix: prefix, MaxKeys: 1 }),
|
|
269
|
+
{
|
|
270
|
+
terminal: (err) => {
|
|
271
|
+
const status = err.$metadata?.httpStatusCode;
|
|
272
|
+
return err.name === 'NoSuchBucket' || status === 404;
|
|
273
|
+
},
|
|
274
|
+
},
|
|
275
|
+
);
|
|
276
|
+
return (res.KeyCount ?? res.Contents?.length ?? 0) > 0;
|
|
277
|
+
} catch (err) {
|
|
278
|
+
const status = err.$metadata?.httpStatusCode;
|
|
279
|
+
if (err.name === 'NoSuchBucket' || status === 404) return false;
|
|
280
|
+
throw err;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Copy every object under `prefix` from `srcBucket` to `destBucket`,
|
|
286
|
+
* preserving keys. Server-side copy (CopyObjectCommand) — no bytes transit
|
|
287
|
+
* the operator. Used to migrate legacy Pulumi state (the `.pulumi/` prefix)
|
|
288
|
+
* out of the app storage bucket into the dedicated state bucket. Both
|
|
289
|
+
* buckets must be in this provider's region.
|
|
290
|
+
* @param {string} srcBucket
|
|
291
|
+
* @param {string} destBucket
|
|
292
|
+
* @param {string} prefix
|
|
293
|
+
* @returns {Promise<{copied: number}>}
|
|
294
|
+
*/
|
|
295
|
+
async copyPrefix(srcBucket, destBucket, prefix) {
|
|
296
|
+
let continuationToken;
|
|
297
|
+
let copied = 0;
|
|
298
|
+
do {
|
|
299
|
+
const listResult = await this._send(
|
|
300
|
+
new ListObjectsV2Command({
|
|
301
|
+
Bucket: srcBucket,
|
|
302
|
+
Prefix: prefix,
|
|
303
|
+
ContinuationToken: continuationToken,
|
|
304
|
+
}),
|
|
305
|
+
);
|
|
306
|
+
for (const obj of listResult.Contents || []) {
|
|
307
|
+
await this._send(
|
|
308
|
+
new CopyObjectCommand({
|
|
309
|
+
Bucket: destBucket,
|
|
310
|
+
Key: obj.Key,
|
|
311
|
+
// CopySource must be URL-encoded; keys contain `/` path separators
|
|
312
|
+
// (`.pulumi/stacks/...`) which encodeURI preserves.
|
|
313
|
+
CopySource: encodeURI(`${srcBucket}/${obj.Key}`),
|
|
314
|
+
}),
|
|
315
|
+
);
|
|
316
|
+
copied += 1;
|
|
317
|
+
}
|
|
318
|
+
continuationToken = listResult.IsTruncated ? listResult.NextContinuationToken : undefined;
|
|
319
|
+
} while (continuationToken);
|
|
320
|
+
return { copied };
|
|
321
|
+
}
|
|
322
|
+
|
|
254
323
|
/**
|
|
255
324
|
* Search all S3 regions for a bucket we own.
|
|
256
325
|
* Each region's HeadBucket failure is intentionally swallowed — finding
|
|
@@ -674,3 +743,19 @@ export function sanitizeBucketName(projectName, suffix = 'storage') {
|
|
|
674
743
|
.replace(/^-|-$/g, '') // Remove leading/trailing hyphens
|
|
675
744
|
.slice(0, 63); // Truncate to max length
|
|
676
745
|
}
|
|
746
|
+
|
|
747
|
+
/**
|
|
748
|
+
* Derive the dedicated Pulumi-state bucket name from the app storage bucket
|
|
749
|
+
* name. Pulumi state used to live in the app storage bucket, which created a
|
|
750
|
+
* destroy-time circular hazard (deleting the storage bucket nuked the state
|
|
751
|
+
* backend mid-destroy). State now lives in a separate bucket derived once,
|
|
752
|
+
* deterministically, from the app bucket — so the derivation is stable across
|
|
753
|
+
* deploy / scale / destroy and identical for both HA stacks (they share one
|
|
754
|
+
* state bucket).
|
|
755
|
+
*
|
|
756
|
+
* @param {string} appBucketName - The app storage bucket (e.g. `myapp-storage`)
|
|
757
|
+
* @returns {string} - Valid state bucket name (e.g. `myapp-storage-pulumi-state`)
|
|
758
|
+
*/
|
|
759
|
+
export function deriveStateBucketName(appBucketName) {
|
|
760
|
+
return sanitizeBucketName(appBucketName, 'pulumi-state');
|
|
761
|
+
}
|
package/src/lib/retry.js
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared retry/polling primitives for the exec layer.
|
|
3
|
+
*
|
|
4
|
+
* Two shapes cover every bespoke loop in the codebase:
|
|
5
|
+
* - runWithRetry: N attempts with explicit inter-attempt delays
|
|
6
|
+
* (kubectl transient retries, ssh transient retries, sideload/push settles).
|
|
7
|
+
* - pollUntil: probe under a time budget with capped exponential backoff
|
|
8
|
+
* (wait-for-k3s, wait-for-schema, wait-for-API-healthy, delete-until-404).
|
|
9
|
+
*/
|
|
10
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
11
|
+
|
|
12
|
+
export async function runWithRetry(
|
|
13
|
+
fn,
|
|
14
|
+
{ delaysMs = [5000, 5000], isTransient = () => true, onRetry } = {},
|
|
15
|
+
) {
|
|
16
|
+
let attempt = 0;
|
|
17
|
+
// attempts = delaysMs.length + 1
|
|
18
|
+
for (;;) {
|
|
19
|
+
try {
|
|
20
|
+
return await fn(attempt);
|
|
21
|
+
} catch (err) {
|
|
22
|
+
if (attempt >= delaysMs.length || !isTransient(err)) throw err;
|
|
23
|
+
attempt += 1;
|
|
24
|
+
if (onRetry) onRetry(err, attempt);
|
|
25
|
+
await sleep(delaysMs[attempt - 1]);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function pollUntil(
|
|
31
|
+
probe,
|
|
32
|
+
{
|
|
33
|
+
budgetMs,
|
|
34
|
+
initialDelayMs = 2000,
|
|
35
|
+
maxDelayMs = 15_000,
|
|
36
|
+
backoffFactor = 2,
|
|
37
|
+
description = 'condition',
|
|
38
|
+
} = {},
|
|
39
|
+
) {
|
|
40
|
+
const deadline = Date.now() + budgetMs;
|
|
41
|
+
let delay = initialDelayMs;
|
|
42
|
+
let lastErr;
|
|
43
|
+
for (;;) {
|
|
44
|
+
try {
|
|
45
|
+
const result = await probe();
|
|
46
|
+
if (result) return result;
|
|
47
|
+
lastErr = undefined;
|
|
48
|
+
} catch (err) {
|
|
49
|
+
lastErr = err;
|
|
50
|
+
}
|
|
51
|
+
if (Date.now() + delay > deadline) break;
|
|
52
|
+
await sleep(delay);
|
|
53
|
+
delay = Math.min(delay * backoffFactor, maxDelayMs);
|
|
54
|
+
}
|
|
55
|
+
const cause = lastErr ? `: ${lastErr.message}` : '';
|
|
56
|
+
const err = new Error(`Timed out after ${budgetMs}ms waiting for ${description}${cause}`);
|
|
57
|
+
err.cause = lastErr;
|
|
58
|
+
throw err;
|
|
59
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure planning helpers for `vibecarbon scale` (k8s / k8s-ha).
|
|
3
|
+
*
|
|
4
|
+
* These functions carry NO side effects — no clack prompts, no SSH, no fetch,
|
|
5
|
+
* no Pulumi. They translate parsed flags and resolved config into the plain
|
|
6
|
+
* data shapes the orchestration code in src/scale.js consumes, which keeps the
|
|
7
|
+
* two behaviors that matter most testable in isolation:
|
|
8
|
+
*
|
|
9
|
+
* - the non-interactive `-yes`/`-type`/bounds flag matrix, and
|
|
10
|
+
* - the Pulumi programConfig assembly, whose output MUST stay byte-identical
|
|
11
|
+
* to what `deploy` produced or Pulumi plans a destructive replace of every
|
|
12
|
+
* node (etcd loss). See the extensive comments at the call site.
|
|
13
|
+
*/
|
|
14
|
+
import { DEFAULT_WORKER_MAX, DEFAULT_WORKER_MIN } from './deploy/utils.js';
|
|
15
|
+
|
|
16
|
+
const REGION_TO_NETWORK_ZONE = {
|
|
17
|
+
nbg1: 'eu-central',
|
|
18
|
+
hel1: 'eu-central',
|
|
19
|
+
fsn1: 'eu-central',
|
|
20
|
+
hil: 'us-west',
|
|
21
|
+
ash: 'us-east',
|
|
22
|
+
sin: 'ap-southeast',
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the non-interactive scale plan from parsed flags.
|
|
27
|
+
*
|
|
28
|
+
* Returns `{ changes, newValues }` for the two scripted branches:
|
|
29
|
+
* - `-yes -type X` → resize every node role to X (plus workerBounds when
|
|
30
|
+
* --min/--max are also present).
|
|
31
|
+
* - `-yes` + bounds → cluster-autoscaler bounds only.
|
|
32
|
+
* Returns `null` when neither branch matches, signalling the caller to fall
|
|
33
|
+
* back to the interactive multiselect prompt.
|
|
34
|
+
*
|
|
35
|
+
* `changes[]` elements are the string ids the caller dispatches on:
|
|
36
|
+
* 'masterType' | 'supabaseType' | 'workerType' | 'workerBounds'.
|
|
37
|
+
*
|
|
38
|
+
* envConfig is accepted for signature parity with the interactive caller; the
|
|
39
|
+
* non-interactive branches are a pure function of `parsed`.
|
|
40
|
+
*
|
|
41
|
+
* @param {{ yes?: boolean, type?: string|null, minWorkers?: number|null, maxWorkers?: number|null }} parsed
|
|
42
|
+
* @param {object} envConfig
|
|
43
|
+
* @returns {{ changes: string[], newValues: Record<string, unknown> } | null}
|
|
44
|
+
*/
|
|
45
|
+
export function planK8sScaleChanges(parsed, _envConfig) {
|
|
46
|
+
const newValues = {};
|
|
47
|
+
|
|
48
|
+
if (parsed.yes && parsed.type) {
|
|
49
|
+
// `--type X` means "scale every node role to X" for parity with compose's
|
|
50
|
+
// `--type X` (which resizes the one VPS).
|
|
51
|
+
const changes = ['masterType', 'supabaseType', 'workerType'];
|
|
52
|
+
newValues.masterType = parsed.type;
|
|
53
|
+
newValues.supabaseType = parsed.type;
|
|
54
|
+
newValues.workerType = parsed.type;
|
|
55
|
+
// Allow --type and bounds to be combined in one invocation.
|
|
56
|
+
if (parsed.minWorkers != null || parsed.maxWorkers != null) {
|
|
57
|
+
changes.push('workerBounds');
|
|
58
|
+
if (parsed.minWorkers != null) newValues.minWorkers = parsed.minWorkers;
|
|
59
|
+
if (parsed.maxWorkers != null) newValues.maxWorkers = parsed.maxWorkers;
|
|
60
|
+
}
|
|
61
|
+
return { changes, newValues };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (parsed.yes && (parsed.minWorkers != null || parsed.maxWorkers != null)) {
|
|
65
|
+
// Bounds-only path: --min-workers and/or --max-workers without --type.
|
|
66
|
+
const changes = ['workerBounds'];
|
|
67
|
+
if (parsed.minWorkers != null) newValues.minWorkers = parsed.minWorkers;
|
|
68
|
+
if (parsed.maxWorkers != null) newValues.maxWorkers = parsed.maxWorkers;
|
|
69
|
+
return { changes, newValues };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Assemble the Hetzner-k8s Pulumi programConfig. Its output MUST be
|
|
77
|
+
* byte-identical to what deployK3s passed at deploy time — every field that
|
|
78
|
+
* flows into a Server's `sshKeys`, `firewallIds`, `labels`, or `userData` is a
|
|
79
|
+
* tripwire for a destructive replace, and replacing master wipes etcd.
|
|
80
|
+
*
|
|
81
|
+
* @param {object} inputs
|
|
82
|
+
* @param {string} inputs.projectName
|
|
83
|
+
* @param {string} inputs.environment Cluster env (e.g. `prod`, `prod-primary`).
|
|
84
|
+
* @param {string} inputs.sshPublicKey
|
|
85
|
+
* @param {string[]} inputs.allowedCidrs Operator CIDRs (SSH + k8s API).
|
|
86
|
+
* @param {string|undefined} inputs.existingSshKeyId
|
|
87
|
+
* @param {string} inputs.location Cluster region.
|
|
88
|
+
* @param {Record<string, unknown>} inputs.newValues Plan output; may set *Type/*Workers.
|
|
89
|
+
* @param {string} inputs.currentMasterType
|
|
90
|
+
* @param {string} inputs.currentSupabaseType
|
|
91
|
+
* @param {string} inputs.currentWorkerType
|
|
92
|
+
* @param {number|undefined} inputs.persistedMinWorkers envConfig.minWorkers
|
|
93
|
+
* @param {number|undefined} inputs.persistedMaxWorkers envConfig.maxWorkers
|
|
94
|
+
* @param {string} inputs.k3sVersion
|
|
95
|
+
* @param {Record<string, string>} inputs.labels
|
|
96
|
+
* @param {string} inputs.apiToken
|
|
97
|
+
* @returns {object}
|
|
98
|
+
*/
|
|
99
|
+
export function buildProgramConfig({
|
|
100
|
+
projectName,
|
|
101
|
+
environment,
|
|
102
|
+
sshPublicKey,
|
|
103
|
+
allowedCidrs,
|
|
104
|
+
existingSshKeyId,
|
|
105
|
+
location,
|
|
106
|
+
newValues,
|
|
107
|
+
currentMasterType,
|
|
108
|
+
currentSupabaseType,
|
|
109
|
+
currentWorkerType,
|
|
110
|
+
persistedMinWorkers,
|
|
111
|
+
persistedMaxWorkers,
|
|
112
|
+
k3sVersion,
|
|
113
|
+
labels,
|
|
114
|
+
apiToken,
|
|
115
|
+
}) {
|
|
116
|
+
return {
|
|
117
|
+
projectName,
|
|
118
|
+
environment,
|
|
119
|
+
sshPublicKey,
|
|
120
|
+
allowedSshIps: allowedCidrs,
|
|
121
|
+
allowedK8sApiIps: allowedCidrs,
|
|
122
|
+
// HA deploys share one Hetzner SshKey across primary + standby and pass
|
|
123
|
+
// `existingSshKeyId` so neither stack manages the resource. Single-cluster
|
|
124
|
+
// lets Pulumi own the SshKey (existingSshKeyId = undefined).
|
|
125
|
+
existingSshKeyId,
|
|
126
|
+
location,
|
|
127
|
+
networkZone: REGION_TO_NETWORK_ZONE[location] || 'eu-central',
|
|
128
|
+
masterServerType: newValues.masterType ?? currentMasterType,
|
|
129
|
+
supabaseServerType: newValues.supabaseType ?? currentSupabaseType,
|
|
130
|
+
workerServerType: newValues.workerType ?? currentWorkerType,
|
|
131
|
+
// minWorkers is Pulumi's static worker floor; maxWorkers flows to the CA
|
|
132
|
+
// patch (not consumed by Pulumi). Replay persisted values when the operator
|
|
133
|
+
// didn't override, keeping Pulumi's worker-pool plan a no-op.
|
|
134
|
+
minWorkers: newValues.minWorkers ?? persistedMinWorkers ?? DEFAULT_WORKER_MIN,
|
|
135
|
+
maxWorkers: newValues.maxWorkers ?? persistedMaxWorkers ?? DEFAULT_WORKER_MAX,
|
|
136
|
+
k3sVersion,
|
|
137
|
+
labels,
|
|
138
|
+
apiToken,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Rewrite the cluster-autoscaler container's `--nodes=` arg to a freshly
|
|
144
|
+
* formatted spec, preserving every other arg. Appends the flag if the deploy
|
|
145
|
+
* container somehow shipped without it. Pure list transform.
|
|
146
|
+
*
|
|
147
|
+
* @param {string[]} argList Current container args (from kubectl jsonpath).
|
|
148
|
+
* @param {string} caNodesSpec New `<min>:<max>:<type>:<region>:<pool>` spec.
|
|
149
|
+
* @returns {string[]}
|
|
150
|
+
*/
|
|
151
|
+
export function updateCaNodesArgs(argList, caNodesSpec) {
|
|
152
|
+
const updated = (Array.isArray(argList) ? argList : []).map((a) =>
|
|
153
|
+
typeof a === 'string' && a.startsWith('--nodes=') ? `--nodes=${caNodesSpec}` : a,
|
|
154
|
+
);
|
|
155
|
+
if (!updated.some((a) => typeof a === 'string' && a.startsWith('--nodes='))) {
|
|
156
|
+
updated.push(`--nodes=${caNodesSpec}`);
|
|
157
|
+
}
|
|
158
|
+
return updated;
|
|
159
|
+
}
|
package/src/lib/ssh.js
CHANGED
|
@@ -8,16 +8,22 @@
|
|
|
8
8
|
* of dynamic tokens), or
|
|
9
9
|
* 2. Use sshRunScript to execute a bash script SCP'd as a file.
|
|
10
10
|
*
|
|
11
|
-
* Host-key checking:
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
* Host-key checking: EVERY call pins against a per-environment known_hosts
|
|
12
|
+
* file — there is no /dev/null + StrictHostKeyChecking=no bypass. When a
|
|
13
|
+
* caller passes an explicit `env`, first-provision callers pass
|
|
14
|
+
* { firstConnect: true } to accept-new (TOFU) once and strict-check ('yes')
|
|
15
|
+
* thereafter. Callers that only have `(ip, sshKeyPath)` get the per-env
|
|
16
|
+
* known_hosts derived from the key path (knownHostsPathForKey) with
|
|
17
|
+
* StrictHostKeyChecking=accept-new — which still REJECTS a changed key for an
|
|
18
|
+
* already-pinned host (MITM on an established env fails) while TOFU-ing a fresh
|
|
19
|
+
* or recycled IP. The operator's ~/.ssh/known_hosts is never touched.
|
|
14
20
|
*/
|
|
15
21
|
|
|
16
22
|
import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
17
23
|
import { tmpdir } from 'node:os';
|
|
18
24
|
import { join } from 'node:path';
|
|
19
25
|
import { runCommandAsync } from './command.js';
|
|
20
|
-
import { knownHostsPath } from './host-keys.js';
|
|
26
|
+
import { knownHostsPath, knownHostsPathForKey } from './host-keys.js';
|
|
21
27
|
import { shEscape } from './shell.js';
|
|
22
28
|
|
|
23
29
|
// k3s writes its kubeconfig here; kubectl doesn't find it by default in SSH sessions.
|
|
@@ -53,30 +59,36 @@ export function getSSHKeyPath(environment) {
|
|
|
53
59
|
}
|
|
54
60
|
|
|
55
61
|
/**
|
|
56
|
-
* Build the SSH `-o` option args for host-key checking.
|
|
62
|
+
* Build the SSH `-o` option args for host-key checking. Always pins against a
|
|
63
|
+
* per-environment known_hosts file under .vibecarbon/ — never /dev/null.
|
|
64
|
+
* GlobalKnownHostsFile=/dev/null ignores the system-wide file; the operator's
|
|
65
|
+
* ~/.ssh/known_hosts is never read or polluted (Hetzner recycles public IPs,
|
|
66
|
+
* and a stale entry there would reject an unrelated reconnect).
|
|
57
67
|
*
|
|
58
|
-
* - With `env`:
|
|
59
|
-
* (or accept-new
|
|
68
|
+
* - With explicit `env`: pin to .vibecarbon/known_hosts_<env>,
|
|
69
|
+
* StrictHostKeyChecking=yes (or accept-new when firstConnect=true, used
|
|
70
|
+
* during first provision).
|
|
60
71
|
*
|
|
61
|
-
* -
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
* remediation-design.md) replaces this with env-scoped pinned known_hosts
|
|
69
|
-
* across every CLI command — callers pass `env` once the deploy path
|
|
70
|
-
* populates .vibecarbon/known_hosts_<env>.
|
|
72
|
+
* - With only `sshKeyPath` (backup/restore/scale/failover only receive the key
|
|
73
|
+
* path, not an env): derive .vibecarbon/known_hosts_<env> from the key path
|
|
74
|
+
* (knownHostsPathForKey) and use StrictHostKeyChecking=accept-new. accept-new
|
|
75
|
+
* bootstraps an empty pin and TOFU's a fresh/recycled IP, but REJECTS a
|
|
76
|
+
* changed key for an already-pinned host — so a MITM against an established
|
|
77
|
+
* env fails, while destroy→redeploy (same IP, new key, re-seeded on
|
|
78
|
+
* provision) does not spuriously hard-fail.
|
|
71
79
|
*/
|
|
72
|
-
function sshHostKeyOpts(env, { firstConnect = false } = {}) {
|
|
80
|
+
function sshHostKeyOpts(env, { firstConnect = false, sshKeyPath } = {}) {
|
|
73
81
|
const opts = [];
|
|
74
82
|
if (env) {
|
|
75
83
|
opts.push('-o', `UserKnownHostsFile=${knownHostsPath(env)}`);
|
|
84
|
+
opts.push('-o', 'GlobalKnownHostsFile=/dev/null');
|
|
76
85
|
opts.push('-o', `StrictHostKeyChecking=${firstConnect ? 'accept-new' : 'yes'}`);
|
|
86
|
+
} else if (sshKeyPath) {
|
|
87
|
+
opts.push('-o', `UserKnownHostsFile=${knownHostsPathForKey(sshKeyPath)}`);
|
|
88
|
+
opts.push('-o', 'GlobalKnownHostsFile=/dev/null');
|
|
89
|
+
opts.push('-o', 'StrictHostKeyChecking=accept-new');
|
|
77
90
|
} else {
|
|
78
|
-
|
|
79
|
-
opts.push('-o', 'StrictHostKeyChecking=no');
|
|
91
|
+
throw new Error('sshHostKeyOpts requires either `env` or `sshKeyPath` for host-key pinning');
|
|
80
92
|
}
|
|
81
93
|
opts.push('-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10');
|
|
82
94
|
// ConnectTimeout only covers TCP connect — once the socket is open, ssh
|
|
@@ -94,18 +106,6 @@ function sshHostKeyOpts(env, { firstConnect = false } = {}) {
|
|
|
94
106
|
return opts;
|
|
95
107
|
}
|
|
96
108
|
|
|
97
|
-
/**
|
|
98
|
-
* The no-host-key-pinning SSH option set as a single space-joined string,
|
|
99
|
-
* for the compose deploy path's string-command runners (sshRun/sshRunAsync
|
|
100
|
-
* in compose/index.js) and raw `ssh ${SSH_OPTS}` interpolations.
|
|
101
|
-
*
|
|
102
|
-
* Derived from sshHostKeyOpts() so the string form can never drift from the
|
|
103
|
-
* argv form — the historical footgun was compose/ha.js hand-copying this
|
|
104
|
-
* string and silently dropping the ServerAlive keepalives, letting a
|
|
105
|
-
* banner-exchange hang stall an HA deploy for ~600s.
|
|
106
|
-
*/
|
|
107
|
-
export const NO_PIN_SSH_OPTS = sshHostKeyOpts().join(' ');
|
|
108
|
-
|
|
109
109
|
/**
|
|
110
110
|
* Execute a command on a remote server via SSH using argv form.
|
|
111
111
|
*
|
|
@@ -143,7 +143,7 @@ export async function sshRun(ip, sshKeyPath, argv, options = {}) {
|
|
|
143
143
|
'ssh',
|
|
144
144
|
'-i',
|
|
145
145
|
sshKeyPath,
|
|
146
|
-
...sshHostKeyOpts(env, { firstConnect }),
|
|
146
|
+
...sshHostKeyOpts(env, { firstConnect, sshKeyPath }),
|
|
147
147
|
'--',
|
|
148
148
|
`root@${ip}`,
|
|
149
149
|
remoteCmd,
|
|
@@ -189,7 +189,7 @@ export async function scpDownload(ip, sshKeyPath, remotePath, localPath, options
|
|
|
189
189
|
'scp',
|
|
190
190
|
'-i',
|
|
191
191
|
sshKeyPath,
|
|
192
|
-
...sshHostKeyOpts(env, { firstConnect }),
|
|
192
|
+
...sshHostKeyOpts(env, { firstConnect, sshKeyPath }),
|
|
193
193
|
'--',
|
|
194
194
|
`root@${ip}:${remotePath}`,
|
|
195
195
|
localPath,
|
|
@@ -206,7 +206,7 @@ export async function scpUpload(ip, sshKeyPath, localPath, remotePath, options =
|
|
|
206
206
|
'scp',
|
|
207
207
|
'-i',
|
|
208
208
|
sshKeyPath,
|
|
209
|
-
...sshHostKeyOpts(env, { firstConnect }),
|
|
209
|
+
...sshHostKeyOpts(env, { firstConnect, sshKeyPath }),
|
|
210
210
|
'--',
|
|
211
211
|
localPath,
|
|
212
212
|
`root@${ip}:${remotePath}`,
|
package/src/restore.js
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
import { existsSync } from 'node:fs';
|
|
20
20
|
import { basename } from 'node:path';
|
|
21
21
|
import * as p from '@clack/prompts';
|
|
22
|
+
import { identifyServers } from './failover.js';
|
|
22
23
|
import { formatInstant } from './lib/backup-format.js';
|
|
23
24
|
import { renderHelp } from './lib/cli/help.js';
|
|
24
25
|
import { parseFlags } from './lib/cli/parse-flags.js';
|
|
@@ -31,9 +32,14 @@ import { loadCredentials, loadProjectConfig } from './lib/config.js';
|
|
|
31
32
|
// while still accepting injected mocks for unit tests.
|
|
32
33
|
import { configureStandbyReplication as realConfigureStandbyReplication } from './lib/deploy/compose/ha.js';
|
|
33
34
|
import { restoreCompose as realRestoreCompose } from './lib/deploy/compose/index.js';
|
|
34
|
-
import {
|
|
35
|
+
import {
|
|
36
|
+
readK8sReplicationState as realReadK8sReplicationState,
|
|
37
|
+
reseedStandbyFromPrimary as realReseedStandbyFromPrimary,
|
|
38
|
+
verifyStreaming,
|
|
39
|
+
} from './lib/deploy/replication.js';
|
|
35
40
|
import { ensureOperatorIpAccess } from './lib/operator-ip.js';
|
|
36
41
|
import { perfAsync } from './lib/perf.js';
|
|
42
|
+
import { confirmProdOrExit } from './lib/prod-confirm.js';
|
|
37
43
|
import { assertInProjectDir } from './lib/project-guard.js';
|
|
38
44
|
import { getPostgresPod, getSSHKeyPath, sshKubectl } from './lib/ssh.js';
|
|
39
45
|
import { createTracker } from './lib/tracker.js';
|
|
@@ -245,7 +251,6 @@ export async function run(args) {
|
|
|
245
251
|
// restore` from a parent directory emits the canonical message.
|
|
246
252
|
assertInProjectDir();
|
|
247
253
|
|
|
248
|
-
requireLicense('restore');
|
|
249
254
|
printBanner();
|
|
250
255
|
p.intro(`${c.bold('vibecarbon restore')} ${c.dim(`v${VERSION}`)}`);
|
|
251
256
|
|
|
@@ -392,19 +397,15 @@ export async function run(args) {
|
|
|
392
397
|
p.cancel('Operation cancelled.');
|
|
393
398
|
process.exit(0);
|
|
394
399
|
}
|
|
395
|
-
|
|
396
|
-
if (envName === 'prod' || envName === 'production') {
|
|
397
|
-
const doubleConfirm = await p.text({
|
|
398
|
-
message: `Type "${envName}" to confirm restoration of production database:`,
|
|
399
|
-
validate: (v) => (v !== envName ? `Please type "${envName}" to confirm` : undefined),
|
|
400
|
-
});
|
|
401
|
-
if (p.isCancel(doubleConfirm)) {
|
|
402
|
-
p.cancel('Operation cancelled.');
|
|
403
|
-
process.exit(0);
|
|
404
|
-
}
|
|
405
|
-
}
|
|
406
400
|
}
|
|
407
401
|
|
|
402
|
+
// SECURITY: the production type-to-confirm runs UNCONDITIONALLY — even with
|
|
403
|
+
// -y — so `restore -y prod` can never silently overwrite the production
|
|
404
|
+
// database. This is deliberately OUTSIDE the `!values.y` block above (the
|
|
405
|
+
// soft y/N confirm is skippable with -y; this hard gate is not). Mirrors
|
|
406
|
+
// destroy's prod guard; shared helper so failover can reuse it.
|
|
407
|
+
await confirmProdOrExit(envName, { actionLabel: 'restore', yes: !!values.y });
|
|
408
|
+
|
|
408
409
|
// Dispatch by mode
|
|
409
410
|
const tracker = createTracker('restore', { environment: envName });
|
|
410
411
|
const s = tracker.spinner();
|
|
@@ -427,6 +428,7 @@ export async function run(args) {
|
|
|
427
428
|
envName,
|
|
428
429
|
serverIp,
|
|
429
430
|
sshKeyPath,
|
|
431
|
+
envConfig,
|
|
430
432
|
});
|
|
431
433
|
}
|
|
432
434
|
} catch (error) {
|
|
@@ -645,7 +647,21 @@ async function setRestoreMarker(ip, sshKeyPath, value) {
|
|
|
645
647
|
]);
|
|
646
648
|
}
|
|
647
649
|
|
|
648
|
-
async function runK8sRestore({
|
|
650
|
+
export async function runK8sRestore({
|
|
651
|
+
s,
|
|
652
|
+
chosenSource,
|
|
653
|
+
envName,
|
|
654
|
+
serverIp,
|
|
655
|
+
sshKeyPath,
|
|
656
|
+
envConfig = {},
|
|
657
|
+
// Injected so the k8s-HA standby re-seed dispatch is unit-testable without a
|
|
658
|
+
// real cluster (mirrors runComposeRestore). Production passes none.
|
|
659
|
+
reseedStandbyFromPrimary = realReseedStandbyFromPrimary,
|
|
660
|
+
readReplicationState = realReadK8sReplicationState,
|
|
661
|
+
// Pacing for the post-reseed streaming verify — injectable so tests don't wait
|
|
662
|
+
// on real timers. Production uses the default setTimeout-backed sleep.
|
|
663
|
+
verifySleep = (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
664
|
+
}) {
|
|
649
665
|
if (chosenSource.kind === 'local') {
|
|
650
666
|
throw new Error(
|
|
651
667
|
'k8s restore is wal-g-based and pulls from S3 — local backup files are not supported. Use `-source latest` (or a timestamp for PITR).',
|
|
@@ -695,6 +711,78 @@ async function runK8sRestore({ s, chosenSource, envName, serverIp, sshKeyPath })
|
|
|
695
711
|
|
|
696
712
|
p.log.success('Restore completed successfully');
|
|
697
713
|
p.log.info('The application has been scaled back up.');
|
|
714
|
+
|
|
715
|
+
// k8s-HA: the standby is a SEPARATE cluster deployed with `restore: null` — a
|
|
716
|
+
// streaming replica, never wal-g-restored (a wal-g fetch on the standby would
|
|
717
|
+
// create a divergent timeline; see k8s/ha/index.js). This restore rewound the
|
|
718
|
+
// PRIMARY to an earlier LSN, so the standby now has WAL AHEAD of the primary
|
|
719
|
+
// and CANNOT resume streaming (timeline divergence).
|
|
720
|
+
//
|
|
721
|
+
// Finding #2: instead of the old warn-and-point-to-`deploy`, perform a
|
|
722
|
+
// first-class VERIFIED re-seed of the standby using the SHARED reseed
|
|
723
|
+
// primitive (the same hardened staged-basebackup+atomic-swap the failover path
|
|
724
|
+
// uses), then confirm streaming from the restored primary. Fail loudly if it
|
|
725
|
+
// can't complete — never silently leave a diverged standby.
|
|
726
|
+
const isK8sHA = !!(envConfig.ha?.enabled || envConfig.ha === true || envConfig.secondaryRegion);
|
|
727
|
+
if (isK8sHA) {
|
|
728
|
+
const servers = identifyServers(envName, envConfig, {});
|
|
729
|
+
const standbyIp = servers?.standby?.ip;
|
|
730
|
+
const primarySupabaseIp = servers?.primary?.supabaseIp || servers?.primary?.ip;
|
|
731
|
+
if (!standbyIp || !primarySupabaseIp) {
|
|
732
|
+
// We can't locate the standby / primary source — fail loudly rather than
|
|
733
|
+
// report a clean restore while the standby silently diverges.
|
|
734
|
+
throw new Error(
|
|
735
|
+
'HA restore: the primary was restored, but the standby cluster or the primary ' +
|
|
736
|
+
'supabase IP could not be resolved from config, so the standby could NOT be ' +
|
|
737
|
+
`re-seeded and now has a DIVERGED timeline. DR is not guaranteed. Resync with ` +
|
|
738
|
+
`\`vibecarbon deploy ${envName}\` (re-runs the verified replication setup).`,
|
|
739
|
+
);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
s.start('Re-seeding standby cluster from restored primary (pg_basebackup)');
|
|
743
|
+
let reseedResult;
|
|
744
|
+
try {
|
|
745
|
+
reseedResult = await reseedStandbyFromPrimary(standbyIp, sshKeyPath, primarySupabaseIp);
|
|
746
|
+
} catch (err) {
|
|
747
|
+
s.stop('Standby re-seed failed');
|
|
748
|
+
throw new Error(
|
|
749
|
+
`HA restore: the primary was restored and is serving, but re-seeding the standby ` +
|
|
750
|
+
`FAILED: ${err.message}\nThe standby has a DIVERGED timeline and is NOT replicating — ` +
|
|
751
|
+
`DR is not guaranteed. Resync with \`vibecarbon deploy ${envName}\`.`,
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
if (reseedResult === 'skipped') {
|
|
755
|
+
s.stop('Standby re-seed skipped — primary unreachable from standby');
|
|
756
|
+
throw new Error(
|
|
757
|
+
'HA restore: the standby could NOT reach the restored primary to re-seed ' +
|
|
758
|
+
'(cross-cluster connectivity). The standby has a DIVERGED timeline and is NOT ' +
|
|
759
|
+
`replicating — DR is not guaranteed. Verify the replication firewall/network, then ` +
|
|
760
|
+
`resync with \`vibecarbon deploy ${envName}\`.`,
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
s.stop('Standby re-seeded from restored primary');
|
|
764
|
+
|
|
765
|
+
// Confirm streaming from the restored primary — a re-seed that boots but
|
|
766
|
+
// never streams is still a diverged standby.
|
|
767
|
+
s.start('Verifying standby is streaming from restored primary');
|
|
768
|
+
const { streaming, lastState } = await verifyStreaming({
|
|
769
|
+
readState: async () => readReplicationState(servers.primary.ip, sshKeyPath),
|
|
770
|
+
attempts: 15,
|
|
771
|
+
delaysMs: [1000, 2000, 3000],
|
|
772
|
+
sleep: verifySleep,
|
|
773
|
+
});
|
|
774
|
+
if (!streaming) {
|
|
775
|
+
s.stop('Standby streaming NOT confirmed');
|
|
776
|
+
throw new Error(
|
|
777
|
+
`HA restore: the standby was re-seeded but streaming replication is NOT confirmed ` +
|
|
778
|
+
`(last pg_stat_replication.state=${
|
|
779
|
+
lastState ? JSON.stringify(lastState) : 'no replica connected'
|
|
780
|
+
}). DR is not guaranteed. Resync with \`vibecarbon deploy ${envName}\`.`,
|
|
781
|
+
);
|
|
782
|
+
}
|
|
783
|
+
s.stop('Standby streaming from restored primary — DR restored');
|
|
784
|
+
}
|
|
785
|
+
|
|
698
786
|
p.log.info(`Verify with: ${c.info(`vibecarbon status ${envName}`)}`);
|
|
699
787
|
}
|
|
700
788
|
|