vibecarbon 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/carbon/db/Dockerfile +14 -3
- package/carbon/docker-compose.dns01.prod.yml +42 -0
- package/carbon/k8s/base/backup/configmap-walg.yaml +47 -0
- package/carbon/k8s/base/backup/cronjob.yaml +67 -92
- package/carbon/k8s/base/backup/kustomization.yaml +1 -0
- package/carbon/k8s/base/backup/network-policy.yaml +9 -14
- package/carbon/k8s/base/backup/rbac.yaml +40 -0
- package/carbon/k8s/base/network-policies.yaml +33 -0
- package/carbon/k8s/values/supabase.values.yaml +144 -0
- package/package.json +1 -2
- package/src/backup.js +2 -36
- package/src/create.js +9 -4
- package/src/deploy.js +12 -0
- package/src/lib/backup-s3.js +0 -31
- package/src/lib/cloudflare.js +0 -59
- package/src/lib/config.js +1 -42
- package/src/lib/deploy/acme.js +57 -0
- package/src/lib/deploy/admin-user.js +105 -0
- package/src/lib/deploy/bundle.js +17 -0
- package/src/lib/deploy/compose/ha.js +94 -91
- package/src/lib/deploy/compose/index.js +50 -248
- package/src/lib/deploy/k8s/ha/index.js +12 -148
- package/src/lib/deploy/k8s/index.js +1 -21
- package/src/lib/deploy/k8s/k3s.js +314 -128
- package/src/lib/deploy/orchestrator.js +70 -35
- package/src/lib/deploy/utils.js +20 -0
- package/src/lib/hetzner-guided-setup.js +0 -7
- package/src/lib/iac/index.js +0 -9
- package/src/lib/images.js +17 -0
- package/src/lib/licensing/index.js +1 -59
- package/src/lib/licensing/tiers.js +0 -8
- package/src/lib/pod-backups.js +53 -0
- package/src/lib/providers/index.js +0 -47
- package/src/lib/ssh.js +12 -0
- package/src/restore.js +98 -265
- package/src/scale.js +43 -20
- package/carbon/backup/Dockerfile +0 -75
- package/carbon/backup/backup.sh +0 -95
- package/carbon/runtime.Dockerfile +0 -17
- package/src/lib/deploy/compose/acme-verify.js +0 -121
- package/src/lib/deploy/index.js +0 -119
- package/src/lib/kubectl.js +0 -81
package/src/restore.js
CHANGED
|
@@ -35,15 +35,9 @@ import { getS3Credentials } from './lib/hetzner-guided-setup.js';
|
|
|
35
35
|
import { requireLicense } from './lib/licensing/index.js';
|
|
36
36
|
import { ensureOperatorIpAccess } from './lib/operator-ip.js';
|
|
37
37
|
import { perfAsync } from './lib/perf.js';
|
|
38
|
+
import { listPodBackups } from './lib/pod-backups.js';
|
|
38
39
|
import { assertInProjectDir } from './lib/project-guard.js';
|
|
39
|
-
import {
|
|
40
|
-
getPostgresPod,
|
|
41
|
-
getSSHKeyPath,
|
|
42
|
-
scpUpload,
|
|
43
|
-
sshKubectl,
|
|
44
|
-
sshRun,
|
|
45
|
-
sshRunScript,
|
|
46
|
-
} from './lib/ssh.js';
|
|
40
|
+
import { getPostgresPod, getSSHKeyPath, scpUpload, sshKubectl, sshRunScript } from './lib/ssh.js';
|
|
47
41
|
import { createTracker } from './lib/tracker.js';
|
|
48
42
|
import { validateBackupFilename } from './lib/validators.js';
|
|
49
43
|
import { VERSION } from './lib/version.js';
|
|
@@ -97,45 +91,6 @@ const SPEC = {
|
|
|
97
91
|
],
|
|
98
92
|
};
|
|
99
93
|
|
|
100
|
-
// ============================================================================
|
|
101
|
-
// KUBECTL HELPERS (k8s mode)
|
|
102
|
-
// ============================================================================
|
|
103
|
-
|
|
104
|
-
function listRemoteBackups(ip, sshKeyPath) {
|
|
105
|
-
const pod = getPostgresPod(ip, sshKeyPath);
|
|
106
|
-
|
|
107
|
-
try {
|
|
108
|
-
const output = sshKubectl(ip, sshKeyPath, [
|
|
109
|
-
'exec',
|
|
110
|
-
'-n',
|
|
111
|
-
'vibecarbon',
|
|
112
|
-
pod,
|
|
113
|
-
'--',
|
|
114
|
-
'sh',
|
|
115
|
-
'-c',
|
|
116
|
-
'ls -lt /backups/*_full.tar.gz /backups/*.sql.gz 2>/dev/null || echo NO_BACKUPS',
|
|
117
|
-
]);
|
|
118
|
-
|
|
119
|
-
if (output === 'NO_BACKUPS' || !output) {
|
|
120
|
-
return [];
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
return output
|
|
124
|
-
.split('\n')
|
|
125
|
-
.filter((line) => line.includes('.tar.gz') || line.includes('.sql.gz'))
|
|
126
|
-
.map((line) => {
|
|
127
|
-
const parts = line.split(/\s+/);
|
|
128
|
-
return {
|
|
129
|
-
size: parts[4],
|
|
130
|
-
date: `${parts[5]} ${parts[6]} ${parts[7]}`,
|
|
131
|
-
name: parts[parts.length - 1].replace('/backups/', ''),
|
|
132
|
-
};
|
|
133
|
-
});
|
|
134
|
-
} catch {
|
|
135
|
-
return [];
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
|
|
139
94
|
// ============================================================================
|
|
140
95
|
// RESTORE OPERATIONS (k8s mode)
|
|
141
96
|
// ============================================================================
|
|
@@ -175,175 +130,8 @@ function verifyPostgres(ip, sshKeyPath) {
|
|
|
175
130
|
]);
|
|
176
131
|
}
|
|
177
132
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
*/
|
|
181
|
-
function restoreFromArchive(ip, sshKeyPath, remotePath) {
|
|
182
|
-
const pod = getPostgresPod(ip, sshKeyPath);
|
|
183
|
-
|
|
184
|
-
// Extract the archive on the server — argv form; remotePath is never shell-interpolated.
|
|
185
|
-
sshRun(ip, sshKeyPath, ['mkdir', '-p', '/tmp/restore']);
|
|
186
|
-
// GNU tar strips leading `/` by default (no `-P` / `--absolute-names`),
|
|
187
|
-
// so absolute paths in a hostile archive never escape /tmp/restore.
|
|
188
|
-
sshRun(ip, sshKeyPath, [
|
|
189
|
-
'tar',
|
|
190
|
-
'--no-xattrs',
|
|
191
|
-
'--no-same-owner',
|
|
192
|
-
'-xzf',
|
|
193
|
-
remotePath,
|
|
194
|
-
'-C',
|
|
195
|
-
'/tmp/restore',
|
|
196
|
-
]);
|
|
197
|
-
|
|
198
|
-
// Check archive contents
|
|
199
|
-
const contents = sshRun(ip, sshKeyPath, ['ls', '/tmp/restore/']);
|
|
200
|
-
|
|
201
|
-
// Restore database
|
|
202
|
-
if (contents.includes('postgres.sql.gz')) {
|
|
203
|
-
sshKubectl(ip, sshKeyPath, [
|
|
204
|
-
'cp',
|
|
205
|
-
'/tmp/restore/postgres.sql.gz',
|
|
206
|
-
`vibecarbon/${pod}:/tmp/postgres.sql.gz`,
|
|
207
|
-
]);
|
|
208
|
-
sshKubectl(ip, sshKeyPath, [
|
|
209
|
-
'exec',
|
|
210
|
-
'-n',
|
|
211
|
-
'vibecarbon',
|
|
212
|
-
pod,
|
|
213
|
-
'--',
|
|
214
|
-
'sh',
|
|
215
|
-
'-c',
|
|
216
|
-
'gunzip -c /tmp/postgres.sql.gz | psql -U supabase_admin postgres',
|
|
217
|
-
]);
|
|
218
|
-
sshKubectl(ip, sshKeyPath, [
|
|
219
|
-
'exec',
|
|
220
|
-
'-n',
|
|
221
|
-
'vibecarbon',
|
|
222
|
-
pod,
|
|
223
|
-
'--',
|
|
224
|
-
'rm',
|
|
225
|
-
'-f',
|
|
226
|
-
'/tmp/postgres.sql.gz',
|
|
227
|
-
]);
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
// Restore storage files if present and using file backend
|
|
231
|
-
if (contents.includes('storage.tar.gz')) {
|
|
232
|
-
// kubectl get returns empty string if STORAGE_BACKEND env var is unset;
|
|
233
|
-
// default to 'file' so the legacy single-backend path still works.
|
|
234
|
-
let storageBackend;
|
|
235
|
-
try {
|
|
236
|
-
storageBackend = sshKubectl(ip, sshKeyPath, [
|
|
237
|
-
'get',
|
|
238
|
-
'deployment',
|
|
239
|
-
'storage',
|
|
240
|
-
'-n',
|
|
241
|
-
'vibecarbon',
|
|
242
|
-
'-o',
|
|
243
|
-
'jsonpath={.spec.template.spec.containers[0].env[?(@.name=="STORAGE_BACKEND")].value}',
|
|
244
|
-
]);
|
|
245
|
-
} catch {
|
|
246
|
-
storageBackend = 'file';
|
|
247
|
-
}
|
|
248
|
-
if (!storageBackend) storageBackend = 'file';
|
|
249
|
-
|
|
250
|
-
if (storageBackend === 'file') {
|
|
251
|
-
const storagePod = sshKubectl(ip, sshKeyPath, [
|
|
252
|
-
'get',
|
|
253
|
-
'pods',
|
|
254
|
-
'-n',
|
|
255
|
-
'vibecarbon',
|
|
256
|
-
'-l',
|
|
257
|
-
'app.kubernetes.io/name=supabase-storage',
|
|
258
|
-
'-o',
|
|
259
|
-
'jsonpath={.items[0].metadata.name}',
|
|
260
|
-
]);
|
|
261
|
-
|
|
262
|
-
sshKubectl(ip, sshKeyPath, [
|
|
263
|
-
'cp',
|
|
264
|
-
'/tmp/restore/storage.tar.gz',
|
|
265
|
-
`vibecarbon/${storagePod}:/tmp/storage.tar.gz`,
|
|
266
|
-
]);
|
|
267
|
-
sshKubectl(ip, sshKeyPath, [
|
|
268
|
-
'exec',
|
|
269
|
-
'-n',
|
|
270
|
-
'vibecarbon',
|
|
271
|
-
storagePod,
|
|
272
|
-
'--',
|
|
273
|
-
'sh',
|
|
274
|
-
'-c',
|
|
275
|
-
'tar --no-xattrs --no-same-owner -xzf /tmp/storage.tar.gz -C / && rm -f /tmp/storage.tar.gz',
|
|
276
|
-
]);
|
|
277
|
-
} else {
|
|
278
|
-
p.log.info('Storage backend is S3 — skipping file restore (files already in S3)');
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
// Cleanup
|
|
283
|
-
sshRun(ip, sshKeyPath, ['rm', '-rf', '/tmp/restore']);
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
/**
|
|
287
|
-
* Restore from a legacy .sql.gz file (database only)
|
|
288
|
-
*/
|
|
289
|
-
function restoreFromSqlGz(ip, sshKeyPath, backupName) {
|
|
290
|
-
const pod = getPostgresPod(ip, sshKeyPath);
|
|
291
|
-
|
|
292
|
-
// backupName is validated via validateBackupFilename at entry; still pass it
|
|
293
|
-
// as a positional sh arg so the remote shell never expands it as code.
|
|
294
|
-
sshKubectl(ip, sshKeyPath, [
|
|
295
|
-
'exec',
|
|
296
|
-
'-n',
|
|
297
|
-
'vibecarbon',
|
|
298
|
-
pod,
|
|
299
|
-
'--',
|
|
300
|
-
'sh',
|
|
301
|
-
'-c',
|
|
302
|
-
'gunzip -c "/backups/$1" | psql -U supabase_admin postgres',
|
|
303
|
-
'_',
|
|
304
|
-
backupName,
|
|
305
|
-
]);
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
async function restoreFromRemote(ip, sshKeyPath, backupName) {
|
|
309
|
-
scaleDownApp(ip, sshKeyPath);
|
|
310
|
-
|
|
311
|
-
if (backupName.endsWith('_full.tar.gz')) {
|
|
312
|
-
restoreFromArchive(ip, sshKeyPath, `/tmp/${backupName}`);
|
|
313
|
-
} else {
|
|
314
|
-
restoreFromSqlGz(ip, sshKeyPath, backupName);
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
scaleUpApp(ip, sshKeyPath);
|
|
318
|
-
verifyPostgres(ip, sshKeyPath);
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
async function uploadAndRestore(ip, sshKeyPath, localFile) {
|
|
322
|
-
const filename = basename(localFile);
|
|
323
|
-
|
|
324
|
-
// Upload local file to server
|
|
325
|
-
scpUpload(ip, sshKeyPath, localFile, `/tmp/${filename}`);
|
|
326
|
-
|
|
327
|
-
scaleDownApp(ip, sshKeyPath);
|
|
328
|
-
|
|
329
|
-
if (filename.endsWith('_full.tar.gz')) {
|
|
330
|
-
restoreFromArchive(ip, sshKeyPath, `/tmp/${filename}`);
|
|
331
|
-
} else if (filename.endsWith('.sql.gz')) {
|
|
332
|
-
const pod = getPostgresPod(ip, sshKeyPath);
|
|
333
|
-
sshKubectl(ip, sshKeyPath, [
|
|
334
|
-
'cp',
|
|
335
|
-
`/tmp/${filename}`,
|
|
336
|
-
`vibecarbon/${pod}:/backups/${filename}`,
|
|
337
|
-
]);
|
|
338
|
-
restoreFromSqlGz(ip, sshKeyPath, filename);
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
// Clean up temp file on host
|
|
342
|
-
sshRun(ip, sshKeyPath, ['rm', '-f', `/tmp/${filename}`]);
|
|
343
|
-
|
|
344
|
-
scaleUpApp(ip, sshKeyPath);
|
|
345
|
-
verifyPostgres(ip, sshKeyPath);
|
|
346
|
-
}
|
|
133
|
+
// (k8s pg_dump restore helpers removed — k8s restore is now wal-g-native via
|
|
134
|
+
// the db pod's init container; see runK8sRestore below.)
|
|
347
135
|
|
|
348
136
|
// ============================================================================
|
|
349
137
|
// SOURCE INTERPRETATION
|
|
@@ -511,19 +299,32 @@ export async function run(args) {
|
|
|
511
299
|
// restores) ask for "the most recent" without first listing + parsing
|
|
512
300
|
// names from `vibecarbon backup -l`.
|
|
513
301
|
const wantsLatest = values.source === 'latest';
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
302
|
+
let chosenSource;
|
|
303
|
+
if (!wantsLatest && values.source) {
|
|
304
|
+
chosenSource = classifySource(/** @type {string} */ (values.source));
|
|
305
|
+
} else if (wantsLatest && !isCompose) {
|
|
306
|
+
// k8s restore is wal-g-native: `latest` resolves to the LATEST base backup
|
|
307
|
+
// INSIDE the cluster — the db pod's walg-restore init container runs
|
|
308
|
+
// `wal-g backup-fetch LATEST` (see runK8sRestore). pickInteractiveSource
|
|
309
|
+
// lists pg_dump `*_full.tar.gz` backups via listS3Backups, which cannot see
|
|
310
|
+
// wal-g `base_*` objects, so routing k8s `-source latest` through it wrongly
|
|
311
|
+
// exits "No backups found in S3" even though a wal-g base backup exists.
|
|
312
|
+
// Hand runK8sRestore the `latest` sentinel directly and let wal-g resolve
|
|
313
|
+
// it. (RCA 2026-05-31: e2e k8s/k8s-ha restore step — never reached before
|
|
314
|
+
// the backup chain (#4/#5/#6) was fixed.)
|
|
315
|
+
chosenSource = { kind: 's3', name: 'latest' };
|
|
316
|
+
} else {
|
|
317
|
+
chosenSource = await pickInteractiveSource({
|
|
318
|
+
isCompose,
|
|
319
|
+
envName,
|
|
320
|
+
projectName,
|
|
321
|
+
serverIp,
|
|
322
|
+
sshKeyPath,
|
|
323
|
+
s3,
|
|
324
|
+
useS3,
|
|
325
|
+
yes: wantsLatest || !!values.y,
|
|
326
|
+
});
|
|
327
|
+
}
|
|
527
328
|
|
|
528
329
|
if (!chosenSource) {
|
|
529
330
|
p.log.error('No backup selected.');
|
|
@@ -579,8 +380,6 @@ export async function run(args) {
|
|
|
579
380
|
envName,
|
|
580
381
|
serverIp,
|
|
581
382
|
sshKeyPath,
|
|
582
|
-
s3,
|
|
583
|
-
useS3,
|
|
584
383
|
});
|
|
585
384
|
}
|
|
586
385
|
} catch (error) {
|
|
@@ -662,7 +461,7 @@ async function runList({ isCompose, envName, projectName, serverIp, sshKeyPath }
|
|
|
662
461
|
}
|
|
663
462
|
|
|
664
463
|
// K8s pod fallback
|
|
665
|
-
const backups =
|
|
464
|
+
const backups = listPodBackups(serverIp, sshKeyPath);
|
|
666
465
|
printBackupList(backups, envName);
|
|
667
466
|
}
|
|
668
467
|
|
|
@@ -757,7 +556,7 @@ async function pickInteractiveSource({
|
|
|
757
556
|
|
|
758
557
|
// K8s pod fallback
|
|
759
558
|
s.start('Fetching available backups');
|
|
760
|
-
const backups =
|
|
559
|
+
const backups = listPodBackups(serverIp, sshKeyPath);
|
|
761
560
|
s.stop('Backups retrieved');
|
|
762
561
|
if (backups.length === 0) {
|
|
763
562
|
p.log.error('No backups found on the server.');
|
|
@@ -822,42 +621,76 @@ async function runComposeRestore({
|
|
|
822
621
|
void envName;
|
|
823
622
|
}
|
|
824
623
|
|
|
825
|
-
|
|
826
|
-
|
|
624
|
+
// k8s restore is wal-g-native: postgres + wal-g live in the supabase-db pod,
|
|
625
|
+
// which continuously archives WAL + base backups to S3. Restore = set a
|
|
626
|
+
// RESTORE_TARGET marker on vibecarbon-secrets → bounce the db StatefulSet so
|
|
627
|
+
// its walg-restore init container runs `wal-g backup-fetch` into PGDATA before
|
|
628
|
+
// postgres starts → clear the marker. No bytes flow through the operator and
|
|
629
|
+
// no pg_dump/kubectl-cp; wal-g pulls from S3 inside the cluster. Mirrors the
|
|
630
|
+
// compose wal-g restore, adapted to the StatefulSet/init-container shape.
|
|
631
|
+
const DB_STATEFULSET = 'supabase-supabase-db';
|
|
632
|
+
|
|
633
|
+
/** Set or clear the RESTORE_TARGET marker the init container reads. */
|
|
634
|
+
function setRestoreMarker(ip, sshKeyPath, value) {
|
|
635
|
+
sshKubectl(ip, sshKeyPath, [
|
|
636
|
+
'patch',
|
|
637
|
+
'secret',
|
|
638
|
+
'vibecarbon-secrets',
|
|
639
|
+
'-n',
|
|
640
|
+
'vibecarbon',
|
|
641
|
+
'--type=merge',
|
|
642
|
+
'-p',
|
|
643
|
+
JSON.stringify({ stringData: { RESTORE_TARGET: value } }),
|
|
644
|
+
]);
|
|
645
|
+
}
|
|
827
646
|
|
|
647
|
+
async function runK8sRestore({ s, chosenSource, envName, serverIp, sshKeyPath }) {
|
|
828
648
|
if (chosenSource.kind === 'local') {
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
}
|
|
832
|
-
await perfAsync('restore.uploadAndRestore', () =>
|
|
833
|
-
uploadAndRestore(serverIp, sshKeyPath, chosenSource.path),
|
|
834
|
-
);
|
|
835
|
-
s.stop('Database restored');
|
|
836
|
-
} else if (useS3) {
|
|
837
|
-
s.message('Downloading backup from S3');
|
|
838
|
-
const tempPath = join(process.cwd(), `.restore-tmp-${chosenSource.name}`);
|
|
839
|
-
await perfAsync('restore.downloadS3', () =>
|
|
840
|
-
downloadS3Backup(s3, `backups/${chosenSource.name}`, tempPath),
|
|
841
|
-
);
|
|
842
|
-
s.message('Uploading backup to server');
|
|
843
|
-
await perfAsync('restore.scpUpload', async () =>
|
|
844
|
-
scpUpload(serverIp, sshKeyPath, tempPath, `/tmp/${chosenSource.name}`),
|
|
649
|
+
throw new Error(
|
|
650
|
+
'k8s restore is wal-g-based and pulls from S3 — local backup files are not supported. Use `-source latest` (or a timestamp for PITR).',
|
|
845
651
|
);
|
|
846
|
-
try {
|
|
847
|
-
unlinkSync(tempPath);
|
|
848
|
-
} catch {
|
|
849
|
-
// ignore cleanup errors
|
|
850
|
-
}
|
|
851
|
-
s.message('Restoring database');
|
|
852
|
-
await perfAsync('restore.restoreFromRemote', () =>
|
|
853
|
-
restoreFromRemote(serverIp, sshKeyPath, chosenSource.name),
|
|
854
|
-
);
|
|
855
|
-
s.stop('Database restored');
|
|
856
|
-
} else {
|
|
857
|
-
// Pod-only fallback: backup name refers to a file already in the pod.
|
|
858
|
-
await restoreFromRemote(serverIp, sshKeyPath, chosenSource.name);
|
|
859
|
-
s.stop('Database restored');
|
|
860
652
|
}
|
|
653
|
+
// wal-g target: 'latest' → LATEST base backup; otherwise treat the source
|
|
654
|
+
// name as an ISO-8601 timestamp for point-in-time recovery.
|
|
655
|
+
const target =
|
|
656
|
+
chosenSource.name === 'latest' || !chosenSource.name ? 'latest' : chosenSource.name;
|
|
657
|
+
|
|
658
|
+
s.start('Restoring database via wal-g (app will be temporarily unavailable)');
|
|
659
|
+
// 1. Mark the restore so the db pod's init container fetches on next boot.
|
|
660
|
+
setRestoreMarker(serverIp, sshKeyPath, target);
|
|
661
|
+
// 2. Quiesce the app so it isn't querying mid-swap.
|
|
662
|
+
await perfAsync('restore.scaleDownApp', async () => scaleDownApp(serverIp, sshKeyPath));
|
|
663
|
+
try {
|
|
664
|
+
// 3. Bounce the db StatefulSet → walg-restore init container clears PGDATA
|
|
665
|
+
// and runs `wal-g backup-fetch` before postgres starts.
|
|
666
|
+
s.message('Fetching from S3 + recovering (wal-g)');
|
|
667
|
+
await perfAsync('restore.walgFetch', async () => {
|
|
668
|
+
sshKubectl(serverIp, sshKeyPath, [
|
|
669
|
+
'rollout',
|
|
670
|
+
'restart',
|
|
671
|
+
`statefulset/${DB_STATEFULSET}`,
|
|
672
|
+
'-n',
|
|
673
|
+
'vibecarbon',
|
|
674
|
+
]);
|
|
675
|
+
sshKubectl(serverIp, sshKeyPath, [
|
|
676
|
+
'rollout',
|
|
677
|
+
'status',
|
|
678
|
+
`statefulset/${DB_STATEFULSET}`,
|
|
679
|
+
'-n',
|
|
680
|
+
'vibecarbon',
|
|
681
|
+
'--timeout=600s',
|
|
682
|
+
]);
|
|
683
|
+
});
|
|
684
|
+
// 4. Confirm postgres accepts connections post-recovery.
|
|
685
|
+
await perfAsync('restore.verifyPostgres', async () => verifyPostgres(serverIp, sshKeyPath));
|
|
686
|
+
} finally {
|
|
687
|
+
// 5. Always clear the marker so an unrelated future pod restart does NOT
|
|
688
|
+
// re-fetch and wipe live data. Runs even if the restore failed.
|
|
689
|
+
setRestoreMarker(serverIp, sshKeyPath, '');
|
|
690
|
+
}
|
|
691
|
+
// 6. Bring the app back.
|
|
692
|
+
await perfAsync('restore.scaleUpApp', async () => scaleUpApp(serverIp, sshKeyPath));
|
|
693
|
+
s.stop('Database restored');
|
|
861
694
|
|
|
862
695
|
p.log.success('Restore completed successfully');
|
|
863
696
|
p.log.info('The application has been scaled back up.');
|
package/src/scale.js
CHANGED
|
@@ -25,6 +25,7 @@ import { c, printBanner } from './lib/colors.js';
|
|
|
25
25
|
import { runCommand } from './lib/command.js';
|
|
26
26
|
import { loadCredentials, saveProjectConfig } from './lib/config.js';
|
|
27
27
|
import { buildCostBreakdown, formatCostLines } from './lib/cost.js';
|
|
28
|
+
import { useDnsChallenge } from './lib/deploy/acme.js';
|
|
28
29
|
import {
|
|
29
30
|
backupCompose,
|
|
30
31
|
dockerLoginOnServer,
|
|
@@ -265,6 +266,11 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
|
|
|
265
266
|
const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}`);
|
|
266
267
|
const sshPubKeyPath = `${sshKeyPath}.pub`;
|
|
267
268
|
const domain = envConfig.domain || null;
|
|
269
|
+
// Managed-DNS (cloudflare/hetzner) deploys issue certs via ACME DNS-01, so
|
|
270
|
+
// the new/recreated server needs the DNS-01 Traefik override and has no
|
|
271
|
+
// HTTP-01 race to wait on or reset. `manual` keeps HTTP-01.
|
|
272
|
+
const dnsProvider = envConfig.dnsProvider || envConfig.dns?.provider || null;
|
|
273
|
+
const dnsChallenge = useDnsChallenge(dnsProvider);
|
|
268
274
|
|
|
269
275
|
// Read SSH public key and register/reuse in Hetzner
|
|
270
276
|
const sshPubKey = readFileSync(sshPubKeyPath, 'utf-8').trim();
|
|
@@ -383,6 +389,14 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
|
|
|
383
389
|
n8n: services.n8n,
|
|
384
390
|
metabase: services.metabase,
|
|
385
391
|
redis: services.redis,
|
|
392
|
+
// DNS-01: ship the Traefik override + re-affirm ACME_DNS_PROVIDER.
|
|
393
|
+
// The provider token flows through `envOverrides: oldEnv` (the
|
|
394
|
+
// original deploy baked it into the server .env); tokens are passed
|
|
395
|
+
// too so a fresh bundle still works if the old server was unreachable.
|
|
396
|
+
dnsChallenge,
|
|
397
|
+
dnsProvider,
|
|
398
|
+
hetznerApiToken: apiToken,
|
|
399
|
+
cloudflareApiToken: loadCredentials().cloudflare?.apiToken || null,
|
|
386
400
|
// Replay every env var the old server had. renderBundle's existing
|
|
387
401
|
// envOverrides path overlays these onto the project-local `.env`
|
|
388
402
|
// baseline, then re-applies the deploy-time overrides (DOMAIN,
|
|
@@ -508,34 +522,41 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
|
|
|
508
522
|
);
|
|
509
523
|
s.stop('Database restored');
|
|
510
524
|
|
|
511
|
-
// 9a. Update DNS
|
|
512
|
-
//
|
|
513
|
-
//
|
|
525
|
+
// 9a. Update DNS to the new server. On HTTP-01 this must happen BEFORE
|
|
526
|
+
// recreating services (Traefik challenges the A record on startup; a
|
|
527
|
+
// stale record fails all challenges with a 30+ min retry). On DNS-01 the
|
|
528
|
+
// ordering is irrelevant — lego validates a TXT record, not the A record.
|
|
514
529
|
if (domain) {
|
|
515
530
|
s.start('Updating DNS to new server...');
|
|
516
531
|
await perfAsync('scale.updateDNS', () => updateDNS(envConfig, apiToken, domain, newIp));
|
|
517
532
|
s.stop('DNS updated');
|
|
518
533
|
|
|
519
|
-
//
|
|
520
|
-
//
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
534
|
+
// HTTP-01 only: poll until public resolvers return the new IP (max
|
|
535
|
+
// 120s) so Traefik's first challenge sees it. DNS-01 doesn't gate on
|
|
536
|
+
// A-record propagation, so skip the wait.
|
|
537
|
+
if (!dnsChallenge) {
|
|
538
|
+
s.start('Waiting for DNS propagation...');
|
|
539
|
+
const dnsOk = await perfAsync('scale.waitForDNS', () =>
|
|
540
|
+
waitForDNSPropagation(domain, newIp, 120_000),
|
|
541
|
+
);
|
|
542
|
+
s.stop(dnsOk ? 'DNS propagated' : 'DNS propagation timed out (proceeding anyway)');
|
|
543
|
+
}
|
|
526
544
|
}
|
|
527
545
|
|
|
528
|
-
// 9b.
|
|
529
|
-
//
|
|
530
|
-
//
|
|
531
|
-
//
|
|
546
|
+
// 9b. HTTP-01 only: reset ACME state so Traefik retries challenges with
|
|
547
|
+
// the corrected DNS — step 6's startComposeStack ran before the A record
|
|
548
|
+
// moved, so failed challenges are cached as errors in letsencrypt_data.
|
|
549
|
+
// DNS-01 has no such cached HTTP-01 failure to clear (and wiping acme.json
|
|
550
|
+
// would force a needless re-issue), so skip it.
|
|
532
551
|
// Uses an SCP'd script so quoting/redirection stay intact without local shell.
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
552
|
+
if (!dnsChallenge) {
|
|
553
|
+
sshRunScript(
|
|
554
|
+
newIp,
|
|
555
|
+
sshKeyPath,
|
|
556
|
+
`cd ${remoteDir} && docker compose exec -T traefik sh -c 'echo "{}" > /letsencrypt/acme.json && chmod 600 /letsencrypt/acme.json'`,
|
|
557
|
+
{ timeout: 30_000 },
|
|
558
|
+
);
|
|
559
|
+
}
|
|
539
560
|
|
|
540
561
|
// 9c. Recreate all services after restore so they reconnect to the restored DB.
|
|
541
562
|
// Include all feature compose file flags so feature containers (redis, n8n,
|
|
@@ -545,6 +566,8 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
|
|
|
545
566
|
const composeFlags = [
|
|
546
567
|
'-f docker-compose.yml',
|
|
547
568
|
'-f docker-compose.prod.yml',
|
|
569
|
+
// DNS-01 override must follow prod.yml so it replaces the Traefik command.
|
|
570
|
+
...(dnsChallenge ? ['-f docker-compose.dns01.prod.yml'] : []),
|
|
548
571
|
...(services.observability
|
|
549
572
|
? ['-f docker-compose.observability.yml', '-f docker-compose.observability.prod.yml']
|
|
550
573
|
: []),
|
package/carbon/backup/Dockerfile
DELETED
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
# syntax=docker/dockerfile:1
|
|
2
|
-
FROM ubuntu:22.04
|
|
3
|
-
|
|
4
|
-
# Base image is ubuntu, not alpine, because wal-g v3.x stopped publishing
|
|
5
|
-
# alpine-musl prebuilt binaries — only ubuntu glibc builds remain. We need
|
|
6
|
-
# wal-g v3+ for the current S3 + Postgres 16 wire protocol support.
|
|
7
|
-
|
|
8
|
-
ENV DEBIAN_FRONTEND=noninteractive
|
|
9
|
-
|
|
10
|
-
# Install PostgreSQL 16 client + utilities. PG 16 isn't in the default
|
|
11
|
-
# ubuntu 22.04 repos; pull it from the postgresql apt repo. AWS CLI is
|
|
12
|
-
# installed as the standalone v2 binary further down (the apt-shipped
|
|
13
|
-
# `awscli` is v1 and pulls ~500MB of Python deps; v2 is ~80MB self-
|
|
14
|
-
# contained and the only version AWS itself supports for new code).
|
|
15
|
-
#
|
|
16
|
-
# apt cache mount: keeps /var/cache/apt populated across rebuilds so
|
|
17
|
-
# `apt-get install` reuses .deb files. The cache lives outside the layer,
|
|
18
|
-
# so we don't need the historical `rm -rf /var/lib/apt/lists/*` to keep
|
|
19
|
-
# the final image small — that's still done below as a belt-and-braces
|
|
20
|
-
# measure for cases where the cache mount isn't honored (e.g. legacy
|
|
21
|
-
# `docker build` without BuildKit).
|
|
22
|
-
# `sharing=locked` serializes parallel builds against the same cache
|
|
23
|
-
# (apt's lock semantics expect exclusive access).
|
|
24
|
-
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
|
25
|
-
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
|
26
|
-
apt-get update && apt-get install -y --no-install-recommends \
|
|
27
|
-
ca-certificates curl gnupg lsb-release \
|
|
28
|
-
bash tar gzip unzip && \
|
|
29
|
-
install -d /usr/share/postgresql-common/pgdg && \
|
|
30
|
-
curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \
|
|
31
|
-
-o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc && \
|
|
32
|
-
echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" \
|
|
33
|
-
> /etc/apt/sources.list.d/pgdg.list && \
|
|
34
|
-
apt-get update && apt-get install -y --no-install-recommends \
|
|
35
|
-
postgresql-client-16
|
|
36
|
-
|
|
37
|
-
# Install AWS CLI v2 standalone binary (~80MB) instead of `apt-get install
|
|
38
|
-
# awscli` (v1 with ~500MB of Python/boto3 deps). Saves ~400MB on the
|
|
39
|
-
# final image, which matters because the backup image is sideloaded to
|
|
40
|
-
# every k3s node (~6 nodes in HA = ~2.4GB less network per deploy).
|
|
41
|
-
# --fail makes curl exit non-zero on HTTP 4xx/5xx instead of piping a
|
|
42
|
-
# 404 HTML body into unzip.
|
|
43
|
-
RUN curl -fL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip \
|
|
44
|
-
-o /tmp/awscli.zip && \
|
|
45
|
-
unzip -q /tmp/awscli.zip -d /tmp && \
|
|
46
|
-
/tmp/aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli && \
|
|
47
|
-
rm -rf /tmp/aws /tmp/awscli.zip
|
|
48
|
-
|
|
49
|
-
# Install WAL-G v3.0.5 (ubuntu 22.04 amd64 build).
|
|
50
|
-
# --fail makes curl exit non-zero on HTTP 4xx/5xx instead of piping a 404
|
|
51
|
-
# HTML body into tar (the previous v3.0.3-alpine URL 404'd, tar saw "<html>"
|
|
52
|
-
# and errored "not in gzip format" — observed 2026-04-26 matrix run #2).
|
|
53
|
-
RUN curl -fL https://github.com/wal-g/wal-g/releases/download/v3.0.5/wal-g-pg-ubuntu-22.04-amd64.tar.gz \
|
|
54
|
-
| tar -xz && \
|
|
55
|
-
mv wal-g-pg-ubuntu-22.04-amd64 /usr/local/bin/wal-g && \
|
|
56
|
-
chmod +x /usr/local/bin/wal-g
|
|
57
|
-
|
|
58
|
-
# Security: non-root system user. We can't reuse the name `backup` —
|
|
59
|
-
# ubuntu pre-populates a `backup` system group (GID 34, used for system
|
|
60
|
-
# backup operations) and `groupadd backup` errors with exit 9 ("group
|
|
61
|
-
# 'backup' already exists"). The previous alpine version sidestepped
|
|
62
|
-
# this because alpine doesn't ship the same default. Use a unique name
|
|
63
|
-
# (`vbbackup`) and let --system pick a free system GID/UID.
|
|
64
|
-
# The k8s manifest doesn't pin runAsUser so any UID works.
|
|
65
|
-
RUN groupadd --system vbbackup && \
|
|
66
|
-
useradd --system -g vbbackup -s /usr/sbin/nologin -M vbbackup
|
|
67
|
-
|
|
68
|
-
# Copy backup script
|
|
69
|
-
COPY backup.sh /usr/local/bin/backup.sh
|
|
70
|
-
RUN chmod +x /usr/local/bin/backup.sh
|
|
71
|
-
|
|
72
|
-
USER vbbackup
|
|
73
|
-
|
|
74
|
-
# CronJob handles scheduling — script runs once and exits
|
|
75
|
-
ENTRYPOINT ["/usr/local/bin/backup.sh"]
|