vibecarbon 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/carbon/db/Dockerfile +14 -3
  2. package/carbon/docker-compose.dns01.prod.yml +42 -0
  3. package/carbon/k8s/base/backup/configmap-walg.yaml +47 -0
  4. package/carbon/k8s/base/backup/cronjob.yaml +67 -92
  5. package/carbon/k8s/base/backup/kustomization.yaml +1 -0
  6. package/carbon/k8s/base/backup/network-policy.yaml +9 -14
  7. package/carbon/k8s/base/backup/rbac.yaml +40 -0
  8. package/carbon/k8s/base/network-policies.yaml +33 -0
  9. package/carbon/k8s/values/supabase.values.yaml +144 -0
  10. package/package.json +1 -2
  11. package/src/backup.js +2 -36
  12. package/src/create.js +9 -4
  13. package/src/deploy.js +12 -0
  14. package/src/lib/backup-s3.js +0 -31
  15. package/src/lib/cloudflare.js +0 -59
  16. package/src/lib/config.js +1 -42
  17. package/src/lib/deploy/acme.js +57 -0
  18. package/src/lib/deploy/admin-user.js +105 -0
  19. package/src/lib/deploy/bundle.js +17 -0
  20. package/src/lib/deploy/compose/ha.js +94 -91
  21. package/src/lib/deploy/compose/index.js +50 -248
  22. package/src/lib/deploy/k8s/ha/index.js +12 -148
  23. package/src/lib/deploy/k8s/index.js +1 -21
  24. package/src/lib/deploy/k8s/k3s.js +314 -128
  25. package/src/lib/deploy/orchestrator.js +70 -35
  26. package/src/lib/deploy/utils.js +20 -0
  27. package/src/lib/hetzner-guided-setup.js +0 -7
  28. package/src/lib/iac/index.js +0 -9
  29. package/src/lib/images.js +17 -0
  30. package/src/lib/licensing/index.js +1 -59
  31. package/src/lib/licensing/tiers.js +0 -8
  32. package/src/lib/pod-backups.js +53 -0
  33. package/src/lib/providers/index.js +0 -47
  34. package/src/lib/ssh.js +12 -0
  35. package/src/restore.js +98 -265
  36. package/src/scale.js +43 -20
  37. package/carbon/backup/Dockerfile +0 -75
  38. package/carbon/backup/backup.sh +0 -95
  39. package/carbon/runtime.Dockerfile +0 -17
  40. package/src/lib/deploy/compose/acme-verify.js +0 -121
  41. package/src/lib/deploy/index.js +0 -119
  42. package/src/lib/kubectl.js +0 -81
package/src/restore.js CHANGED
@@ -35,15 +35,9 @@ import { getS3Credentials } from './lib/hetzner-guided-setup.js';
35
35
  import { requireLicense } from './lib/licensing/index.js';
36
36
  import { ensureOperatorIpAccess } from './lib/operator-ip.js';
37
37
  import { perfAsync } from './lib/perf.js';
38
+ import { listPodBackups } from './lib/pod-backups.js';
38
39
  import { assertInProjectDir } from './lib/project-guard.js';
39
- import {
40
- getPostgresPod,
41
- getSSHKeyPath,
42
- scpUpload,
43
- sshKubectl,
44
- sshRun,
45
- sshRunScript,
46
- } from './lib/ssh.js';
40
+ import { getPostgresPod, getSSHKeyPath, scpUpload, sshKubectl, sshRunScript } from './lib/ssh.js';
47
41
  import { createTracker } from './lib/tracker.js';
48
42
  import { validateBackupFilename } from './lib/validators.js';
49
43
  import { VERSION } from './lib/version.js';
@@ -97,45 +91,6 @@ const SPEC = {
97
91
  ],
98
92
  };
99
93
 
100
- // ============================================================================
101
- // KUBECTL HELPERS (k8s mode)
102
- // ============================================================================
103
-
104
- function listRemoteBackups(ip, sshKeyPath) {
105
- const pod = getPostgresPod(ip, sshKeyPath);
106
-
107
- try {
108
- const output = sshKubectl(ip, sshKeyPath, [
109
- 'exec',
110
- '-n',
111
- 'vibecarbon',
112
- pod,
113
- '--',
114
- 'sh',
115
- '-c',
116
- 'ls -lt /backups/*_full.tar.gz /backups/*.sql.gz 2>/dev/null || echo NO_BACKUPS',
117
- ]);
118
-
119
- if (output === 'NO_BACKUPS' || !output) {
120
- return [];
121
- }
122
-
123
- return output
124
- .split('\n')
125
- .filter((line) => line.includes('.tar.gz') || line.includes('.sql.gz'))
126
- .map((line) => {
127
- const parts = line.split(/\s+/);
128
- return {
129
- size: parts[4],
130
- date: `${parts[5]} ${parts[6]} ${parts[7]}`,
131
- name: parts[parts.length - 1].replace('/backups/', ''),
132
- };
133
- });
134
- } catch {
135
- return [];
136
- }
137
- }
138
-
139
94
  // ============================================================================
140
95
  // RESTORE OPERATIONS (k8s mode)
141
96
  // ============================================================================
@@ -175,175 +130,8 @@ function verifyPostgres(ip, sshKeyPath) {
175
130
  ]);
176
131
  }
177
132
 
178
- /**
179
- * Restore from a full archive (.tar.gz containing postgres.sql.gz + optional storage.tar.gz)
180
- */
181
- function restoreFromArchive(ip, sshKeyPath, remotePath) {
182
- const pod = getPostgresPod(ip, sshKeyPath);
183
-
184
- // Extract the archive on the server — argv form; remotePath is never shell-interpolated.
185
- sshRun(ip, sshKeyPath, ['mkdir', '-p', '/tmp/restore']);
186
- // GNU tar strips leading `/` by default (no `-P` / `--absolute-names`),
187
- // so absolute paths in a hostile archive never escape /tmp/restore.
188
- sshRun(ip, sshKeyPath, [
189
- 'tar',
190
- '--no-xattrs',
191
- '--no-same-owner',
192
- '-xzf',
193
- remotePath,
194
- '-C',
195
- '/tmp/restore',
196
- ]);
197
-
198
- // Check archive contents
199
- const contents = sshRun(ip, sshKeyPath, ['ls', '/tmp/restore/']);
200
-
201
- // Restore database
202
- if (contents.includes('postgres.sql.gz')) {
203
- sshKubectl(ip, sshKeyPath, [
204
- 'cp',
205
- '/tmp/restore/postgres.sql.gz',
206
- `vibecarbon/${pod}:/tmp/postgres.sql.gz`,
207
- ]);
208
- sshKubectl(ip, sshKeyPath, [
209
- 'exec',
210
- '-n',
211
- 'vibecarbon',
212
- pod,
213
- '--',
214
- 'sh',
215
- '-c',
216
- 'gunzip -c /tmp/postgres.sql.gz | psql -U supabase_admin postgres',
217
- ]);
218
- sshKubectl(ip, sshKeyPath, [
219
- 'exec',
220
- '-n',
221
- 'vibecarbon',
222
- pod,
223
- '--',
224
- 'rm',
225
- '-f',
226
- '/tmp/postgres.sql.gz',
227
- ]);
228
- }
229
-
230
- // Restore storage files if present and using file backend
231
- if (contents.includes('storage.tar.gz')) {
232
- // kubectl get returns empty string if STORAGE_BACKEND env var is unset;
233
- // default to 'file' so the legacy single-backend path still works.
234
- let storageBackend;
235
- try {
236
- storageBackend = sshKubectl(ip, sshKeyPath, [
237
- 'get',
238
- 'deployment',
239
- 'storage',
240
- '-n',
241
- 'vibecarbon',
242
- '-o',
243
- 'jsonpath={.spec.template.spec.containers[0].env[?(@.name=="STORAGE_BACKEND")].value}',
244
- ]);
245
- } catch {
246
- storageBackend = 'file';
247
- }
248
- if (!storageBackend) storageBackend = 'file';
249
-
250
- if (storageBackend === 'file') {
251
- const storagePod = sshKubectl(ip, sshKeyPath, [
252
- 'get',
253
- 'pods',
254
- '-n',
255
- 'vibecarbon',
256
- '-l',
257
- 'app.kubernetes.io/name=supabase-storage',
258
- '-o',
259
- 'jsonpath={.items[0].metadata.name}',
260
- ]);
261
-
262
- sshKubectl(ip, sshKeyPath, [
263
- 'cp',
264
- '/tmp/restore/storage.tar.gz',
265
- `vibecarbon/${storagePod}:/tmp/storage.tar.gz`,
266
- ]);
267
- sshKubectl(ip, sshKeyPath, [
268
- 'exec',
269
- '-n',
270
- 'vibecarbon',
271
- storagePod,
272
- '--',
273
- 'sh',
274
- '-c',
275
- 'tar --no-xattrs --no-same-owner -xzf /tmp/storage.tar.gz -C / && rm -f /tmp/storage.tar.gz',
276
- ]);
277
- } else {
278
- p.log.info('Storage backend is S3 — skipping file restore (files already in S3)');
279
- }
280
- }
281
-
282
- // Cleanup
283
- sshRun(ip, sshKeyPath, ['rm', '-rf', '/tmp/restore']);
284
- }
285
-
286
- /**
287
- * Restore from a legacy .sql.gz file (database only)
288
- */
289
- function restoreFromSqlGz(ip, sshKeyPath, backupName) {
290
- const pod = getPostgresPod(ip, sshKeyPath);
291
-
292
- // backupName is validated via validateBackupFilename at entry; still pass it
293
- // as a positional sh arg so the remote shell never expands it as code.
294
- sshKubectl(ip, sshKeyPath, [
295
- 'exec',
296
- '-n',
297
- 'vibecarbon',
298
- pod,
299
- '--',
300
- 'sh',
301
- '-c',
302
- 'gunzip -c "/backups/$1" | psql -U supabase_admin postgres',
303
- '_',
304
- backupName,
305
- ]);
306
- }
307
-
308
- async function restoreFromRemote(ip, sshKeyPath, backupName) {
309
- scaleDownApp(ip, sshKeyPath);
310
-
311
- if (backupName.endsWith('_full.tar.gz')) {
312
- restoreFromArchive(ip, sshKeyPath, `/tmp/${backupName}`);
313
- } else {
314
- restoreFromSqlGz(ip, sshKeyPath, backupName);
315
- }
316
-
317
- scaleUpApp(ip, sshKeyPath);
318
- verifyPostgres(ip, sshKeyPath);
319
- }
320
-
321
- async function uploadAndRestore(ip, sshKeyPath, localFile) {
322
- const filename = basename(localFile);
323
-
324
- // Upload local file to server
325
- scpUpload(ip, sshKeyPath, localFile, `/tmp/${filename}`);
326
-
327
- scaleDownApp(ip, sshKeyPath);
328
-
329
- if (filename.endsWith('_full.tar.gz')) {
330
- restoreFromArchive(ip, sshKeyPath, `/tmp/${filename}`);
331
- } else if (filename.endsWith('.sql.gz')) {
332
- const pod = getPostgresPod(ip, sshKeyPath);
333
- sshKubectl(ip, sshKeyPath, [
334
- 'cp',
335
- `/tmp/${filename}`,
336
- `vibecarbon/${pod}:/backups/${filename}`,
337
- ]);
338
- restoreFromSqlGz(ip, sshKeyPath, filename);
339
- }
340
-
341
- // Clean up temp file on host
342
- sshRun(ip, sshKeyPath, ['rm', '-f', `/tmp/${filename}`]);
343
-
344
- scaleUpApp(ip, sshKeyPath);
345
- verifyPostgres(ip, sshKeyPath);
346
- }
133
+ // (k8s pg_dump restore helpers removed — k8s restore is now wal-g-native via
134
+ // the db pod's init container; see runK8sRestore below.)
347
135
 
348
136
  // ============================================================================
349
137
  // SOURCE INTERPRETATION
@@ -511,19 +299,32 @@ export async function run(args) {
511
299
  // restores) ask for "the most recent" without first listing + parsing
512
300
  // names from `vibecarbon backup -l`.
513
301
  const wantsLatest = values.source === 'latest';
514
- const chosenSource =
515
- !wantsLatest && values.source
516
- ? classifySource(/** @type {string} */ (values.source))
517
- : await pickInteractiveSource({
518
- isCompose,
519
- envName,
520
- projectName,
521
- serverIp,
522
- sshKeyPath,
523
- s3,
524
- useS3,
525
- yes: wantsLatest || !!values.y,
526
- });
302
+ let chosenSource;
303
+ if (!wantsLatest && values.source) {
304
+ chosenSource = classifySource(/** @type {string} */ (values.source));
305
+ } else if (wantsLatest && !isCompose) {
306
+ // k8s restore is wal-g-native: `latest` resolves to the LATEST base backup
307
+ // INSIDE the cluster — the db pod's walg-restore init container runs
308
+ // `wal-g backup-fetch LATEST` (see runK8sRestore). pickInteractiveSource
309
+ // lists pg_dump `*_full.tar.gz` backups via listS3Backups, which cannot see
310
+ // wal-g `base_*` objects, so routing k8s `-source latest` through it wrongly
311
+ // exits "No backups found in S3" even though a wal-g base backup exists.
312
+ // Hand runK8sRestore the `latest` sentinel directly and let wal-g resolve
313
+ // it. (RCA 2026-05-31: e2e k8s/k8s-ha restore step — never reached before
314
+ // the backup chain (#4/#5/#6) was fixed.)
315
+ chosenSource = { kind: 's3', name: 'latest' };
316
+ } else {
317
+ chosenSource = await pickInteractiveSource({
318
+ isCompose,
319
+ envName,
320
+ projectName,
321
+ serverIp,
322
+ sshKeyPath,
323
+ s3,
324
+ useS3,
325
+ yes: wantsLatest || !!values.y,
326
+ });
327
+ }
527
328
 
528
329
  if (!chosenSource) {
529
330
  p.log.error('No backup selected.');
@@ -579,8 +380,6 @@ export async function run(args) {
579
380
  envName,
580
381
  serverIp,
581
382
  sshKeyPath,
582
- s3,
583
- useS3,
584
383
  });
585
384
  }
586
385
  } catch (error) {
@@ -662,7 +461,7 @@ async function runList({ isCompose, envName, projectName, serverIp, sshKeyPath }
662
461
  }
663
462
 
664
463
  // K8s pod fallback
665
- const backups = listRemoteBackups(serverIp, sshKeyPath);
464
+ const backups = listPodBackups(serverIp, sshKeyPath);
666
465
  printBackupList(backups, envName);
667
466
  }
668
467
 
@@ -757,7 +556,7 @@ async function pickInteractiveSource({
757
556
 
758
557
  // K8s pod fallback
759
558
  s.start('Fetching available backups');
760
- const backups = listRemoteBackups(serverIp, sshKeyPath);
559
+ const backups = listPodBackups(serverIp, sshKeyPath);
761
560
  s.stop('Backups retrieved');
762
561
  if (backups.length === 0) {
763
562
  p.log.error('No backups found on the server.');
@@ -822,42 +621,76 @@ async function runComposeRestore({
822
621
  void envName;
823
622
  }
824
623
 
825
- async function runK8sRestore({ s, chosenSource, envName, serverIp, sshKeyPath, s3, useS3 }) {
826
- s.start('Restoring database (app will be temporarily unavailable)');
624
+ // k8s restore is wal-g-native: postgres + wal-g live in the supabase-db pod,
625
+ // which continuously archives WAL + base backups to S3. Restore = set a
626
+ // RESTORE_TARGET marker on vibecarbon-secrets → bounce the db StatefulSet so
627
+ // its walg-restore init container runs `wal-g backup-fetch` into PGDATA before
628
+ // postgres starts → clear the marker. No bytes flow through the operator and
629
+ // no pg_dump/kubectl-cp; wal-g pulls from S3 inside the cluster. Mirrors the
630
+ // compose wal-g restore, adapted to the StatefulSet/init-container shape.
631
+ const DB_STATEFULSET = 'supabase-supabase-db';
632
+
633
+ /** Set or clear the RESTORE_TARGET marker the init container reads. */
634
+ function setRestoreMarker(ip, sshKeyPath, value) {
635
+ sshKubectl(ip, sshKeyPath, [
636
+ 'patch',
637
+ 'secret',
638
+ 'vibecarbon-secrets',
639
+ '-n',
640
+ 'vibecarbon',
641
+ '--type=merge',
642
+ '-p',
643
+ JSON.stringify({ stringData: { RESTORE_TARGET: value } }),
644
+ ]);
645
+ }
827
646
 
647
+ async function runK8sRestore({ s, chosenSource, envName, serverIp, sshKeyPath }) {
828
648
  if (chosenSource.kind === 'local') {
829
- if (!chosenSource.path.endsWith('.sql.gz') && !chosenSource.path.endsWith('.tar.gz')) {
830
- throw new Error('Backup file must be a .sql.gz or .tar.gz file');
831
- }
832
- await perfAsync('restore.uploadAndRestore', () =>
833
- uploadAndRestore(serverIp, sshKeyPath, chosenSource.path),
834
- );
835
- s.stop('Database restored');
836
- } else if (useS3) {
837
- s.message('Downloading backup from S3');
838
- const tempPath = join(process.cwd(), `.restore-tmp-${chosenSource.name}`);
839
- await perfAsync('restore.downloadS3', () =>
840
- downloadS3Backup(s3, `backups/${chosenSource.name}`, tempPath),
841
- );
842
- s.message('Uploading backup to server');
843
- await perfAsync('restore.scpUpload', async () =>
844
- scpUpload(serverIp, sshKeyPath, tempPath, `/tmp/${chosenSource.name}`),
649
+ throw new Error(
650
+ 'k8s restore is wal-g-based and pulls from S3 — local backup files are not supported. Use `-source latest` (or a timestamp for PITR).',
845
651
  );
846
- try {
847
- unlinkSync(tempPath);
848
- } catch {
849
- // ignore cleanup errors
850
- }
851
- s.message('Restoring database');
852
- await perfAsync('restore.restoreFromRemote', () =>
853
- restoreFromRemote(serverIp, sshKeyPath, chosenSource.name),
854
- );
855
- s.stop('Database restored');
856
- } else {
857
- // Pod-only fallback: backup name refers to a file already in the pod.
858
- await restoreFromRemote(serverIp, sshKeyPath, chosenSource.name);
859
- s.stop('Database restored');
860
652
  }
653
+ // wal-g target: 'latest' → LATEST base backup; otherwise treat the source
654
+ // name as an ISO-8601 timestamp for point-in-time recovery.
655
+ const target =
656
+ chosenSource.name === 'latest' || !chosenSource.name ? 'latest' : chosenSource.name;
657
+
658
+ s.start('Restoring database via wal-g (app will be temporarily unavailable)');
659
+ // 1. Mark the restore so the db pod's init container fetches on next boot.
660
+ setRestoreMarker(serverIp, sshKeyPath, target);
661
+ // 2. Quiesce the app so it isn't querying mid-swap.
662
+ await perfAsync('restore.scaleDownApp', async () => scaleDownApp(serverIp, sshKeyPath));
663
+ try {
664
+ // 3. Bounce the db StatefulSet → walg-restore init container clears PGDATA
665
+ // and runs `wal-g backup-fetch` before postgres starts.
666
+ s.message('Fetching from S3 + recovering (wal-g)');
667
+ await perfAsync('restore.walgFetch', async () => {
668
+ sshKubectl(serverIp, sshKeyPath, [
669
+ 'rollout',
670
+ 'restart',
671
+ `statefulset/${DB_STATEFULSET}`,
672
+ '-n',
673
+ 'vibecarbon',
674
+ ]);
675
+ sshKubectl(serverIp, sshKeyPath, [
676
+ 'rollout',
677
+ 'status',
678
+ `statefulset/${DB_STATEFULSET}`,
679
+ '-n',
680
+ 'vibecarbon',
681
+ '--timeout=600s',
682
+ ]);
683
+ });
684
+ // 4. Confirm postgres accepts connections post-recovery.
685
+ await perfAsync('restore.verifyPostgres', async () => verifyPostgres(serverIp, sshKeyPath));
686
+ } finally {
687
+ // 5. Always clear the marker so an unrelated future pod restart does NOT
688
+ // re-fetch and wipe live data. Runs even if the restore failed.
689
+ setRestoreMarker(serverIp, sshKeyPath, '');
690
+ }
691
+ // 6. Bring the app back.
692
+ await perfAsync('restore.scaleUpApp', async () => scaleUpApp(serverIp, sshKeyPath));
693
+ s.stop('Database restored');
861
694
 
862
695
  p.log.success('Restore completed successfully');
863
696
  p.log.info('The application has been scaled back up.');
package/src/scale.js CHANGED
@@ -25,6 +25,7 @@ import { c, printBanner } from './lib/colors.js';
25
25
  import { runCommand } from './lib/command.js';
26
26
  import { loadCredentials, saveProjectConfig } from './lib/config.js';
27
27
  import { buildCostBreakdown, formatCostLines } from './lib/cost.js';
28
+ import { useDnsChallenge } from './lib/deploy/acme.js';
28
29
  import {
29
30
  backupCompose,
30
31
  dockerLoginOnServer,
@@ -265,6 +266,11 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
265
266
  const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}`);
266
267
  const sshPubKeyPath = `${sshKeyPath}.pub`;
267
268
  const domain = envConfig.domain || null;
269
+ // Managed-DNS (cloudflare/hetzner) deploys issue certs via ACME DNS-01, so
270
+ // the new/recreated server needs the DNS-01 Traefik override and has no
271
+ // HTTP-01 race to wait on or reset. `manual` keeps HTTP-01.
272
+ const dnsProvider = envConfig.dnsProvider || envConfig.dns?.provider || null;
273
+ const dnsChallenge = useDnsChallenge(dnsProvider);
268
274
 
269
275
  // Read SSH public key and register/reuse in Hetzner
270
276
  const sshPubKey = readFileSync(sshPubKeyPath, 'utf-8').trim();
@@ -383,6 +389,14 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
383
389
  n8n: services.n8n,
384
390
  metabase: services.metabase,
385
391
  redis: services.redis,
392
+ // DNS-01: ship the Traefik override + re-affirm ACME_DNS_PROVIDER.
393
+ // The provider token flows through `envOverrides: oldEnv` (the
394
+ // original deploy baked it into the server .env); tokens are passed
395
+ // too so a fresh bundle still works if the old server was unreachable.
396
+ dnsChallenge,
397
+ dnsProvider,
398
+ hetznerApiToken: apiToken,
399
+ cloudflareApiToken: loadCredentials().cloudflare?.apiToken || null,
386
400
  // Replay every env var the old server had. renderBundle's existing
387
401
  // envOverrides path overlays these onto the project-local `.env`
388
402
  // baseline, then re-applies the deploy-time overrides (DOMAIN,
@@ -508,34 +522,41 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
508
522
  );
509
523
  s.stop('Database restored');
510
524
 
511
- // 9a. Update DNS BEFORE recreating services so Traefik can get ACME certs.
512
- // Traefik attempts HTTP-01 challenges immediately on startup; if DNS still
513
- // points to the old server, all challenges fail and the next retry is 30+ min.
525
+ // 9a. Update DNS to the new server. On HTTP-01 this must happen BEFORE
526
+ // recreating services (Traefik challenges the A record on startup; a
527
+ // stale record fails all challenges with a 30+ min retry). On DNS-01 the
528
+ // ordering is irrelevant — lego validates a TXT record, not the A record.
514
529
  if (domain) {
515
530
  s.start('Updating DNS to new server...');
516
531
  await perfAsync('scale.updateDNS', () => updateDNS(envConfig, apiToken, domain, newIp));
517
532
  s.stop('DNS updated');
518
533
 
519
- // Poll until Cloudflare/Google DNS returns the new IP (max 120s).
520
- // Fixed waits are unreliable propagation speed varies per resolver.
521
- s.start('Waiting for DNS propagation...');
522
- const dnsOk = await perfAsync('scale.waitForDNS', () =>
523
- waitForDNSPropagation(domain, newIp, 120_000),
524
- );
525
- s.stop(dnsOk ? 'DNS propagated' : 'DNS propagation timed out (proceeding anyway)');
534
+ // HTTP-01 only: poll until public resolvers return the new IP (max
535
+ // 120s) so Traefik's first challenge sees it. DNS-01 doesn't gate on
536
+ // A-record propagation, so skip the wait.
537
+ if (!dnsChallenge) {
538
+ s.start('Waiting for DNS propagation...');
539
+ const dnsOk = await perfAsync('scale.waitForDNS', () =>
540
+ waitForDNSPropagation(domain, newIp, 120_000),
541
+ );
542
+ s.stop(dnsOk ? 'DNS propagated' : 'DNS propagation timed out (proceeding anyway)');
543
+ }
526
544
  }
527
545
 
528
- // 9b. Reset ACME state so Traefik retries cert challenges with the correct DNS.
529
- // startComposeStack (step 6) ran before DNS was updated — all challenges failed
530
- // and are cached as errors in the letsencrypt_data volume.
531
- // Use docker compose exec (Traefik is already running) instead of pulling alpine.
546
+ // 9b. HTTP-01 only: reset ACME state so Traefik retries challenges with
547
+ // the corrected DNS — step 6's startComposeStack ran before the A record
548
+ // moved, so failed challenges are cached as errors in letsencrypt_data.
549
+ // DNS-01 has no such cached HTTP-01 failure to clear (and wiping acme.json
550
+ // would force a needless re-issue), so skip it.
532
551
  // Uses an SCP'd script so quoting/redirection stay intact without local shell.
533
- sshRunScript(
534
- newIp,
535
- sshKeyPath,
536
- `cd ${remoteDir} && docker compose exec -T traefik sh -c 'echo "{}" > /letsencrypt/acme.json && chmod 600 /letsencrypt/acme.json'`,
537
- { timeout: 30_000 },
538
- );
552
+ if (!dnsChallenge) {
553
+ sshRunScript(
554
+ newIp,
555
+ sshKeyPath,
556
+ `cd ${remoteDir} && docker compose exec -T traefik sh -c 'echo "{}" > /letsencrypt/acme.json && chmod 600 /letsencrypt/acme.json'`,
557
+ { timeout: 30_000 },
558
+ );
559
+ }
539
560
 
540
561
  // 9c. Recreate all services after restore so they reconnect to the restored DB.
541
562
  // Include all feature compose file flags so feature containers (redis, n8n,
@@ -545,6 +566,8 @@ async function scaleCompose(environment, envConfig, projectConfig, options = {})
545
566
  const composeFlags = [
546
567
  '-f docker-compose.yml',
547
568
  '-f docker-compose.prod.yml',
569
+ // DNS-01 override must follow prod.yml so it replaces the Traefik command.
570
+ ...(dnsChallenge ? ['-f docker-compose.dns01.prod.yml'] : []),
548
571
  ...(services.observability
549
572
  ? ['-f docker-compose.observability.yml', '-f docker-compose.observability.prod.yml']
550
573
  : []),
@@ -1,75 +0,0 @@
1
- # syntax=docker/dockerfile:1
2
- FROM ubuntu:22.04
3
-
4
- # Base image is ubuntu, not alpine, because wal-g v3.x stopped publishing
5
- # alpine-musl prebuilt binaries — only ubuntu glibc builds remain. We need
6
- # wal-g v3+ for the current S3 + Postgres 16 wire protocol support.
7
-
8
- ENV DEBIAN_FRONTEND=noninteractive
9
-
10
- # Install PostgreSQL 16 client + utilities. PG 16 isn't in the default
11
- # ubuntu 22.04 repos; pull it from the postgresql apt repo. AWS CLI is
12
- # installed as the standalone v2 binary further down (the apt-shipped
13
- # `awscli` is v1 and pulls ~500MB of Python deps; v2 is ~80MB self-
14
- # contained and the only version AWS itself supports for new code).
15
- #
16
- # apt cache mount: keeps /var/cache/apt populated across rebuilds so
17
- # `apt-get install` reuses .deb files. The cache lives outside the layer,
18
- # so we don't need the historical `rm -rf /var/lib/apt/lists/*` to keep
19
- # the final image small — that's still done below as a belt-and-braces
20
- # measure for cases where the cache mount isn't honored (e.g. legacy
21
- # `docker build` without BuildKit).
22
- # `sharing=locked` serializes parallel builds against the same cache
23
- # (apt's lock semantics expect exclusive access).
24
- RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
25
- --mount=type=cache,target=/var/lib/apt,sharing=locked \
26
- apt-get update && apt-get install -y --no-install-recommends \
27
- ca-certificates curl gnupg lsb-release \
28
- bash tar gzip unzip && \
29
- install -d /usr/share/postgresql-common/pgdg && \
30
- curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \
31
- -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc && \
32
- echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" \
33
- > /etc/apt/sources.list.d/pgdg.list && \
34
- apt-get update && apt-get install -y --no-install-recommends \
35
- postgresql-client-16
36
-
37
- # Install AWS CLI v2 standalone binary (~80MB) instead of `apt-get install
38
- # awscli` (v1 with ~500MB of Python/boto3 deps). Saves ~400MB on the
39
- # final image, which matters because the backup image is sideloaded to
40
- # every k3s node (~6 nodes in HA = ~2.4GB less network per deploy).
41
- # --fail makes curl exit non-zero on HTTP 4xx/5xx instead of piping a
42
- # 404 HTML body into unzip.
43
- RUN curl -fL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip \
44
- -o /tmp/awscli.zip && \
45
- unzip -q /tmp/awscli.zip -d /tmp && \
46
- /tmp/aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli && \
47
- rm -rf /tmp/aws /tmp/awscli.zip
48
-
49
- # Install WAL-G v3.0.5 (ubuntu 22.04 amd64 build).
50
- # --fail makes curl exit non-zero on HTTP 4xx/5xx instead of piping a 404
51
- # HTML body into tar (the previous v3.0.3-alpine URL 404'd, tar saw "<html>"
52
- # and errored "not in gzip format" — observed 2026-04-26 matrix run #2).
53
- RUN curl -fL https://github.com/wal-g/wal-g/releases/download/v3.0.5/wal-g-pg-ubuntu-22.04-amd64.tar.gz \
54
- | tar -xz && \
55
- mv wal-g-pg-ubuntu-22.04-amd64 /usr/local/bin/wal-g && \
56
- chmod +x /usr/local/bin/wal-g
57
-
58
- # Security: non-root system user. We can't reuse the name `backup` —
59
- # ubuntu pre-populates a `backup` system group (GID 34, used for system
60
- # backup operations) and `groupadd backup` errors with exit 9 ("group
61
- # 'backup' already exists"). The previous alpine version sidestepped
62
- # this because alpine doesn't ship the same default. Use a unique name
63
- # (`vbbackup`) and let --system pick a free system GID/UID.
64
- # The k8s manifest doesn't pin runAsUser so any UID works.
65
- RUN groupadd --system vbbackup && \
66
- useradd --system -g vbbackup -s /usr/sbin/nologin -M vbbackup
67
-
68
- # Copy backup script
69
- COPY backup.sh /usr/local/bin/backup.sh
70
- RUN chmod +x /usr/local/bin/backup.sh
71
-
72
- USER vbbackup
73
-
74
- # CronJob handles scheduling — script runs once and exits
75
- ENTRYPOINT ["/usr/local/bin/backup.sh"]