vibecarbon 0.1.0

package/src/lib/iac/cloud-init.js

@@ -0,0 +1,52 @@
+/**
+ * Shared helpers for loading + rendering the k3s cloud-init scripts in
+ * `carbon/cloud-init/k3s/` (master-init.sh / supabase-init.sh /
+ * worker-init.sh).
+ *
+ * Used by:
+ *   - `src/lib/iac/programs/hetzner-k8s.js` to render user_data baked
+ *     into the master/supabase/static-worker servers at Pulumi up.
+ *   - `src/lib/deploy/k8s/k3s.js` (`renderCAClusterConfig`) to render
+ *     the worker-init.sh body that cluster-autoscaler stuffs into its
+ *     `cluster-config.json` Secret. CA spawns workers on demand and
+ *     base64-encodes this body into user_data on each `ServerCreate`.
+ */
+
+import { readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// Cloud-init scripts live in carbon/cloud-init/k3s/.
+// Relative to this file: src/lib/iac/ → ../../../carbon/cloud-init/k3s
+const CLOUD_INIT_DIR = join(__dirname, '../../../carbon/cloud-init/k3s');
+
+/**
+ * Read a cloud-init script template by basename (e.g. 'master-init.sh').
+ * Returns the raw template — `${var}` placeholders are NOT yet
+ * substituted; pass to `renderScript()` to substitute.
+ *
+ * @param {string} name  Basename inside carbon/cloud-init/k3s/.
+ * @returns {string}     UTF-8 contents of the script.
+ */
+export function loadCloudInit(name) {
+  return readFileSync(join(CLOUD_INIT_DIR, name), 'utf-8');
+}
+
+/**
+ * Substitute ${var} placeholders in a cloud-init script template with
+ * values from a JS object. Used to render the master/supabase/worker
+ * cloud-init scripts at deploy time.
+ *
+ * @param {string} tmpl
+ * @param {Record<string, unknown>} vars
+ * @returns {string}
+ */
+export function renderScript(tmpl, vars) {
+  return Object.entries(vars).reduce(
+    (acc, [k, v]) => acc.replace(new RegExp(`\\$\\{${k}\\}`, 'g'), String(v ?? '')),
+    tmpl,
+  );
+}

package/src/lib/iac/index.js

@@ -0,0 +1,325 @@
+/**
+ * Pulumi-based Infrastructure-as-Code runtime for Vibecarbon.
+ *
+ * Entry point for Pulumi's Automation API. Programs live under
+ * `src/lib/iac/programs/` and are invoked from here. State is stored in
+ * the user's Hetzner S3 bucket by default; we fall back to a local file
+ * backend when no S3 is configured.
+ *
+ * Design notes:
+ * - We use inline programs (Automation API, no Pulumi.yaml files) so there's
+ *   nothing for users to manage on disk. Stack state lives in the backend;
+ *   the program code lives in this npm package.
+ * - One project name ("vibecarbon"), many stacks (one per environment like
+ *   "prod", "staging"). HA deploys use two stacks: "<env>-primary" and
+ *   "<env>-standby".
+ * - Pulumi's own secrets provider isn't used — secrets stay in
+ *   ~/.vibecarbon/credentials.json; we pass them to programs as plain config.
+ */
+
+import { mkdirSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { LocalWorkspace } from '@pulumi/pulumi/automation/index.js';
+
+const PULUMI_PROJECT = 'vibecarbon';
+const LOCAL_STATE_DIR = join(homedir(), '.vibecarbon', 'pulumi-state');
+
+// Pulumi's file:// backend requires the target directory to exist before
+// `stack select` runs, otherwise the first scenario to race for a stack
+// fails with `error: unable to open bucket file:///... no such file or
+// directory`. Create it at module load so every call path sees it.
+mkdirSync(LOCAL_STATE_DIR, { recursive: true });
+
+/**
+ * Compute the Pulumi backend URL. Prefers Hetzner S3 when s3Config is
+ * provided; otherwise falls back to a local file:// backend under
+ * ~/.vibecarbon/pulumi-state. Pulumi Cloud is opt-in via
+ * PULUMI_BACKEND_URL env var before invoking vibecarbon.
+ *
+ * Hetzner S3 uses path-style URLs via the Hetzner-provided endpoint, so we
+ * pass both `endpoint` and `s3ForcePathStyle=true` in the query string.
+ */
+export function resolveBackendUrl(s3Config) {
+  if (process.env.PULUMI_BACKEND_URL) {
+    return process.env.PULUMI_BACKEND_URL;
+  }
+  if (s3Config?.bucket && s3Config?.endpoint) {
+    const endpointHost = s3Config.endpoint.replace(/^https?:\/\//, '');
+    const region = s3Config.region || 'nbg1';
+    return `s3://${s3Config.bucket}?endpoint=${endpointHost}&region=${region}&s3ForcePathStyle=true`;
+  }
+  return `file://${LOCAL_STATE_DIR}`;
+}
+
+/**
+ * Build the env var bag Pulumi needs. Hetzner provider reads HCLOUD_TOKEN;
+ * S3 backend reads AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY; our programs
+ * read any additional config we pass through `stack.setConfig(...)`.
+ */
+function buildEnv(options) {
+  const env = { ...process.env };
+  if (options.hcloudToken) env.HCLOUD_TOKEN = options.hcloudToken;
+  if (options.s3Config?.accessKey) env.AWS_ACCESS_KEY_ID = options.s3Config.accessKey;
+  if (options.s3Config?.secretKey) env.AWS_SECRET_ACCESS_KEY = options.s3Config.secretKey;
+  // Disable Pulumi's telemetry for solo-hosted users. No data leaves the box.
+  env.PULUMI_SKIP_UPDATE_CHECK = 'true';
+  // Inline programs don't need a passphrase — but Pulumi insists on one for
+  // the secrets provider. A deterministic empty value keeps the automation
+  // API happy without us caring about secrets encryption at the state level
+  // (we never store secrets in Pulumi state; they flow in at program time).
+  env.PULUMI_CONFIG_PASSPHRASE = options.configPassphrase || '';
+  return env;
+}
+
+/**
+ * Create or select a stack with the given name + inline program. Returns
+ * the Stack instance (caller drives up/destroy/refresh).
+ */
+export async function getOrCreateStack(stackName, program, options = {}) {
+  const backendUrl = resolveBackendUrl(options.s3Config);
+  const envVars = buildEnv(options);
+
+  const stack = await LocalWorkspace.createOrSelectStack(
+    {
+      stackName,
+      projectName: PULUMI_PROJECT,
+      program,
+    },
+    {
+      projectSettings: {
+        name: PULUMI_PROJECT,
+        runtime: 'nodejs',
+        backend: { url: backendUrl },
+      },
+      envVars,
+    },
+  );
+
+  return stack;
+}
+
+/**
+ * Pre-initialize a Pulumi S3 backend by creating a no-op stack and removing
+ * it. The first `pulumi stack select` against a freshly-created S3 bucket
+ * fails with `could not list bucket: blob (code=NotFound): NoSuchKey`
+ * because the backend hasn't laid down its `.pulumi/meta.yaml` yet. When
+ * `deployK8sHA` launches primary + standby `deployK3s` in parallel against
+ * the SAME bucket, both threads race the bucket-init and both see the
+ * NoSuchKey error, killing the deploy in <20s. Calling this once
+ * sequentially before the parallel block writes the backend metadata so
+ * subsequent stack ops succeed.
+ *
+ * Idempotent: if the bucket is already initialized, the no-op stack is
+ * created and removed cleanly. If it fails (e.g., bucket access issue),
+ * we log and proceed — the parallel deploys will surface the underlying
+ * problem with a clearer error.
+ *
+ * @param {object} options - Same shape as getOrCreateStack options
+ *   (s3Config, hcloudToken, configPassphrase). The stack name we use is
+ *   internal (`__init`); callers don't see it.
+ */
+export async function initBackend(options = {}) {
+  const initStackName = '__init';
+  // No-op program — returns nothing. We don't run `up` so this never
+  // executes Hetzner API calls. The act of `createStack` writing the
+  // stack's checkpoint file to S3 forces the backend to lay down its
+  // metadata.
+  const noopProgram = async () => ({});
+  try {
+    const stack = await getOrCreateStack(initStackName, noopProgram, options);
+    // Remove it so it doesn't leak into stack listings.
+    await stack.workspace.removeStack(initStackName).catch(() => {});
+  } catch (err) {
+    console.error(`[pulumi] backend pre-init warning: ${err.message?.split('\n')[0] || err}`);
+  }
+}
+
+/**
+ * Run `pulumi up` and return the raw outputs. Streams Pulumi logs to an
+ * optional `onOutput` sink so callers can forward progress to a spinner.
+ */
+export async function upStack(stackName, program, options = {}) {
+  const stack = await getOrCreateStack(stackName, program, options);
+  // Clear any stale update lock before `up` — when a prior run aborted
+  // mid-flight (Ctrl-C, scenario timeout, background-task stop), Pulumi's
+  // file backend leaves a `.lock.*` in the bucket and subsequent `up`
+  // calls fail with:
+  //
+  //   error: the stack is currently locked by 1 lock(s). Either wait for
+  //   the other process(es) to end or delete the lock file with
+  //   `pulumi cancel`.
+  //
+  // Since e2e runs one scenario per stack at a time, it's always
+  // safe to clear. `stack.cancel()` is idempotent (no-op if no lock).
+  try {
+    await stack.cancel();
+  } catch {
+    /* no lock — ignore */
+  }
+
+  // Refresh state before `up` to purge pending operations from a crashed
+  // prior run. Without this, Pulumi's `up` with "N pending operations
+  // from previous deployment" warnings can trigger a delete-before-replace
+  // on resources that don't actually exist in the cloud, hitting provider
+  // panics (e.g., hcloud@1.32.1 nil-deref on Firewall Delete). `refresh`
+  // reconciles state with live cloud resources and clears the pending-op
+  // markers. Idempotent — no-op when state already matches reality.
+  //
+  // Skip on truly-empty stacks: a freshly-created stack has only the
+  // bookkeeping `pulumi:pulumi:Stack` resource (no real cloud resources
+  // to reconcile). Refresh still pays plugin spawn + cloud API listing,
+  // ~3-8s on cold compose deploys for nothing. exportStack() failure
+  // falls through to the safer always-refresh path.
+  let needsRefresh = true;
+  try {
+    const state = await stack.exportStack();
+    const resources = state?.deployment?.resources ?? [];
+    needsRefresh = resources.some((r) => r.type !== 'pulumi:pulumi:Stack');
+  } catch {
+    /* exportStack failed — fall through and refresh as before */
+  }
+  if (needsRefresh) {
+    try {
+      await stack.refresh({ onOutput: options.onOutput, color: 'never' });
+    } catch (err) {
+      // refresh can fail if the state file is completely corrupt or a
+      // resource is unreachable. Log and proceed — `up` may still succeed
+      // or surface a cleaner error.
+      console.error(`[pulumi] pre-up refresh warning: ${err.message?.split('\n')[0] || err}`);
+    }
+  }
+
+  const result = await stack.up({
+    onOutput: options.onOutput,
+    color: 'never',
+  });
+  // Flatten outputs: { foo: { value: 'bar', secret: false } } → { foo: 'bar' }
+  const outputs = {};
+  for (const [key, { value }] of Object.entries(result.outputs || {})) {
+    outputs[key] = value;
+  }
+  return { outputs, summary: result.summary };
+}
+
+/**
+ * Run `pulumi destroy` and remove the stack. Idempotent — if the stack
+ * doesn't exist the function resolves without error.
+ */
+export async function destroyStack(stackName, program, options = {}) {
+  let stack;
+  try {
+    stack = await getOrCreateStack(stackName, program, options);
+  } catch (err) {
+    // Stack doesn't exist yet — treat as a no-op so destroy is idempotent.
+    if (/no stack named/i.test(err.message || '')) return { destroyed: false };
+    throw err;
+  }
+  // Refresh first so destroy tolerates resources that were already reaped
+  // out-of-band (e.g. Hetzner server deleted by a prior partial destroy or
+  // by direct API cleanup). Without this, hcloud returns 404 and pulumi
+  // treats it as a hard error — surfaced in orphan-stack cleanup as
+  // "server not found (not_found, ...)".
+  try {
+    await stack.refresh({ onOutput: options.onOutput, color: 'never' });
+  } catch (err) {
+    console.error(`[pulumi] pre-destroy refresh warning: ${err.message?.split('\n')[0] || err}`);
+  }
+  await stack.destroy({ onOutput: options.onOutput, color: 'never' });
+  await stack.workspace.removeStack(stackName);
+  return { destroyed: true };
+}
+
+/**
+ * Read the current outputs of a stack without running an update. Used by
+ * commands like `status`, `scale`, `failover` that need infra identifiers
+ * (IPs, IDs) but shouldn't drift the state.
+ */
+export async function getStackOutputs(stackName, program, options = {}) {
+  const stack = await getOrCreateStack(stackName, program, options);
+  const raw = await stack.outputs();
+  const outputs = {};
+  for (const [key, { value }] of Object.entries(raw)) {
+    outputs[key] = value;
+  }
+  return outputs;
+}
+
+/**
+ * Classify a prior-outputs probe for the k3sToken probe-and-replay path.
+ *
+ * `scale.js` reads the prior stack outputs before constructing the real
+ * Pulumi program so it can replay the deploy-time `k3sToken` and avoid
+ * destructive userData drift (see the master-replace defense). The probe
+ * has three observable outcomes:
+ *
+ *   - `recovered`: outputs has a non-empty k3sToken — replay it
+ *   - `empty`:     outputs object exists but k3sToken is missing or empty —
+ *                  legitimate when the stack is fresh (no prior `up`), but
+ *                  also what we'd see if the deploy somehow skipped the
+ *                  output declaration. Caller must not silently mint a
+ *                  fresh token; the master-replace defense catches the
+ *                  destructive case.
+ *   - `errored`:   the probe call itself threw — backend unreachable,
+ *                  expired creds, etc. Caller should surface loudly.
+ *
+ * Pure function; the Pulumi state read happens upstream.
+ *
+ * @param {{ outputs?: Record<string, unknown> | null, error?: Error | null }} probe
+ * @returns {{ status: 'recovered' | 'empty' | 'errored', priorK3sToken?: string, reason: string }}
+ */
+export function classifyK3sTokenProbe({ outputs, error } = {}) {
+  if (error) {
+    const msg = error.message?.split('\n')[0] || String(error);
+    return { status: 'errored', reason: `probe threw: ${msg}` };
+  }
+  if (!outputs || typeof outputs !== 'object') {
+    return { status: 'empty', reason: 'no outputs returned' };
+  }
+  const token = outputs.k3sToken;
+  if (typeof token === 'string' && token.length > 0) {
+    return { status: 'recovered', priorK3sToken: token, reason: 'k3sToken present' };
+  }
+  const keys = Object.keys(outputs);
+  return {
+    status: 'empty',
+    reason:
+      keys.length === 0
+        ? 'outputs object empty (fresh stack)'
+        : `outputs missing k3sToken (keys=${keys.join(',')})`,
+  };
+}
+
+/**
+ * Refresh the stack's view of reality without applying program changes.
+ * Useful before a partial scale operation.
+ */
+export async function refreshStack(stackName, program, options = {}) {
+  const stack = await getOrCreateStack(stackName, program, options);
+  await stack.refresh({ onOutput: options.onOutput, color: 'never' });
+}
+
+/**
+ * List stack names that exist in the configured backend for this project.
+ * Used by `destroy` to surface orphans (stacks without a matching envConfig).
+ * Returns an empty list if no backend is reachable yet.
+ */
+export async function listStacks(options = {}) {
+  const backendUrl = resolveBackendUrl(options.s3Config);
+  const envVars = buildEnv(options);
+  try {
+    const workspace = await LocalWorkspace.create({
+      projectSettings: {
+        name: PULUMI_PROJECT,
+        runtime: 'nodejs',
+        backend: { url: backendUrl },
+      },
+      envVars,
+    });
+    const stacks = await workspace.listStacks();
+    return stacks.map((s) => s.name);
+  } catch {
+    // No backend yet (fresh install) — no orphans possible.
+    return [];
+  }
+}

package/src/lib/iac/programs/hetzner-compose.js

@@ -0,0 +1,130 @@
+/**
+ * Pulumi program for a single Hetzner Docker Compose VPS.
+ *
+ * Much simpler than the k8s program — one server, one firewall, optional
+ * floating IP. Cloud-init installs Docker via the shipped docker-ce-setup.yaml
+ * (carbon/cloud-init/), which is reused as-is (no templating needed).
+ *
+ * Outputs: serverIp, serverId, firewallId, sshKeyId.
+ */
+
+import { readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as hcloud from '@pulumi/hcloud';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const CLOUD_INIT_PATH = join(__dirname, '../../../../carbon/cloud-init/docker-ce-setup.yaml');
+
+/**
+ * @typedef {Object} ComposeStackConfig
+ * @property {string} projectName
+ * @property {string} environment
+ * @property {string} sshPublicKey
+ * @property {string} [existingSshKeyId]
+ * @property {string} location
+ * @property {string} serverType
+ * @property {Record<string,string>} [labels]
+ * @property {string[]} [allowedSshIps]
+ */
+
+/**
+ * @param {ComposeStackConfig} config
+ * @returns {() => Promise<Record<string, unknown>>}
+ */
+export function buildHetznerComposeProgram(config) {
+  const name = `${config.projectName}-${config.environment}`;
+  const labels = {
+    project: config.projectName,
+    environment: config.environment,
+    ...(config.labels ?? {}),
+  };
+  // Operator-IP firewall lock (H-2). The deploy flow populates this from
+  // projectConfig.operatorCidrs; an empty list = nothing can SSH in. We
+  // refuse to build a program that would silently apply an open default.
+  if (!config.allowedSshIps?.length) {
+    throw new Error(
+      'allowedSshIps required (no default; pass operatorCidrs from .vibecarbon.json or set ALLOWED_SSH_IPS=...)',
+    );
+  }
+  const allowedSshIps = config.allowedSshIps;
+  const userData = readFileSync(CLOUD_INIT_PATH, 'utf-8');
+
+  return async () => {
+    let sshKeyId;
+    if (config.existingSshKeyId) {
+      sshKeyId = config.existingSshKeyId;
+    } else {
+      // Name includes region so parallel or re-deploys across regions don't
+      // collide in Hetzner's per-project SSH-key namespace.
+      const sshKey = new hcloud.SshKey('ssh-key', {
+        name: `${name}-${config.location}-key`,
+        publicKey: config.sshPublicKey,
+        labels,
+      });
+      sshKeyId = sshKey.id;
+    }
+
+    const firewall = new hcloud.Firewall(
+      'firewall',
+      {
+        name: `${name}-firewall`,
+        labels,
+        rules: [
+          {
+            direction: 'in',
+            protocol: 'tcp',
+            port: '22',
+            sourceIps: allowedSshIps,
+            description: 'SSH',
+          },
+          {
+            direction: 'in',
+            protocol: 'tcp',
+            port: '80',
+            sourceIps: ['0.0.0.0/0', '::/0'],
+            description: 'HTTP',
+          },
+          {
+            direction: 'in',
+            protocol: 'tcp',
+            port: '443',
+            sourceIps: ['0.0.0.0/0', '::/0'],
+            description: 'HTTPS',
+          },
+        ],
+      },
+      // Upstream hcloud-go@1.32.1 bug: the Firewall `Update` code path
+      // nil-pointer-panics when rules change on a live firewall. Forcing
+      // full replacement avoids the Update call site entirely. Recreate
+      // is cheap (~2s).
+      { replaceOnChanges: ['*'], deleteBeforeReplace: true },
+    );
+
+    const server = new hcloud.Server('server', {
+      name,
+      serverType: config.serverType,
+      // Hetzner-maintained app image: Ubuntu LTS with docker + the
+      // docker-compose plugin pre-installed. Matches the contract in
+      // carbon/cloud-init/docker-ce-setup.yaml ("VPS boot time in parallel
+      // with Hetzner's default docker-ce app"). Plain `ubuntu-24.04` leaves
+      // the VM without docker and the deploy hits `docker: command not
+      // found` on the first `docker compose up` call.
+      image: 'docker-ce',
+      location: config.location,
+      sshKeys: [sshKeyId],
+      firewallIds: [firewall.id.apply((id) => Number.parseInt(id, 10))],
+      labels: { ...labels, role: 'compose' },
+      userData,
+    });
+
+    return {
+      serverIp: server.ipv4Address,
+      serverId: server.id.apply((id) => String(id)),
+      firewallId: firewall.id.apply((id) => String(id)),
+      sshKeyId: typeof sshKeyId === 'string' ? sshKeyId : sshKeyId.apply((id) => String(id)),
+    };
+  };
+}