redkite 0.1.7 → 0.1.9

package/src/deploy.ts

@@ -2,8 +2,9 @@ import { build, type BuildContext, type BuildResult } from "./build.js";
 import { assertCheckable, runChecks } from "./checks.js";
 import { environmentOf } from "./config.js";
 import { Docker } from "./docker.js";
+import { envFileFor } from "./environment.js";
 import { healthcheck, type HealthDeps } from "./health.js";
-import type { Host } from "./host.js";
+import { finalHost, type Host } from "./host.js";
 import { localHost } from "./localHost.js";
 import { silent, type Log } from "./log.js";
 import {
@@ -21,6 +22,7 @@ import {
   type Run,
   type Start,
 } from "./pipeline.js";
+import { pluginSteps } from "./plugin.js";
 import { readEnv, readRef, type SecretStores } from "./secrets/refs.js";
 import { ensureService } from "./services/ensure.js";
 import { plannedServices } from "./services/planned.js";
@@ -78,8 +80,11 @@ async function start(run: Run, options: DeployOptions): Promise<Finished> {
     run,
   };
 
-  const steps = merge(supplied(options), options.config.steps ?? []);
-  return await runPipeline(run, steps, setting, options.signal);
+  // A plugin's steps lead, so a snapshot listed as a plugin runs above the
+  // migration the deployment writes after it
+  const added = [...pluginSteps(options.config.plugins), ...(options.config.steps ?? [])];
+
+  return await runPipeline(run, merge(supplied(options), added), setting, options.signal);
 }
 
 // Every step redkite supplies, in the order the phases name. A run walks the
@@ -171,33 +176,66 @@ async function release(
     context.config.apps.map((app) => [app.name, app.environment ?? {}]),
   );
 
-  task.detail("retiring the running containers");
-  await Promise.all(apps.map((app) => retire(docker, topology, app)));
-
-  task.detail("creating the new ones");
-
-  await Promise.all(
-    apps.map((app) =>
-      create(docker, topology, app, environments.get(app.name)),
+  // Resolved now and handed over when the container is made. Nothing from the
+  // vault is in the image, so this is the only way the running process sees it
+  const files = new Map(
+    await Promise.all(
+      context.config.apps.map(
+        async (app) => [app.name, await envFileFor(app, context)] as const,
+      ),
     ),
   );
 
-  task.detail("starting them");
-  await Promise.all(apps.map((app) => docker.container.start(app.container)));
-
-  // One unhealthy app reverts all of them. A half-swapped deployment is the
-  // one state nothing downstream knows how to reason about
-  if (!(await checkAll(context, health))) {
-    context.log.fail("Health checks failed, reverting");
-    await Promise.all(apps.map((app) => revert(docker, topology, app)));
-
-    return {
-      ...input,
-      ok: false,
-      released: [],
-      reverted: apps.map((app) => app.container),
-      checked: [],
-    };
+  // What has actually been moved, rather than what was going to be. A stop
+  // lands between two docker commands, and only this says which side of it
+  const moved: AppTopology[] = [];
+
+  try {
+    task.detail("retiring the running containers");
+
+    await Promise.all(
+      apps.map(async (app) => {
+        await retire(docker, topology, app);
+        moved.push(app);
+      }),
+    );
+
+    task.detail("creating the new ones");
+
+    await Promise.all(
+      apps.map((app) =>
+        create(docker, topology, app, environments.get(app.name), files.get(app.name)),
+      ),
+    );
+
+    task.detail("starting them");
+    await Promise.all(apps.map((app) => docker.container.start(app.container)));
+
+    // Inside, because the swap is not over until this says so. A stop lands
+    // here more often than anywhere else: it is the longest part, and by now
+    // every address has already moved
+    if (!(await checkAll(context, health))) {
+      context.log.fail("Health checks failed, reverting");
+      await Promise.all(apps.map((app) => revert(docker, topology, app)));
+
+      return {
+        ...input,
+        ok: false,
+        released: [],
+        reverted: apps.map((app) => app.container),
+        checked: [],
+      };
+    }
+  } catch (error) {
+    // Whatever ended this, the addresses have moved and something has to put
+    // them back. A stop is the usual one, so the revert runs on a host that
+    // has been told to stop: it is the abort that made this necessary
+    if (moved.length > 0) {
+      context.log.fail(`Putting ${moved.length} back where they were`);
+      await putBack(context, moved);
+    }
+
+    throw error;
   }
 
   return {
@@ -209,6 +247,23 @@ async function release(
   };
 }
 
+// Through a host with the stop lifted, because the commands that undo a swap
+// cannot be refused by the same signal that interrupted it. One app failing to
+// go back must not stop the others, so each is settled on its own
+async function putBack(context: Context, moved: AppTopology[]) {
+  const docker = new Docker(finalHost(context.host));
+
+  const put = moved.map(async (app) => {
+    try {
+      await revert(docker, context.topology, app);
+    } catch (error) {
+      context.log.fail(`${app.container} could not be put back: ${String(error)}`);
+    }
+  });
+
+  await Promise.all(put);
+}
+
 // A build leaves its image on the host rather than sending one, so without this
 // every run adds a runtime image and a builder to a disk nobody is watching
 async function finish(input: Released, context: Context): Promise<Finished> {
@@ -342,6 +397,7 @@ async function create(
   topology: Topology,
   app: AppTopology,
   environment: Record<string, string> = {},
+  envFile?: string,
 ) {
   const builder = docker.container
     .builder()
@@ -357,19 +413,35 @@ async function create(
     builder.extraHost(host, ip);
   }
 
+  if (envFile) builder.envFile(envFile);
+
   for (const volume of app.volumes) builder.volume(volume.volume, volume.mountPath);
   for (const [name, value] of Object.entries(environment)) builder.env(name, value);
 
   await builder.create();
 }
 
-// Put the retired container back on the live address and its original name
-async function revert(docker: Docker, topology: Topology, app: AppTopology) {
+// Put the retired container back on the live address and its original name.
+// Answers with whether there was one: a first deploy has nothing behind it, and
+// the container that just failed is simply left where it was renamed to
+export async function revert(docker: Docker, topology: Topology, app: AppTopology) {
   await docker.container.stop(app.container);
+
+  // The slot may still hold what an earlier interrupted run put there, and a
+  // rename refuses rather than clobbers. Retiring clears its own slot the same
+  // way, and a revert that skipped this would leave the new container live
+  await docker.container.stop(app.failed);
+  await docker.container.remove(app.failed);
+
   await docker.container.rename(app.container, app.failed);
+
+  if (!(await docker.container.exists(app.retired))) return false;
+
   await docker.network.reconnect(topology.network, app.retired, app.currentAddress);
   await docker.container.rename(app.retired, app.container);
   await docker.container.start(app.container);
+
+  return true;
 }
 
 async function cleanup(docker: Docker, app: AppTopology) {

package/src/dockerfile.ts

@@ -82,22 +82,18 @@ function builderStage(spec: BuildSpec, context: DockerfileContext) {
   // Only now, so the steps below read the app's own manifest and write beside it
   if (context.dir) lines.push(`WORKDIR ${workdir}`);
 
-  lines.push(
-    `RUN --mount=type=secret,id=${context.envSecret} cp /run/secrets/${context.envSecret} ${workdir}/.env`,
-    `ENV SENTRY_RELEASE=${context.release}`,
-  );
+  lines.push(`ENV SENTRY_RELEASE=${context.release}`);
+
+  // Mounted for the length of each step rather than copied into the tree. A
+  // file that is copied is in that layer for good, and no later instruction
+  // can take it back out: rm leaves it readable in the layer underneath
+  const env = `--mount=type=secret,id=${context.envSecret},target=${workdir}/.env `;
 
   // Unquoted: the shell form hands everything after RUN to sh -c, so a step is
   // written exactly as it would be typed. Quoting it made the whole command one
   // word, and the shell went looking for a program by that name
   for (const step of spec.steps) {
-    lines.push(`RUN ${mounts}${step}`);
-  }
-
-  if (spec.sourcemaps) {
-    // The built .env ships inside the image, and an upload token is build-time
-    const token = spec.sourcemaps.stripFromImage;
-    lines.push(`RUN sed -i '/^${token}=/d' ${rootedAt(spec.output, context.dir)}/.env`);
+    lines.push(`RUN ${env}${mounts}${step}`);
   }
 
   return lines;

package/src/environment.ts

@@ -0,0 +1,24 @@
+import type { Context } from "./pipeline.js";
+import { readEnv } from "./secrets/refs.js";
+import type { AppSpec } from "./types.js";
+
+// What the vault resolved to, handed over as a file rather than baked into an
+// image. Nothing from a vault is in a layer, so this is the only way anything
+// running the app's image sees its environment: the container itself, and the
+// migrations and checks that run in the builder.
+
+// Written to the host and handed over as a file, so no value appears in an
+// argument list. Docker reads it when a container is created, and again for
+// each `run --env-file`, so the file goes with the rest of the deploy's scratch
+export async function envFileFor(app: AppSpec, context: Context) {
+  if (!app.secrets) return undefined;
+
+  const contents = await readEnv(app.secrets, context.secrets);
+  return await context.host.write(`apps/${app.name}/env`, contents);
+}
+
+// The flag, or nothing at all for an app that names no secrets
+export async function envFlags(app: AppSpec, context: Context) {
+  const path = await envFileFor(app, context);
+  return path ? [`--env-file ${path}`] : [];
+}

package/src/host.ts

@@ -25,9 +25,20 @@ export type Host = {
   // Signals everything this host started and answers with how many are still
   // running. Zero is the only answer that means nothing was left behind
   stop(signal: "TERM" | "KILL"): Promise<number>;
+  // A command that runs even once a stop has been asked for. Putting a swap
+  // back is the reason there is one: the abort is what made the work necessary,
+  // so it cannot also be what refuses to do it
+  final(command: string): Promise<Result>;
   close?(): Promise<void>;
 };
 
+// The same host with the stop lifted. Handed to a Docker, it gives every
+// command below it the same exemption, which is how a revert reaches the
+// containers an abort left half moved
+export function finalHost(host: Host): Host {
+  return { ...host, sh: async (command) => await host.final(command) };
+}
+
 // A streamed command keeps this many lines, so a build that prints tens of
 // megabytes still fits in the error a failure reports
 const TAIL = 200;

package/src/index.ts

@@ -2,6 +2,7 @@ export * from "./build.js";
 export * from "./config.js";
 export * from "./deploy.js";
 export * from "./docker.js";
+export * from "./environment.js";
 export * from "./dockerfile.js";
 export * from "./health.js";
 export * from "./host.js";
@@ -10,8 +11,10 @@ export * from "./localHost.js";
 export * from "./log.js";
 export * from "./nginx.js";
 export * from "./pipeline.js";
+export * from "./plugin.js";
+export * from "./plugins/index.js";
+export * from "./recover.js";
 export * from "./presets/index.js";
-export * from "./secrets/bitwarden.js";
 export * from "./secrets/refs.js";
 export * from "./secrets/store.js";
 export * from "./shell.js";

package/src/localHost.ts

@@ -43,6 +43,9 @@ export async function localHost(options: LocalOptions = {}): Promise<Host> {
     // is a process group leader, so one signal reaches the whole tree
     stop: async (name) => signalEverything(name === "KILL" ? "SIGKILL" : "SIGTERM"),
 
+    // Without the signal, so a stop cannot refuse the work it created
+    final: async (command) => await spawnCollect("sh", ["-c", command]),
+
     close: async () => {
       await rm(directory, { recursive: true, force: true });
     },

package/src/plugin.ts

@@ -0,0 +1,50 @@
+import { assertSteps, type AnyStep } from "./pipeline.js";
+import type { SecretStore } from "./secrets/refs.js";
+
+// What a deployment can be handed that redkite did not write. A plugin adds
+// work to the run, or teaches it to resolve a kind of secret, and nothing it
+// brings happens until a deployment lists it. Redkite's own vault is one of
+// these rather than something wired in behind them.
+
+export type OpenContext = {
+  // What the step row says while the store is being opened. Unlocking a vault
+  // is the slowest thing a deploy does before it starts building
+  detail: (message: string) => void;
+};
+
+// Opened once, and only when a ref names the provider it answers for
+export type OpenStore = (context: OpenContext) => Promise<SecretStore>;
+
+export type Plugin = {
+  // Says which one when a deployment registers the same plugin twice, and what
+  // a failure to open a store is reported against
+  name: string;
+  // Run in the order the deployment lists the plugins, above its own steps
+  steps?: AnyStep[];
+  // Provider tag to how that store is opened
+  stores?: Record<string, OpenStore>;
+};
+
+// Identity, but the points a plugin claims are checked where it is written
+// rather than where it is registered, so a typo is the plugin's own failure
+export function definePlugin<const T extends Plugin>(plugin: T): T {
+  assertSteps(plugin.steps ?? []);
+  return plugin;
+}
+
+export function pluginSteps(plugins: Plugin[] = []): AnyStep[] {
+  return plugins.flatMap((plugin) => plugin.steps ?? []);
+}
+
+// The one that answers for this provider. Two plugins claiming it would be two
+// vaults for one tag, which is a config nobody can read the intent of
+export function storeFor(plugins: Plugin[] = [], provider: string) {
+  const claiming = plugins.filter((plugin) => plugin.stores?.[provider]);
+
+  if (claiming.length > 1) {
+    const names = claiming.map((plugin) => plugin.name).join(" and ");
+    throw new Error(`${names} both resolve ${provider} secrets, and only one can`);
+  }
+
+  return claiming[0]?.stores?.[provider];
+}

package/src/plugins/bitwarden.ts

@@ -0,0 +1,57 @@
+import { definePlugin, type OpenContext, type Plugin } from "../plugin.js";
+import { bitwardenStore } from "../secrets/store.js";
+import type { SecretRef } from "../types.js";
+
+// Redkite's own vault, and a plugin like any other: a deployment that does not
+// register it cannot resolve a bitwarden ref, and says so before it builds.
+
+// The one variable a vault needs when the session was obtained elsewhere. A CI
+// job that unlocked once and reuses it across several deploys is the case
+const KEY = "BW_KEY";
+
+export type BitwardenOptions = {
+  // Whether this vault answers secret refs, and what unlocks it. true reads
+  // BW_KEY, and falls back to the api credentials when that is not set. A
+  // string is the session itself, for a config that names its own variable.
+  // false registers the plugin without a store, for a deployment that has
+  // stopped reading from it but has not yet taken the refs out
+  secrets?: boolean | string;
+};
+
+function vault(options: BitwardenOptions = {}): Plugin {
+  const secrets = options.secrets ?? true;
+  if (secrets === false) return definePlugin({ name: "bitwarden" });
+
+  return definePlugin({
+    name: "bitwarden",
+    stores: { bitwarden: async (context) => await open(secrets, context) },
+  });
+}
+
+async function open(secrets: true | string, context: OpenContext) {
+  const session = typeof secrets === "string" ? secrets : process.env[KEY];
+  if (session) return await bitwardenStore({ session, detail: context.detail });
+
+  return await bitwardenStore({
+    detail: context.detail,
+    clientId: required("BW_CLIENT_ID"),
+    clientSecret: required("BW_CLIENT_SECRET"),
+    password: required("BW_PASSWORD"),
+  });
+}
+
+function required(name: string) {
+  const value = process.env[name];
+  if (value) return value;
+
+  throw new Error(
+    `Neither ${KEY} nor ${name} is set, and this deployment reads its environment from Bitwarden`,
+  );
+}
+
+// The plugin is what a deployment registers; the item is what an app points at.
+// Both are named bitwarden because both are the same vault, and the id is a
+// pointer rather than a secret, so it belongs in the config
+export const bitwarden = Object.assign(vault, {
+  item: (id: string): SecretRef => ({ provider: "bitwarden", id }),
+});

package/src/plugins/digitalOcean.ts

@@ -0,0 +1,111 @@
+import type { Step } from "../pipeline.js";
+import { definePlugin, type Plugin } from "../plugin.js";
+
+import { slug, snapshotName, tokenFrom, type SnapshotPoint } from "./snapshot.js";
+
+// A snapshot of the disk a database sits on, taken before anything migrates it.
+//
+// DigitalOcean's managed databases have no endpoint that takes one on demand:
+// their backups are automatic and the API only lists them. So what this
+// snapshots is the block storage volume redkite's own postgres service keeps
+// its data on, or the droplet when the database is the whole machine. A volume
+// snapshot of a running Postgres is crash consistent rather than clean, which
+// Postgres is built to survive: it replays the log on the way back up.
+
+const API = "https://api.digitalocean.com/v2";
+
+// Injected for the same reason the ssh runner is: what this builds is asserted
+// on rather than trusted
+export type Request = (
+  url: string,
+  init: { method: string; headers: Record<string, string>; body: string },
+) => Promise<{ ok: boolean; status: number; text(): Promise<string> }>;
+
+export type DigitalOceanSnapshotOptions = {
+  // The block storage volume the data lives on, by id
+  volume?: string;
+  // Or the droplet, by id, for a database that is the whole machine
+  droplet?: number;
+  // Goes in front of the snapshot's name, which carries the environment and
+  // the minute after it
+  name?: string;
+  // Environment variable the API token is read from, never the token itself:
+  // a config is committed and a token is not
+  tokenFrom?: string;
+  request?: Request;
+};
+
+const TOKEN = "DIGITALOCEAN_TOKEN";
+
+export function digitalOceanSnapshot(options: DigitalOceanSnapshotOptions): Plugin {
+  assertOneTarget(options);
+
+  return definePlugin({
+    name: `digitalocean-snapshot-${slug(options.volume ?? String(options.droplet ?? ""))}`,
+    steps: [snapshotStep(options)],
+  });
+}
+
+function snapshotStep(options: DigitalOceanSnapshotOptions): Step<SnapshotPoint> {
+  const target = options.volume ?? String(options.droplet ?? "");
+  const send = options.request ?? ((url, init) => fetch(url, init));
+  const variable = options.tokenFrom ?? TOKEN;
+
+  return {
+    point: `swap:before:snapshot-${slug(target)}`,
+    check: (plan) => tokenFrom(variable, plan),
+
+    run: async (input, context) => {
+      const name = snapshotName(options.name ?? target, context.environment, new Date());
+      context.task.detail(`snapshotting ${target} as ${name}`);
+
+      const token = tokenFrom(variable, {
+        config: context.config,
+        environment: context.environment,
+      });
+
+      const answer = await send(urlFor(options), {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          authorization: `Bearer ${token}`,
+        },
+        body: JSON.stringify(bodyFor(options, name)),
+      });
+
+      if (!answer.ok) {
+        throw new Error(
+          `DigitalOcean refused the snapshot of ${target} (${answer.status}): ${await answer.text()}`,
+        );
+      }
+
+      return input;
+    },
+  };
+}
+
+function urlFor(options: DigitalOceanSnapshotOptions) {
+  if (options.volume) return `${API}/volumes/${options.volume}/snapshots`;
+  return `${API}/droplets/${options.droplet}/actions`;
+}
+
+// A volume takes a snapshot, a droplet is asked to perform one. Two endpoints
+// with two shapes, which is why the target is not one field
+function bodyFor(options: DigitalOceanSnapshotOptions, name: string) {
+  if (options.volume) return { name };
+  return { type: "snapshot", name };
+}
+
+// Checked where the plugin is written rather than where it runs, so the config
+// fails to load. The token cannot be: a plan reads the config without one
+function assertOneTarget(options: DigitalOceanSnapshotOptions) {
+  if (options.volume && options.droplet !== undefined) {
+    throw new Error(
+      "digitalOceanSnapshot names both a volume and a droplet, and they are different disks",
+    );
+  }
+
+  if (!options.volume && options.droplet === undefined) {
+    throw new Error("digitalOceanSnapshot names no volume or droplet to snapshot");
+  }
+}

package/src/plugins/index.ts

@@ -0,0 +1,4 @@
+export * from "./bitwarden.js";
+export * from "./digitalOcean.js";
+export * from "./rds.js";
+export * from "./snapshot.js";

package/src/plugins/rds.ts

@@ -0,0 +1,126 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+import type { Step } from "../pipeline.js";
+import { definePlugin, type Plugin } from "../plugin.js";
+
+import { slug, snapshotName, type SnapshotPoint } from "./snapshot.js";
+
+// A snapshot of a managed Postgres or MySQL, taken before anything migrates it.
+// Through the AWS CLI rather than the REST API: signing a request by hand is a
+// page of crypto that nothing here could check, and every runner already has
+// the CLI with the credentials the job was given.
+
+const run = promisify(execFile);
+
+// Injected so the argv this builds is asserted on rather than trusted
+export type Aws = (args: string[]) => Promise<{ stdout: string }>;
+
+export type RdsSnapshotOptions = {
+  // The instance to snapshot, or the cluster for Aurora. One of them
+  instance?: string;
+  cluster?: string;
+  // Goes in front of the snapshot's name, which carries the environment and
+  // the minute after it
+  name?: string;
+  region?: string;
+  // Waits for the snapshot to finish rather than only to start. RDS captures
+  // the data when it begins, so this is about learning it worked rather than
+  // about the restore point, and it can take a very long time
+  wait?: boolean;
+  aws?: Aws;
+};
+
+export function rdsSnapshot(options: RdsSnapshotOptions): Plugin {
+  assertOneTarget(options);
+  const target = slug(options.instance ?? options.cluster ?? "");
+
+  return definePlugin({
+    name: `rds-snapshot-${target}`,
+    steps: [snapshotStep(options)],
+  });
+}
+
+function snapshotStep(options: RdsSnapshotOptions): Step<SnapshotPoint> {
+  const target = options.instance ?? options.cluster ?? "";
+  const aws = options.aws ?? ((args: string[]) => run("aws", args));
+
+  return {
+    point: `swap:before:snapshot-${slug(target)}`,
+
+    run: async (input, context) => {
+      const identifier = snapshotName(options.name ?? target, context.environment, new Date());
+      context.task.detail(`snapshotting ${target} as ${identifier}`);
+
+      await aws(create(options, identifier));
+
+      // The data is captured when it begins, so this is not what makes the
+      // restore point good. It is what turns a snapshot that silently failed
+      // into a deploy that stops before the migration
+      if (options.wait) {
+        context.task.detail(`waiting for ${identifier}`);
+        await aws(ready(options, identifier));
+      }
+
+      return input;
+    },
+  };
+}
+
+function create(options: RdsSnapshotOptions, identifier: string) {
+  const region = options.region ? ["--region", options.region] : [];
+
+  if (options.cluster) {
+    return [
+      "rds",
+      "create-db-cluster-snapshot",
+      "--db-cluster-identifier",
+      options.cluster,
+      "--db-cluster-snapshot-identifier",
+      identifier,
+      ...region,
+    ];
+  }
+
+  return [
+    "rds",
+    "create-db-snapshot",
+    "--db-instance-identifier",
+    options.instance ?? "",
+    "--db-snapshot-identifier",
+    identifier,
+    ...region,
+  ];
+}
+
+function ready(options: RdsSnapshotOptions, identifier: string) {
+  const region = options.region ? ["--region", options.region] : [];
+
+  if (options.cluster) {
+    return [
+      "rds",
+      "wait",
+      "db-cluster-snapshot-available",
+      "--db-cluster-snapshot-identifier",
+      identifier,
+      ...region,
+    ];
+  }
+
+  return ["rds", "wait", "db-snapshot-available", "--db-snapshot-identifier", identifier, ...region];
+}
+
+// An instance and a cluster are different API calls against different things,
+// and guessing which was meant is worse than being told. Checked where the
+// plugin is written rather than where it runs, so the config fails to load
+function assertOneTarget(options: RdsSnapshotOptions) {
+  if (options.instance && options.cluster) {
+    throw new Error(
+      "rdsSnapshot names both an instance and a cluster, and they are different databases",
+    );
+  }
+
+  if (!options.instance && !options.cluster) {
+    throw new Error("rdsSnapshot names no instance or cluster to snapshot");
+  }
+}