redkite 0.1.7 → 0.1.8

package/src/cli/index.ts

@@ -7,13 +7,14 @@ import { silent, type Log } from "../log.js";
 import { renderNginx } from "../nginx.js";
 import { addressOf, RUNS, SLOTS, type Run } from "../pipeline.js";
 import { listRefs, type SecretStores } from "../secrets/refs.js";
-import { bitwardenStore } from "../secrets/store.js";
+import { pluginSteps, storeFor } from "../plugin.js";
+import { down, rollback } from "../recover.js";
 import { driftOf, plannedServices, type Drift } from "../services/planned.js";
 import { sshHost } from "../sshHost.js";
 import { topologyFor, type Topology } from "../topology.js";
 import type { Deployment, DeployHost } from "../types.js";
 
-import { requireAgent } from "./agent.js";
+import { needsAgent, requireAgent } from "./agent.js";
 import { loadConfig } from "./config.js";
 import { createLog, describeFailure } from "./log.js";
 
@@ -25,6 +26,8 @@ const USAGE = `redkite <command> [environment]
   plan [environment]     Print the derived topology, the pipeline and the nginx
   deploy [environment]   Build, swap, health check, and revert on failure
   verify [environment]   Bring the services up, build, and run each app's checks
+  rollback [environment] Put back any app a run moved and did not finish
+  down [environment]     Stop everything this environment named
 
   --config <path>        Defaults to redkite.config.ts at the root of the project
   --local                Build the images here and ship them to the host
@@ -46,6 +49,7 @@ function measured(host: Host, log?: Log) {
     cache: host.cache,
     pipe: host.pipe.bind(host),
     stop: host.stop.bind(host),
+    final: host.final.bind(host),
 
     sh: async (command, onLine) => {
       const started = Date.now();
@@ -143,6 +147,9 @@ async function dispatch(argv: string[]) {
   const configPath = flag(argv, "--config");
 
   if (command === "plan") return await plan(environment, configPath);
+  if (command === "rollback" || command === "down") {
+    return await recover(command, environment, configPath);
+  }
   if (command === "deploy" || command === "verify") {
     return await run(command, environment, configPath, {
       verbose: argv.includes("--verbose"),
@@ -155,6 +162,49 @@ async function dispatch(argv: string[]) {
   process.exitCode = command ? 1 : 0;
 }
 
+// Neither runs a pipeline. Both exist for the run that was killed rather than
+// asked to stop, so they read what the host is in rather than what a deploy
+// remembered, and both are safe where there is nothing to do
+async function recover(
+  command: "rollback" | "down",
+  environment: string,
+  configPath?: string,
+) {
+  const config = await loadConfig(configPath);
+  const topology = topologyFor(config, environment);
+  const deployHost = environmentOf(config, environment)?.host;
+
+  const say = (message = "") => process.stdout.write(`${message}\n`);
+  if (needsAgent(config, environment)) requireAgent(say);
+
+  const host = await hostFor(deployHost, silent);
+
+  try {
+    if (command === "down") {
+      const { stopped } = await down({ config, environment, host, log: asLog(say) });
+
+      say(stopped.length === 0 ? "Nothing was running" : `Stopped ${stopped.join(", ")}`);
+      return;
+    }
+
+    const { restored } = await rollback({ config, environment, host, log: asLog(say) });
+
+    if (restored.length === 0) {
+      return say(`Nothing to put back in ${topology.environment}`);
+    }
+
+    say(`Put back ${restored.join(", ")}`);
+  } finally {
+    await host.close?.();
+  }
+}
+
+// These print rather than draw: a recovery runs after the view has gone, and
+// often in a job that is already being torn down
+function asLog(say: (message: string) => void): Log {
+  return Object.assign(say, { warn: say, fail: say, done: say, step: () => silent.step("") });
+}
+
 async function plan(environment: string, configPath?: string) {
   const config = await loadConfig(configPath);
   const topology = topologyFor(config, environment);
@@ -203,6 +253,7 @@ async function plan(environment: string, configPath?: string) {
   // run it supports. Nothing declares that: the missing port is what says it
   const serves = Boolean(topology.publicPort);
 
+  sayPlugins(config, say);
   sayPipeline(config, serves, say);
   await sayDrift(config, topology, serves ? "deploy" : "verify", say);
 
@@ -250,6 +301,29 @@ const DRIFT: Record<Drift["reason"], string> = {
   unrecognised: "not created by redkite, will be recreated",
 };
 
+// Everything this deployment opted into. A vault that is missing from here is
+// the reason its refs will not resolve, and that is worth seeing before a run
+function sayPlugins(config: Deployment, say: (message?: string) => void) {
+  const plugins = config.plugins ?? [];
+  if (plugins.length === 0) return;
+
+  say("\nplugins");
+
+  for (const plugin of plugins) {
+    const stores = Object.keys(plugin.stores ?? {});
+    const resolves = stores.length > 0 ? `  resolves ${stores.join(", ")}` : "";
+    const count = plugin.steps?.length ?? 0;
+
+    say(`  ${plugin.name}${gap(plugin.name.length)}${count} ${count === 1 ? "step" : "steps"}${resolves}`);
+  }
+}
+
+const COLUMN = 26;
+
+function gap(width: number) {
+  return " ".repeat(Math.max(2, COLUMN - width));
+}
+
 // What each run will actually do, in the order it will do it. A step is
 // addressed rather than called, so this is the only place a sequence is visible
 function sayPipeline(
@@ -257,9 +331,19 @@ function sayPipeline(
   serves: boolean,
   say: (message?: string) => void,
 ) {
-  const steps = config.steps ?? [];
+  // What the run will walk, which is the plugins' steps and then the
+  // deployment's own. A plan that showed only one of them would be a plan of a
+  // different deploy
+  const steps = [...pluginSteps(config.plugins), ...(config.steps ?? [])];
   const added = new Set(steps.map((step) => step.point));
 
+  // Says which plugin a step came from, because a point on its own does not
+  const from = new Map(
+    (config.plugins ?? []).flatMap((plugin) =>
+      (plugin.steps ?? []).map((step) => [step.point, plugin.name] as const),
+    ),
+  );
+
   const at = (phase: string, slot: string) =>
     steps.filter((step) => {
       const address = addressOf(step.point);
@@ -294,7 +378,12 @@ function sayPipeline(
           say(`  ${phase.padEnd(26)}${who}${what}`);
         }
 
-        for (const step of at(phase, slot)) say(`  ${step.point}`);
+        for (const step of at(phase, slot)) {
+          const owner = from.get(step.point);
+          if (!owner) say(`  ${step.point}`);
+          // A point long enough to fill the column still gets its two spaces
+          else say(`  ${step.point}${gap(step.point.length)}${owner}`);
+        }
       }
     }
   }
@@ -395,7 +484,7 @@ async function hostFor(host: DeployHost | undefined, log: Log, signal?: AbortSig
   const task = log.step(`Opening a connection to ${host.bastion}`);
 
   try {
-    const opened = await sshHost(host.bastion, { signal });
+    const opened = await sshHost(host.bastion, { signal, hostKeys: host.hostKeys });
     task.done();
     return opened;
   } catch (error) {
@@ -406,6 +495,9 @@ async function hostFor(host: DeployHost | undefined, log: Log, signal?: AbortSig
 
 // Only the stores the config actually names. A deployment that keeps its
 // environment somewhere else, or nowhere, needs no credentials to deploy
+// One store per provider a ref names, opened from the plugins the deployment
+// registered. Nothing is opened for a deployment that names no refs, and a ref
+// no plugin answers for is refused rather than reaching for a vault by name
 async function openStores(config: Deployment, log: Log): Promise<SecretStores> {
   // Services name refs too, and a deployment whose only secret is a database
   // password opened no store at all until this counted them
@@ -417,35 +509,30 @@ async function openStores(config: Deployment, log: Log): Promise<SecretStores> {
     ...config.services.flatMap((service) => listRefs(service.secrets)),
   ];
 
-  const providers = new Set(refs.map((ref) => ref.provider));
+  const opened: SecretStores = {};
 
-  if (!providers.has("bitwarden")) return {};
+  for (const provider of new Set(refs.map((ref) => ref.provider))) {
+    const open = storeFor(config.plugins, provider);
 
-  const vault = log.step("Reading the vault");
+    if (!open) {
+      throw new Error(
+        `Nothing resolves ${provider} secrets. Add the plugin that does to ` +
+          "the plugins this deployment registers",
+      );
+    }
 
-  try {
-    const bitwarden = await bitwardenStore({
-      detail: vault.detail,
-      clientId: required("BW_CLIENT_ID"),
-      clientSecret: required("BW_CLIENT_SECRET"),
-      password: required("BW_PASSWORD"),
-    });
+    const task = log.step(`Reading the ${provider} vault`);
 
-    vault.done();
-    return { bitwarden };
-  } catch (error) {
-    vault.fail("Could not read the vault");
-    throw error;
+    try {
+      opened[provider] = await open({ detail: task.detail });
+      task.done();
+    } catch (error) {
+      task.fail(`Could not read the ${provider} vault`);
+      throw error;
+    }
   }
-}
-
-function required(name: string) {
-  const value = process.env[name];
-  if (value) return value;
 
-  throw new Error(
-    `${name} is not set, and this deployment reads its environment from Bitwarden`,
-  );
+  return opened;
 }
 
 type RunOptions = { verbose: boolean; full: boolean; local: boolean };
@@ -465,7 +552,9 @@ async function run(
   // The host clones the repositories over this, so it has to exist before the
   // connection that forwards it is opened. Before the view too: ssh-add asks
   // for a passphrase on the terminal, and by then the view owns it
-  requireAgent((message) => process.stderr.write(`${message}\n`));
+  if (needsAgent(config, environment)) {
+    requireAgent((message) => process.stderr.write(`${message}\n`));
+  }
 
   // Whatever is in flight is killed, the pipeline unwinds through its own
   // failure path, and the finally below removes the scratch directory

package/src/config.ts

@@ -1,4 +1,5 @@
 import { assertSteps } from "./pipeline.js";
+import { pluginSteps } from "./plugin.js";
 import type { Deployment, Environment } from "./types.js";
 
 // The one selected, from the one place worth looking. A deployment that carries
@@ -19,7 +20,10 @@ export function defineDeployment<const T extends Deployment & { environments?: n
   assertOneSource(config);
   assertRoutesResolvable(config);
   assertDirsRelative(config);
-  assertSteps(config.steps ?? []);
+  assertUniquePlugins(config);
+  // Together, because a plugin's step and the deployment's own share the same
+  // points and two of them at one point is the collision worth catching
+  assertSteps([...pluginSteps(config.plugins), ...(config.steps ?? [])]);
   return config;
 }
 
@@ -29,6 +33,20 @@ export function defineEnvironment<const T extends Environment>(environment: T):
   return environment;
 }
 
+// Registering one twice is either a mistake or two configurations of the same
+// thing, and neither is something to pick a winner for
+function assertUniquePlugins(config: Deployment) {
+  const seen = new Set<string>();
+
+  for (const plugin of config.plugins ?? []) {
+    if (seen.has(plugin.name)) {
+      throw new Error(`The plugin ${plugin.name} is registered twice`);
+    }
+
+    seen.add(plugin.name);
+  }
+}
+
 // Cloned or already here, and the two are built differently enough that
 // guessing between them is worse than being told
 function assertOneSource(config: Deployment) {

package/src/deploy.ts

@@ -2,8 +2,9 @@ import { build, type BuildContext, type BuildResult } from "./build.js";
 import { assertCheckable, runChecks } from "./checks.js";
 import { environmentOf } from "./config.js";
 import { Docker } from "./docker.js";
+import { envFileFor } from "./environment.js";
 import { healthcheck, type HealthDeps } from "./health.js";
-import type { Host } from "./host.js";
+import { finalHost, type Host } from "./host.js";
 import { localHost } from "./localHost.js";
 import { silent, type Log } from "./log.js";
 import {
@@ -21,6 +22,7 @@ import {
   type Run,
   type Start,
 } from "./pipeline.js";
+import { pluginSteps } from "./plugin.js";
 import { readEnv, readRef, type SecretStores } from "./secrets/refs.js";
 import { ensureService } from "./services/ensure.js";
 import { plannedServices } from "./services/planned.js";
@@ -78,8 +80,11 @@ async function start(run: Run, options: DeployOptions): Promise<Finished> {
     run,
   };
 
-  const steps = merge(supplied(options), options.config.steps ?? []);
-  return await runPipeline(run, steps, setting, options.signal);
+  // A plugin's steps lead, so a snapshot listed as a plugin runs above the
+  // migration the deployment writes after it
+  const added = [...pluginSteps(options.config.plugins), ...(options.config.steps ?? [])];
+
+  return await runPipeline(run, merge(supplied(options), added), setting, options.signal);
 }
 
 // Every step redkite supplies, in the order the phases name. A run walks the
@@ -171,33 +176,66 @@ async function release(
     context.config.apps.map((app) => [app.name, app.environment ?? {}]),
   );
 
-  task.detail("retiring the running containers");
-  await Promise.all(apps.map((app) => retire(docker, topology, app)));
-
-  task.detail("creating the new ones");
-
-  await Promise.all(
-    apps.map((app) =>
-      create(docker, topology, app, environments.get(app.name)),
+  // Resolved now and handed over when the container is made. Nothing from the
+  // vault is in the image, so this is the only way the running process sees it
+  const files = new Map(
+    await Promise.all(
+      context.config.apps.map(
+        async (app) => [app.name, await envFileFor(app, context)] as const,
+      ),
     ),
   );
 
-  task.detail("starting them");
-  await Promise.all(apps.map((app) => docker.container.start(app.container)));
-
-  // One unhealthy app reverts all of them. A half-swapped deployment is the
-  // one state nothing downstream knows how to reason about
-  if (!(await checkAll(context, health))) {
-    context.log.fail("Health checks failed, reverting");
-    await Promise.all(apps.map((app) => revert(docker, topology, app)));
-
-    return {
-      ...input,
-      ok: false,
-      released: [],
-      reverted: apps.map((app) => app.container),
-      checked: [],
-    };
+  // What has actually been moved, rather than what was going to be. A stop
+  // lands between two docker commands, and only this says which side of it
+  const moved: AppTopology[] = [];
+
+  try {
+    task.detail("retiring the running containers");
+
+    await Promise.all(
+      apps.map(async (app) => {
+        await retire(docker, topology, app);
+        moved.push(app);
+      }),
+    );
+
+    task.detail("creating the new ones");
+
+    await Promise.all(
+      apps.map((app) =>
+        create(docker, topology, app, environments.get(app.name), files.get(app.name)),
+      ),
+    );
+
+    task.detail("starting them");
+    await Promise.all(apps.map((app) => docker.container.start(app.container)));
+
+    // Inside, because the swap is not over until this says so. A stop lands
+    // here more often than anywhere else: it is the longest part, and by now
+    // every address has already moved
+    if (!(await checkAll(context, health))) {
+      context.log.fail("Health checks failed, reverting");
+      await Promise.all(apps.map((app) => revert(docker, topology, app)));
+
+      return {
+        ...input,
+        ok: false,
+        released: [],
+        reverted: apps.map((app) => app.container),
+        checked: [],
+      };
+    }
+  } catch (error) {
+    // Whatever ended this, the addresses have moved and something has to put
+    // them back. A stop is the usual one, so the revert runs on a host that
+    // has been told to stop: it is the abort that made this necessary
+    if (moved.length > 0) {
+      context.log.fail(`Putting ${moved.length} back where they were`);
+      await putBack(context, moved);
+    }
+
+    throw error;
   }
 
   return {
@@ -209,6 +247,23 @@ async function release(
   };
 }
 
+// Through a host with the stop lifted, because the commands that undo a swap
+// cannot be refused by the same signal that interrupted it. One app failing to
+// go back must not stop the others, so each is settled on its own
+async function putBack(context: Context, moved: AppTopology[]) {
+  const docker = new Docker(finalHost(context.host));
+
+  const put = moved.map(async (app) => {
+    try {
+      await revert(docker, context.topology, app);
+    } catch (error) {
+      context.log.fail(`${app.container} could not be put back: ${String(error)}`);
+    }
+  });
+
+  await Promise.all(put);
+}
+
 // A build leaves its image on the host rather than sending one, so without this
 // every run adds a runtime image and a builder to a disk nobody is watching
 async function finish(input: Released, context: Context): Promise<Finished> {
@@ -342,6 +397,7 @@ async function create(
   topology: Topology,
   app: AppTopology,
   environment: Record<string, string> = {},
+  envFile?: string,
 ) {
   const builder = docker.container
     .builder()
@@ -357,19 +413,35 @@ async function create(
     builder.extraHost(host, ip);
   }
 
+  if (envFile) builder.envFile(envFile);
+
   for (const volume of app.volumes) builder.volume(volume.volume, volume.mountPath);
   for (const [name, value] of Object.entries(environment)) builder.env(name, value);
 
   await builder.create();
 }
 
-// Put the retired container back on the live address and its original name
-async function revert(docker: Docker, topology: Topology, app: AppTopology) {
+// Put the retired container back on the live address and its original name.
+// Answers with whether there was one: a first deploy has nothing behind it, and
+// the container that just failed is simply left where it was renamed to
+export async function revert(docker: Docker, topology: Topology, app: AppTopology) {
   await docker.container.stop(app.container);
+
+  // The slot may still hold what an earlier interrupted run put there, and a
+  // rename refuses rather than clobbers. Retiring clears its own slot the same
+  // way, and a revert that skipped this would leave the new container live
+  await docker.container.stop(app.failed);
+  await docker.container.remove(app.failed);
+
   await docker.container.rename(app.container, app.failed);
+
+  if (!(await docker.container.exists(app.retired))) return false;
+
   await docker.network.reconnect(topology.network, app.retired, app.currentAddress);
   await docker.container.rename(app.retired, app.container);
   await docker.container.start(app.container);
+
+  return true;
 }
 
 async function cleanup(docker: Docker, app: AppTopology) {

package/src/dockerfile.ts

@@ -82,22 +82,18 @@ function builderStage(spec: BuildSpec, context: DockerfileContext) {
   // Only now, so the steps below read the app's own manifest and write beside it
   if (context.dir) lines.push(`WORKDIR ${workdir}`);
 
-  lines.push(
-    `RUN --mount=type=secret,id=${context.envSecret} cp /run/secrets/${context.envSecret} ${workdir}/.env`,
-    `ENV SENTRY_RELEASE=${context.release}`,
-  );
+  lines.push(`ENV SENTRY_RELEASE=${context.release}`);
+
+  // Mounted for the length of each step rather than copied into the tree. A
+  // file that is copied is in that layer for good, and no later instruction
+  // can take it back out: rm leaves it readable in the layer underneath
+  const env = `--mount=type=secret,id=${context.envSecret},target=${workdir}/.env `;
 
   // Unquoted: the shell form hands everything after RUN to sh -c, so a step is
   // written exactly as it would be typed. Quoting it made the whole command one
   // word, and the shell went looking for a program by that name
   for (const step of spec.steps) {
-    lines.push(`RUN ${mounts}${step}`);
-  }
-
-  if (spec.sourcemaps) {
-    // The built .env ships inside the image, and an upload token is build-time
-    const token = spec.sourcemaps.stripFromImage;
-    lines.push(`RUN sed -i '/^${token}=/d' ${rootedAt(spec.output, context.dir)}/.env`);
+    lines.push(`RUN ${env}${mounts}${step}`);
   }
 
   return lines;

package/src/environment.ts

@@ -0,0 +1,24 @@
+import type { Context } from "./pipeline.js";
+import { readEnv } from "./secrets/refs.js";
+import type { AppSpec } from "./types.js";
+
+// What the vault resolved to, handed over as a file rather than baked into an
+// image. Nothing from a vault is in a layer, so this is the only way anything
+// running the app's image sees its environment: the container itself, and the
+// migrations and checks that run in the builder.
+
+// Written to the host and handed over as a file, so no value appears in an
+// argument list. Docker reads it when a container is created, and again for
+// each `run --env-file`, so the file goes with the rest of the deploy's scratch
+export async function envFileFor(app: AppSpec, context: Context) {
+  if (!app.secrets) return undefined;
+
+  const contents = await readEnv(app.secrets, context.secrets);
+  return await context.host.write(`apps/${app.name}/env`, contents);
+}
+
+// The flag, or nothing at all for an app that names no secrets
+export async function envFlags(app: AppSpec, context: Context) {
+  const path = await envFileFor(app, context);
+  return path ? [`--env-file ${path}`] : [];
+}

package/src/host.ts

@@ -25,9 +25,20 @@ export type Host = {
   // Signals everything this host started and answers with how many are still
   // running. Zero is the only answer that means nothing was left behind
   stop(signal: "TERM" | "KILL"): Promise<number>;
+  // A command that runs even once a stop has been asked for. Putting a swap
+  // back is the reason there is one: the abort is what made the work necessary,
+  // so it cannot also be what refuses to do it
+  final(command: string): Promise<Result>;
   close?(): Promise<void>;
 };
 
+// The same host with the stop lifted. Handed to a Docker, it gives every
+// command below it the same exemption, which is how a revert reaches the
+// containers an abort left half moved
+export function finalHost(host: Host): Host {
+  return { ...host, sh: async (command) => await host.final(command) };
+}
+
 // A streamed command keeps this many lines, so a build that prints tens of
 // megabytes still fits in the error a failure reports
 const TAIL = 200;

package/src/index.ts

@@ -2,6 +2,7 @@ export * from "./build.js";
 export * from "./config.js";
 export * from "./deploy.js";
 export * from "./docker.js";
+export * from "./environment.js";
 export * from "./dockerfile.js";
 export * from "./health.js";
 export * from "./host.js";
@@ -10,8 +11,10 @@ export * from "./localHost.js";
 export * from "./log.js";
 export * from "./nginx.js";
 export * from "./pipeline.js";
+export * from "./plugin.js";
+export * from "./plugins/index.js";
+export * from "./recover.js";
 export * from "./presets/index.js";
-export * from "./secrets/bitwarden.js";
 export * from "./secrets/refs.js";
 export * from "./secrets/store.js";
 export * from "./shell.js";

package/src/localHost.ts

@@ -43,6 +43,9 @@ export async function localHost(options: LocalOptions = {}): Promise<Host> {
     // is a process group leader, so one signal reaches the whole tree
     stop: async (name) => signalEverything(name === "KILL" ? "SIGKILL" : "SIGTERM"),
 
+    // Without the signal, so a stop cannot refuse the work it created
+    final: async (command) => await spawnCollect("sh", ["-c", command]),
+
     close: async () => {
       await rm(directory, { recursive: true, force: true });
     },

package/src/plugin.ts

@@ -0,0 +1,50 @@
+import { assertSteps, type AnyStep } from "./pipeline.js";
+import type { SecretStore } from "./secrets/refs.js";
+
+// What a deployment can be handed that redkite did not write. A plugin adds
+// work to the run, or teaches it to resolve a kind of secret, and nothing it
+// brings happens until a deployment lists it. Redkite's own vault is one of
+// these rather than something wired in behind them.
+
+export type OpenContext = {
+  // What the step row says while the store is being opened. Unlocking a vault
+  // is the slowest thing a deploy does before it starts building
+  detail: (message: string) => void;
+};
+
+// Opened once, and only when a ref names the provider it answers for
+export type OpenStore = (context: OpenContext) => Promise<SecretStore>;
+
+export type Plugin = {
+  // Says which one when a deployment registers the same plugin twice, and what
+  // a failure to open a store is reported against
+  name: string;
+  // Run in the order the deployment lists the plugins, above its own steps
+  steps?: AnyStep[];
+  // Provider tag to how that store is opened
+  stores?: Record<string, OpenStore>;
+};
+
+// Identity, but the points a plugin claims are checked where it is written
+// rather than where it is registered, so a typo is the plugin's own failure
+export function definePlugin<const T extends Plugin>(plugin: T): T {
+  assertSteps(plugin.steps ?? []);
+  return plugin;
+}
+
+export function pluginSteps(plugins: Plugin[] = []): AnyStep[] {
+  return plugins.flatMap((plugin) => plugin.steps ?? []);
+}
+
+// The one that answers for this provider. Two plugins claiming it would be two
+// vaults for one tag, which is a config nobody can read the intent of
+export function storeFor(plugins: Plugin[] = [], provider: string) {
+  const claiming = plugins.filter((plugin) => plugin.stores?.[provider]);
+
+  if (claiming.length > 1) {
+    const names = claiming.map((plugin) => plugin.name).join(" and ");
+    throw new Error(`${names} both resolve ${provider} secrets, and only one can`);
+  }
+
+  return claiming[0]?.stores?.[provider];
+}