redkite 0.1.6

package/src/shell.ts

@@ -0,0 +1,95 @@
+import { spawn, type ChildProcess } from "node:child_process";
+
+import { lineReader, tail, type OnLine, type Result } from "./host.js";
+
+// One child process, its output collected and streamed. Both hosts ran their
+// own copy of this, and shipping an image needs a third caller.
+
+export type SpawnOptions = {
+  stdin?: string;
+  onLine?: OnLine;
+  // Only stops a command being started. What is already running is stopped by
+  // signalling it, which is the host's to do: over ssh the process to signal is
+  // on the other machine, and killing the client here would only lose the reach
+  signal?: AbortSignal;
+};
+
+// Every child still running. A detached child outlives this process, so a stop
+// that does not go through here leaves a build running
+const live = new Set<ChildProcess>();
+
+// Signals every group and answers with how many are still there. The count is
+// the whole point: nothing may exit while it is above zero
+export function signalEverything(name: NodeJS.Signals) {
+  for (const child of live) {
+    try {
+      if (child.pid) process.kill(-child.pid, name);
+    } catch {
+      // Already gone, which is the outcome being asked for
+    }
+  }
+
+  return live.size;
+}
+
+export function stillRunning() {
+  return live.size;
+}
+
+export function spawnCollect(
+  command: string,
+  args: string[],
+  options: SpawnOptions = {},
+): Promise<Result> {
+  return new Promise((resolve, reject) => {
+    if (options.signal?.aborted) return reject(new Error("Stopped"));
+
+    // Its own process group, so one signal reaches everything it started. A
+    // shell that ran docker dies on its own otherwise, and docker keeps going
+    const child = spawn(command, args, {
+      stdio: [options.stdin === undefined ? "ignore" : "pipe", "pipe", "pipe"],
+      detached: true,
+    });
+
+    live.add(child);
+
+    const out: string[] = [];
+    const err: string[] = [];
+
+    const collect = (into: string[]) =>
+      lineReader((line) => {
+        into.push(line);
+        options.onLine?.(line);
+      });
+
+    const stdout = collect(out);
+    const stderr = collect(err);
+
+    // Only a streamed command is truncated. The host snapshot is one command
+    // listing every container and image, and a busy host overruns any tail
+    const keep = (lines: string[]) => (options.onLine ? tail(lines) : lines.join("\n"));
+
+    child.stdout?.on("data", (chunk: Buffer) => stdout.push(chunk.toString()));
+    child.stderr?.on("data", (chunk: Buffer) => stderr.push(chunk.toString()));
+
+    child.on("error", reject);
+
+    // The only thing that settles this. A signalled command answers here once
+    // it has actually gone, which is what lets a caller wait for exactly that
+    child.on("close", (code) => {
+      live.delete(child);
+
+      stdout.flush();
+      stderr.flush();
+      resolve({ code: code ?? 1, stdout: keep(out), stderr: keep(err) });
+    });
+
+    if (options.stdin !== undefined) child.stdin?.end(options.stdin);
+  });
+}
+
+// Single quotes are what a shell strips, so a value containing double quotes
+// survives being handed to one
+export function quote(value: string) {
+  return `'${value.replaceAll("'", `'\\''`)}'`;
+}

package/src/source.ts

@@ -0,0 +1,171 @@
+import type { Host } from "./host.js";
+
+// Getting the source onto the machine that builds it. A repository the host
+// clones for itself over the forwarded agent, so no tree crosses the wire: a
+// commit that is already mirrored costs a fetch of whatever is new since the
+// last deploy. A directory is already there and is built as it stands.
+
+export type Source = {
+  // What the image is tagged by: the commit for a repository, and the content
+  // of the working tree for a directory
+  release: string;
+  // What the build reads, on the machine that builds it
+  tree: string;
+};
+
+export type SourceRequest = {
+  // Names the mirror and the checkout. Keyed per app and environment rather
+  // than per repository, so two deploys never fetch into one directory at once
+  name: string;
+  // Exactly one of these, which is what the config is checked for
+  repo?: string;
+  path?: string;
+  // What of a path goes into the build. Absent leaves it to the work tree's
+  // own .gitignore, which is the only other thing that can say
+  include?: string[];
+  branch: string;
+  submodules: boolean;
+  detail?: (message: string) => void;
+};
+
+// accept-new rather than no: a host key that changes is still worth refusing
+const GIT_SSH = "ssh -o StrictHostKeyChecking=accept-new";
+
+export async function prepareSource(
+  host: Host,
+  request: SourceRequest,
+): Promise<Source> {
+  if (request.path) return await localSource(host, request.path, request);
+  if (request.repo) return await clonedSource(host, request.repo, request);
+
+  throw new Error(`${request.name} names neither a repo nor a path to build from`);
+}
+
+// Built where it sits. Nothing is cloned, checked out or cleaned: what the
+// build reads is the tree as the person running this left it
+async function localSource(
+  host: Host,
+  path: string,
+  request: SourceRequest,
+): Promise<Source> {
+  const detail = request.detail ?? (() => {});
+  detail(`reading ${path}`);
+
+  await assertKnowable(host, path, request.include);
+
+  const release = await treeOf(host, path, request.include);
+  detail(`built from ${release.slice(0, 7)}`);
+
+  return { release, tree: path };
+}
+
+// Something has to say what belongs in the build. A work tree's .gitignore
+// does; anywhere else the deployment has to say it, or the release would cover
+// whatever happened to be lying in the directory
+async function assertKnowable(host: Host, path: string, include?: string[]) {
+  if (include) return;
+
+  const inside = await host.sh(`git -C '${path}' rev-parse --is-inside-work-tree`);
+  if (inside.code === 0 && inside.stdout.trim() === "true") return;
+
+  throw new Error(
+    `${path} is not a git work tree, so nothing there says what belongs in the ` +
+      'build. Say it: include: ["src", "package.json"] on the app names what ' +
+      "ships, and git init there would let .gitignore name it instead",
+  );
+}
+
+// git's own content addressing, over a scratch repository so nothing about the
+// source changes and no object lands in it. The tree covers what is committed,
+// what is modified and what is untracked, so an edit that is never committed is
+// a new release. Without an include it honours .gitignore, which is the same
+// set the build reads
+async function treeOf(host: Host, path: string, include?: string[]) {
+  // Everything the work tree does not ignore, or exactly what was named. The
+  // -- is only for the second: it would make -A a path rather than a flag
+  const added = include ? `-- ${include.map((item) => `'${item}'`).join(" ")}` : "-A";
+
+  const written = await host.sh(
+    [
+      "scratch=$(mktemp -d)",
+      'git init -q --bare "$scratch/git"',
+      `export GIT_DIR="$scratch/git" GIT_WORK_TREE='${path}' GIT_INDEX_FILE="$scratch/index"`,
+      `git add ${added}`,
+      "git write-tree",
+      'rm -rf "$scratch"',
+    ].join("\n"),
+  );
+
+  const release = written.stdout.trim().split("\n").at(-1) ?? "";
+  if (/^[0-9a-f]{40}$/.test(release)) return release;
+
+  throw new Error(`Could not read the state of ${path}: ${written.stderr || written.stdout}`);
+}
+
+async function clonedSource(
+  host: Host,
+  repo: string,
+  request: SourceRequest,
+): Promise<Source> {
+  const detail = request.detail ?? (() => {});
+  const mirror = `${host.cache}/mirrors/${request.name}.git`;
+  const path = `${host.cache}/source/${request.name}`;
+
+  detail(`fetching ${request.branch}`);
+  await run(host, `updating the mirror of ${repo}`, [
+    `if [ -d '${mirror}' ]; then`,
+    `  git -C '${mirror}' remote set-url origin '${repo}'`,
+    `  git -C '${mirror}' remote update --prune`,
+    "else",
+    `  mkdir -p '${host.cache}/mirrors'`,
+    `  git clone --mirror '${repo}' '${mirror}'`,
+    "fi",
+  ]);
+
+  const resolved = await run(host, `resolving ${request.branch}`, [
+    `git -C '${mirror}' rev-parse 'refs/heads/${request.branch}'`,
+  ]);
+
+  const release = resolved.stdout.trim();
+  if (!/^[0-9a-f]{40}$/.test(release)) {
+    throw new Error(`${repo} has no branch ${request.branch}`);
+  }
+
+  detail(`checking out ${release.slice(0, 7)}`);
+  await run(host, `checking out ${release.slice(0, 7)}`, [
+    // The checkout shares the mirror's object store, so it costs the working
+    // tree and nothing else. Reused between deploys, hence the reset and clean
+    `if [ ! -d '${path}/.git' ]; then`,
+    `  rm -rf '${path}'`,
+    `  mkdir -p '${host.cache}/source'`,
+    `  git clone --shared --no-checkout '${mirror}' '${path}'`,
+    "fi",
+    `git -C '${path}' fetch --prune origin`,
+    `git -C '${path}' checkout --detach --force '${release}'`,
+    // Leaves the submodules alone: they are tracked, and clean only removes
+    // what is not
+    `git -C '${path}' clean -ffdx`,
+  ]);
+
+  if (request.submodules) {
+    // --remote follows the branch named in .gitmodules rather than the commit
+    // the parent recorded, which is what the pipeline has always done
+    detail("updating submodules");
+    await run(host, "updating submodules", [
+      `git -C '${path}' submodule sync --recursive`,
+      `git -C '${path}' submodule update --init --remote --recursive`,
+    ]);
+  }
+
+  return { release, tree: path };
+}
+
+async function run(host: Host, what: string, script: string[]) {
+  const result = await host.sh(
+    [`export GIT_SSH_COMMAND='${GIT_SSH}'`, "set -e", ...script].join("\n"),
+  );
+
+  if (result.code === 0) return result;
+
+  throw new Error(`Failed ${what}: ${result.stderr || result.stdout}`);
+}

package/src/sshHost.ts

@@ -0,0 +1,145 @@
+import { randomUUID } from "node:crypto";
+import { dirname } from "node:path";
+
+import type { Host, OnLine, Result } from "./host.js";
+import { quote, spawnCollect } from "./shell.js";
+
+// The deploy host, reached over ssh. The agent is forwarded rather than a key,
+// there is nothing to forward, and the agent travels with the connection so the
+// host can clone the repositories itself.
+
+// Injected so the argv this builds is asserted on rather than trusted
+export type Ssh = (
+  args: string[],
+  options?: { stdin?: string; onLine?: OnLine },
+) => Promise<Result>;
+
+// The local shell a piped stream is fed through, injected for the same reason
+// run is: what this builds is asserted on rather than trusted
+export type Shell = (command: string) => Promise<Result>;
+
+export type SshOptions = {
+  run?: Ssh;
+  shell?: Shell;
+  directory?: string;
+  cache?: string;
+  // Aborting kills the ssh client. The command it was carrying keeps running on
+  // the other machine, which is what stop is for
+  signal?: AbortSignal;
+};
+
+// One TCP connection and one authentication for the whole deploy. Without it
+// every command pays a handshake, which is most of what a command costs
+const MULTIPLEX = [
+  "-o",
+  "ControlMaster=auto",
+  "-o",
+  "ControlPersist=60s",
+  "-o",
+  "StrictHostKeyChecking=no",
+  // The host clones from GitHub on our behalf rather than us shipping a tree
+  "-A",
+];
+
+export async function sshHost(bastion: string, options: SshOptions = {}): Promise<Host> {
+  const control = `/tmp/redkite-${randomUUID().slice(0, 8)}.control`;
+  const directory = options.directory ?? `/tmp/redkite-${randomUUID().slice(0, 8)}`;
+  const signal = options.signal;
+
+  const run =
+    options.run ?? ((args, extra) => spawnCollect("ssh", args, { ...extra, signal }));
+
+  // Cleanup has to survive the abort that made it necessary, so it is the one
+  // thing here that is not killed along with everything else
+  const final = options.run ?? ((args, extra) => spawnCollect("ssh", args, extra));
+
+  const shell =
+    options.shell ??
+    ((command: string) => spawnCollect("sh", ["-c", command], { signal }));
+
+  const argv = (command: string) => [
+    ...MULTIPLEX,
+    "-o",
+    `ControlPath=${control}`,
+    bastion,
+    command,
+  ];
+
+  const ssh = (command: string, extra?: { stdin?: string; onLine?: OnLine }) =>
+    run(argv(command), extra);
+
+  // Each command records the process group job control gave it, because killing
+  // the ssh client here leaves what it was carrying running there
+  let issued = 0;
+
+  // One round trip for all of it, including the home directory the cache hangs
+  // off. A literal ~ would only survive as long as every caller used a shell
+  const opened = await ssh(
+    `mkdir -p -m 700 '${directory}' && mkdir -p "$HOME/.cache/redkite" && printf %s "$HOME"`,
+  );
+
+  if (opened.code !== 0) {
+    throw new Error(`Could not reach ${bastion}: ${opened.stderr || opened.stdout}`);
+  }
+
+  const cache = options.cache ?? `${opened.stdout.trim()}/.cache/redkite`;
+
+  return {
+    directory,
+    cache,
+
+    sh: async (command, onLine) => {
+      issued += 1;
+      return await ssh(supervised(command, `${directory}/run.${issued}.pid`), { onLine });
+    },
+
+    write: async (name, contents) => {
+      const path = `${directory}/${name}`;
+      const result = await ssh(`mkdir -p '${dirname(path)}' && cat > '${path}'`, {
+        stdin: contents,
+      });
+
+      if (result.code !== 0) {
+        throw new Error(`Could not write ${name}: ${result.stderr}`);
+      }
+
+      return path;
+    },
+
+    // Through the connection that is already open, so an image is streamed
+    // rather than written to a disk at each end and copied between them
+    pipe: async (local, remote) =>
+      await shell(`${local} | ssh ${argv(remote).map(quote).join(" ")}`),
+
+    // Signalled where they are running, not here. Killing the client would
+    // leave the build going with nothing left able to reach it
+    stop: async (name) => {
+      const result = await final(argv(sweep(directory, name)));
+      return Number(result.stdout.trim()) || 0;
+    },
+
+    close: async () => {
+      await final(argv(`rm -rf '${directory}'`));
+      await final(["-o", `ControlPath=${control}`, "-O", "exit", bastion]);
+    },
+  };
+}
+
+// Job control puts a background job in a process group of its own, which is the
+// only handle a second connection has on what the first one started. Nothing is
+// written to stdout by this: an announcement would corrupt the host snapshot
+function supervised(command: string, pidfile: string) {
+  return `set -m; { ${command}; } & __rk=$!; printf %s "$__rk" > '${pidfile}'; wait "$__rk"`;
+}
+
+// Signals every group this deploy started there, then counts the ones still
+// answering. The count is what the caller waits on: a build that ignores the
+// first signal keeps it above zero until a harder one is sent
+function sweep(directory: string, name: string) {
+  return (
+    `left=0; for f in '${directory}'/*.pid; do [ -f "$f" ] || continue; ` +
+    `p=$(cat "$f" 2>/dev/null) || continue; ` +
+    `kill -${name} -"$p" 2>/dev/null; ` +
+    `kill -0 -"$p" 2>/dev/null && left=$((left+1)); done; printf %s "$left"`
+  );
+}

package/src/steps.ts

@@ -0,0 +1,101 @@
+import { environmentOf } from "./config.js";
+import type { Built, Plan, Step } from "./pipeline.js";
+import type { Topology } from "./topology.js";
+import type { AppSpec, StepNetwork } from "./types.js";
+
+// The flags that attach one. The deployment network carries the aliases too,
+// because a service answers to its alias rather than its container name, and a
+// step reaching postgres:5432 is the whole reason to be on that network
+export function attachment(network: StepNetwork, topology: Topology): string[] {
+  if (network === "host") return ["--network host"];
+  if (network === "none") return ["--network none"];
+  if (network !== "deployment") return [`--network ${network.named}`];
+
+  return [
+    // No address is asked for. Docker allocates from the bottom of the subnet
+    // and every derived address starts at .20, so an ephemeral container never
+    // takes one an app is about to be created on
+    `--network ${topology.network}`,
+    ...Object.entries(topology.extraHosts).map(([name, ip]) => `--add-host ${name}:${ip}`),
+  ];
+}
+
+type MigrateOptions = {
+  // The app whose builder image the command runs in. Every app keeps one, so
+  // there is nothing else a config has to set for this to work
+  app: string;
+  command: string;
+  // Defaults to the deploy host's own stack, which is the machine that can
+  // already reach whatever the app's environment file points at. A database
+  // this deployment runs as a service is on "deployment" instead
+  network?: StepNetwork;
+  // Written when the step reached its database through a forward. It runs on
+  // the deploy host now, which is the machine that bastion named, so nothing
+  // is forwarded and the deploy refuses a bastion that is anything else
+  tunnel?: { bastion: string; from: string; alias?: string; port?: number };
+};
+
+// An ordinary step at an ordinary point. Hung before the swap, it runs while
+// the old containers still serve, and it throws, so nothing retires
+export function migrate(options: MigrateOptions): Step<`swap:before:${string}`> {
+  const command = options.command.split(" ");
+
+  return {
+    point: `swap:before:migrate-${options.app}`,
+    check: (plan) => assertReachable(plan, options),
+
+    run: async (input, context) => {
+      const image = builderOf(input, options.app);
+      context.task.detail(`${options.app}: ${options.command}`);
+
+      await context.docker.runOrThrow(
+        [
+          "run --rm",
+          ...attachment(options.network ?? "host", context.topology),
+          "--workdir /app",
+          image,
+          ...command,
+        ].join(" "),
+        `${options.app} failed to migrate`,
+      );
+
+      return input;
+    },
+  };
+}
+
+// Checked before the run starts, so a config naming an app that never kept a
+// builder fails without having touched the host
+function assertReachable(plan: Plan, options: MigrateOptions) {
+  const app = plan.config.apps.find((item) => item.name === options.app);
+  if (!app) throw new Error(`${options.app} names no app in this deployment`);
+
+  const bastion = environmentOf(plan.config, plan.environment)?.host?.bastion;
+  const tunnel = options.tunnel;
+  if (!tunnel || tunnel.bastion === bastion) return;
+
+  throw new Error(
+    `${options.app} tunnels its migration through ${tunnel.bastion}, ` +
+      `but ${plan.environment} deploys to ${bastion ?? "this machine"}. ` +
+      "The step runs on the deploy host, so they have to be the same",
+  );
+}
+
+// A deployment may replace the build step, and a command hung on the pipeline
+// has to run in an image something actually produced
+export function builderOf(input: Built, name: string) {
+  const app = input.apps.find((item) => item.name === name);
+  if (app?.builderTag) return app.builderTag;
+
+  throw new Error(`${name} has no builder image, so nothing can be run in it`);
+}
+
+type SentryOptions = { stripFromImage: string };
+
+export function sentry(options: SentryOptions) {
+  return { provider: "sentry" as const, stripFromImage: options.stripFromImage };
+}
+
+export function routeOf(app: AppSpec) {
+  return app.route;
+}