redkite 0.1.6

package/src/docker.ts

@@ -0,0 +1,433 @@
+import type { Host, OnLine } from "./host.js";
+
+// The docker CLI on the deploy host. Every guard here reads a single snapshot
+// rather than inspecting one object at a time, because a round trip to another
+// machine costs more than the command it carries.
+
+export type BuildInvocation = {
+  // A directory on the host, which is where the checkout already is
+  context: string;
+  // Rendered per app and kept outside the context, so a repository's own
+  // Dockerfile and .dockerignore are neither read nor overwritten
+  dockerfile: string;
+  // The moving name the container refers to, and the versioned one the next
+  // deploy recognises
+  tags: string[];
+  // Secret id to a path on the host holding its contents
+  secrets?: Record<string, string>;
+  // Stops at the builder stage, for the image a step before the swap runs in
+  target?: string;
+};
+
+const RUNNING = new Set(["running", "restarting"]);
+
+// What a container was created from, recorded on the container itself. Docker
+// cannot change a label without recreating, which is exactly when it changes
+export const SPEC_LABEL = "redkite.spec";
+
+// One command in place of an inspect per object. Every guard in this file used
+// to be its own round trip, and a round trip here is a container exec
+const SNAPSHOT = [
+  `ps -a --format '{{.Names}}\t{{.State}}\t{{.Label "${SPEC_LABEL}"}}'`,
+  "docker image ls --format '{{.Repository}}:{{.Tag}}'",
+  "docker network ls --format '{{.Name}}'",
+].join("; echo --- ; ");
+
+type ContainerState = { state: string; spec?: string };
+
+type HostState = {
+  containers: Map<string, ContainerState>;
+  images: Set<string>;
+  networks: Set<string>;
+};
+
+export class Docker {
+  readonly network: DockerNetwork;
+  readonly image: DockerImage;
+  readonly container: DockerContainer;
+
+  // The promise, not the state. Two builds and the infrastructure step all ask
+  // for this at once, and caching the result alone lets every one of them miss
+  private state?: Promise<HostState>;
+
+  constructor(private readonly host: Host) {
+    this.network = new DockerNetwork(this);
+    this.image = new DockerImage(this);
+    this.container = new DockerContainer(this);
+  }
+
+  // Read once, then kept current by the mutations below. A deploy owns the
+  // host for its duration, so nothing else is moving underneath it
+  async snapshot() {
+    this.state ??= this.read();
+    return await this.state;
+  }
+
+  private async read(): Promise<HostState> {
+    const { stdout } = await this.run(SNAPSHOT);
+    const [containers = "", images = "", networks = ""] = stdout.split("---");
+
+    return {
+      containers: new Map(
+        lines(containers).map((line) => {
+          const [name = "", state = "unknown", spec = ""] = line.split("\t");
+          return [name, { state, spec: spec || undefined }];
+        }),
+      ),
+      // Both spellings, so a lookup by bare name and one by name:tag both hit
+      images: new Set(
+        lines(images).flatMap((line) => [line, line.replace(/:latest$/, "")]),
+      ),
+      networks: new Set(lines(networks)),
+    };
+  }
+
+  async run(command: string, onLine?: OnLine) {
+    return await this.host.sh(`docker ${command}`, onLine);
+  }
+
+  // Called by every mutation, so the snapshot never goes stale
+  async track(change: (state: HostState) => void) {
+    change(await this.snapshot());
+  }
+
+  async runOrThrow(command: string, message: string, onLine?: OnLine) {
+    const result = await this.run(command, onLine);
+    if (result.code === 0) return result;
+
+    throw new Error(`${message}: ${result.stderr || result.stdout}`);
+  }
+}
+
+class DockerNetwork {
+  constructor(private readonly docker: Docker) {}
+
+  async exists(name: string) {
+    return (await this.docker.snapshot()).networks.has(name);
+  }
+
+  async create(name: string, subnet?: string) {
+    if (await this.exists(name)) return false;
+
+    const suffix = subnet ? ` --subnet=${subnet}` : "";
+    await this.docker.runOrThrow(
+      `network create ${name}${suffix}`,
+      "Network creation failed",
+    );
+
+    await this.docker.track((state) => state.networks.add(name));
+    return true;
+  }
+
+  async remove(name: string) {
+    if (!(await this.exists(name))) return false;
+
+    await this.docker.runOrThrow(`network rm ${name}`, "Network removal failed");
+    await this.docker.track((state) => state.networks.delete(name));
+    return true;
+  }
+
+  // Disconnect can fail because the container was never attached, which is not
+  // an error. Connect failing is, the address is what nginx resolves to
+  async reconnect(network: string, container: string, ip: string) {
+    if (!(await this.docker.container.exists(container))) return false;
+
+    await this.docker.run(`network disconnect ${network} ${container}`);
+    await this.docker.runOrThrow(
+      `network connect --ip ${ip} ${network} ${container}`,
+      "Network connect failed",
+    );
+
+    return true;
+  }
+}
+
+class DockerImage {
+  constructor(private readonly docker: Docker) {}
+
+  async exists(name: string) {
+    return (await this.docker.snapshot()).images.has(name);
+  }
+
+  // Every version of one image the host is holding. A deploy tags what it built
+  // by release, and without this the previous ones are never reclaimed.
+  // :latest is not one of them: it is the moving name a container is created
+  // from, and reclaiming it leaves the host unable to start the app
+  async versionsOf(repository: string) {
+    const images = (await this.docker.snapshot()).images;
+
+    return [...images].filter(
+      (name) => name.startsWith(`${repository}:`) && name !== `${repository}:latest`,
+    );
+  }
+
+  // BuildKit is already inside the daemon that will run the container, so the
+  // image it produces never has to be serialised, transferred or loaded
+  async build(spec: BuildInvocation, onLine?: OnLine) {
+    const command = [
+      "build",
+      // Provenance attestations make the result a manifest list, which is a
+      // different thing to tag and nothing here consumes them
+      "--progress plain --provenance=false --pull",
+      `-f ${spec.dockerfile}`,
+      ...spec.tags.map((tag) => `-t ${tag}`),
+      ...Object.entries(spec.secrets ?? {}).map(
+        ([id, path]) => `--secret id=${id},src=${path}`,
+      ),
+      ...(spec.target ? [`--target ${spec.target}`] : []),
+      spec.context,
+    ].join(" ");
+
+    await this.docker.runOrThrow(command, "Image build failed", onLine);
+    await this.docker.track((state) => {
+      for (const tag of spec.tags) state.images.add(tag);
+    });
+
+    return true;
+  }
+
+  // A second name for an image that is already local. Costs nothing, and it is
+  // what lets the next deploy recognise a commit it already holds
+  async retag(from: string, to: string) {
+    await this.docker.runOrThrow(`tag ${from} ${to}`, "Image tagging failed");
+    await this.docker.track((state) => state.images.add(to));
+    return true;
+  }
+
+  async remove(name: string) {
+    if (!(await this.exists(name))) return false;
+
+    await this.docker.runOrThrow(
+      `image remove -f ${name}`,
+      "Image removal failed",
+    );
+
+    await this.docker.track((state) => state.images.delete(name));
+    return true;
+  }
+}
+
+class DockerContainer {
+  constructor(private readonly docker: Docker) {}
+
+  async exists(name: string) {
+    return (await this.docker.snapshot()).containers.has(name);
+  }
+
+  async status(name: string) {
+    return (await this.docker.snapshot()).containers.get(name)?.state ?? "none";
+  }
+
+  // What the running container was created from. Absent for one redkite did not
+  // create, or created before it started recording it
+  async specOf(name: string) {
+    return (await this.docker.snapshot()).containers.get(name)?.spec;
+  }
+
+  async isRunning(name: string) {
+    return RUNNING.has(await this.status(name));
+  }
+
+  async start(name: string) {
+    if (!(await this.exists(name))) {
+      throw new Error(`Cannot start ${name}, it does not exist`);
+    }
+
+    await this.docker.runOrThrow(
+      `container start ${name}`,
+      "Container start failed",
+    );
+
+    await this.docker.track((state) => set(state, name, "running"));
+    return true;
+  }
+
+  async stop(name: string) {
+    if (!(await this.exists(name))) return false;
+
+    await this.docker.runOrThrow(
+      `container stop ${name}`,
+      "Container stop failed",
+    );
+
+    await this.docker.track((state) => set(state, name, "exited"));
+    return true;
+  }
+
+  // Refuses rather than clobbers, a rename onto an existing name would lose
+  // whichever container is already there
+  async rename(from: string, to: string) {
+    if (!(await this.exists(from))) return false;
+    if (await this.exists(to)) return false;
+
+    await this.docker.runOrThrow(
+      `container rename ${from} ${to}`,
+      "Container rename failed",
+    );
+
+    await this.docker.track((state) => {
+      const held = state.containers.get(from) ?? { state: "unknown" };
+      state.containers.delete(from);
+      state.containers.set(to, held);
+    });
+
+    return true;
+  }
+
+  async remove(name: string) {
+    if (!(await this.exists(name))) return false;
+    if (await this.isRunning(name)) {
+      throw new Error(`Cannot remove ${name} while it is running`);
+    }
+
+    await this.docker.runOrThrow(
+      `container rm ${name}`,
+      "Container removal failed",
+    );
+
+    await this.docker.track((state) => state.containers.delete(name));
+    return true;
+  }
+
+  async create(builder: DockerBuilder) {
+    await this.docker.runOrThrow(builder.parse(), "Container create failed");
+    await this.docker.track((state) =>
+      state.containers.set(builder.named(), {
+        state: "created",
+        spec: builder.labelled(SPEC_LABEL),
+      }),
+    );
+    return true;
+  }
+
+  builder() {
+    return new DockerBuilder(this);
+  }
+}
+
+// A state change keeps whatever the container was created from: the label is
+// on the container, and only a recreate can move it
+function set(state: HostState, name: string, next: string) {
+  state.containers.set(name, { ...state.containers.get(name), state: next });
+}
+
+function lines(block: string) {
+  return block
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean);
+}
+
+export class DockerBuilder {
+  private _name = "";
+  private _image = "";
+  private _hostname?: string;
+  private _ip?: string;
+  private _restart?: string;
+  private readonly _networks: string[] = [];
+  private readonly _volumes: string[] = [];
+  private readonly _hosts: string[] = [];
+  private readonly _ports: string[] = [];
+  private readonly _env: string[] = [];
+  private readonly _labels = new Map<string, string>();
+  private _envFile?: string;
+
+  constructor(private readonly container: DockerContainer) {}
+
+  name(value: string) {
+    this._name = value;
+    return this;
+  }
+
+  image(value: string) {
+    this._image = value;
+    return this;
+  }
+
+  hostname(value: string) {
+    this._hostname = value;
+    return this;
+  }
+
+  network(value: string) {
+    this._networks.push(value);
+    return this;
+  }
+
+  volume(from: string, to: string) {
+    this._volumes.push(`${from}:${to}`);
+    return this;
+  }
+
+  extraHost(name: string, ip: string) {
+    this._hosts.push(`${name}:${ip}`);
+    return this;
+  }
+
+  restart(value: string) {
+    this._restart = value;
+    return this;
+  }
+
+  ip(value: string) {
+    this._ip = value;
+    return this;
+  }
+
+  env(name: string, value: string) {
+    this._env.push(`${name}=${value}`);
+    return this;
+  }
+
+  // A path on the host rather than the values themselves, so a password is
+  // never an argument in the process list or in the shell history
+  envFile(path: string) {
+    this._envFile = path;
+    return this;
+  }
+
+  label(name: string, value: string) {
+    this._labels.set(name, value);
+    return this;
+  }
+
+  labelled(name: string) {
+    return this._labels.get(name);
+  }
+
+  port(from: number, to: number) {
+    this._ports.push(`${from}:${to}`);
+    return this;
+  }
+
+  named() {
+    return this._name;
+  }
+
+  parse() {
+    if (!this._name) throw new Error("A container needs a name");
+    if (!this._image) throw new Error(`${this._name} has no image`);
+
+    // Defaults to the container name. The original emitted "--hostname
+    // undefined", because nothing ever called the setter
+    const hostname = this._hostname ?? this._name;
+
+    return [
+      `container create --name ${this._name} --hostname ${hostname}`,
+      ...this._volumes.map((volume) => `-v ${volume}`),
+      ...this._networks.map((network) => `--network ${network}`),
+      ...this._ports.map((port) => `-p ${port}`),
+      ...this._hosts.map((host) => `--add-host ${host}`),
+      ...[...this._labels].map(([name, value]) => `--label ${name}=${value}`),
+      ...(this._envFile ? [`--env-file ${this._envFile}`] : []),
+      ...this._env.map((entry) => `-e ${entry}`),
+      ...(this._ip ? [`--ip ${this._ip}`] : []),
+      ...(this._restart ? [`--restart ${this._restart}`] : []),
+      this._image,
+    ].join(" ");
+  }
+
+  async create() {
+    return await this.container.create(this);
+  }
+}

package/src/dockerfile.ts

@@ -0,0 +1,228 @@
+import { appRoot, destinationFor, mountFor, rootedAt } from "./layout.js";
+import type { BuildSpec, CarryPath } from "./types.js";
+
+// The pipeline, as the only thing that runs it: a Dockerfile handed to the
+// BuildKit already inside the deploy host's daemon. Layer order is the
+// contract, everything that does not change between deploys comes first, so a
+// new commit only invalidates from the source copy down.
+
+export type DockerfileContext = {
+  // Cache name to the id BuildKit keys the mount by, one per app and
+  // environment so two builds never share a node_modules
+  caches: Record<string, string>;
+  port: number;
+  release: string;
+  environment: string;
+  // Secrets arrive by id. Their contents never enter this file, which keeps
+  // them out of docker history. The id carries the fingerprint, because a
+  // secret mount is not part of a layer's cache key and a changed environment
+  // would otherwise be answered with the image built from the old one
+  envSecret: string;
+  // Container path to secret id, for credentials that have to be files
+  fileSecrets: Record<string, string>;
+  // Where the app sits in the repository, when it is not the repository. The
+  // build steps and the shipped command run there, and every /app path the
+  // spec names is read against it
+  dir?: string;
+};
+
+// Named here because the build stops at this stage to keep an image a
+// step before the swap can run in, and two spellings of it would drift
+export const BUILDER_STAGE = "builder";
+
+// What BuildKit reads beside the Dockerfile. Without an include the checkout
+// keeps its own rules and only .git is held back. With one, everything is held
+// back and the named paths are let through again, so what the build sees is
+// what the release was taken over
+export function renderDockerignore(include?: string[]) {
+  if (!include) return ".git\n**/.git\n";
+
+  // .git after the exemptions, because the last rule to match is the one that
+  // decides and an included directory would otherwise carry it back in
+  return ["*", ...include.map((path) => `!${path}`), ".git", "**/.git", ""].join("\n");
+}
+
+export function renderDockerfile(
+  spec: BuildSpec,
+  context: DockerfileContext,
+): string {
+  return [
+    "# syntax=docker/dockerfile:1.7",
+    ...builderStage(spec, context),
+    "",
+    ...runtimeStage(spec, context),
+    "",
+  ].join("\n");
+}
+
+function builderStage(spec: BuildSpec, context: DockerfileContext) {
+  const mounts = cacheMounts(spec, context);
+  const workdir = appRoot(context.dir);
+
+  // The repository root, not the app's own directory: a workspace resolves one
+  // lockfile for every package in it, so the install below has to see them all
+  const lines = [
+    `FROM ${spec.builderImage} AS ${BUILDER_STAGE}`,
+    "WORKDIR /app",
+  ];
+
+  if (spec.aptPackages.length > 0) {
+    lines.push(`RUN apk add --no-cache ${spec.aptPackages.join(" ")}`);
+  }
+
+  lines.push(`ENV NODE_ENV=${context.environment}`);
+  lines.push("ENV NEXT_TELEMETRY_DISABLED=1");
+
+  if (spec.dependencies) lines.push(...dependencyLayer(spec, mounts));
+
+  // Submodules are already in the context: the checkout on the host resolved
+  // them, so nothing in the build needs an agent or a .git directory
+  lines.push(copy(".", "/app"));
+
+  // Only now, so the steps below read the app's own manifest and write beside it
+  if (context.dir) lines.push(`WORKDIR ${workdir}`);
+
+  lines.push(
+    `RUN --mount=type=secret,id=${context.envSecret} cp /run/secrets/${context.envSecret} ${workdir}/.env`,
+    `ENV SENTRY_RELEASE=${context.release}`,
+  );
+
+  // Unquoted: the shell form hands everything after RUN to sh -c, so a step is
+  // written exactly as it would be typed. Quoting it made the whole command one
+  // word, and the shell went looking for a program by that name
+  for (const step of spec.steps) {
+    lines.push(`RUN ${mounts}${step}`);
+  }
+
+  if (spec.sourcemaps) {
+    // The built .env ships inside the image, and an upload token is build-time
+    const token = spec.sourcemaps.stripFromImage;
+    lines.push(`RUN sed -i '/^${token}=/d' ${rootedAt(spec.output, context.dir)}/.env`);
+  }
+
+  return lines;
+}
+
+function runtimeStage(spec: BuildSpec, context: DockerfileContext) {
+  const output = rootedAt(spec.output, context.dir);
+
+  // Where the app lands inside the tree that was just copied to /app. An output
+  // keeping the repository's structure puts it back under dir, and everything
+  // else flattens it to the top
+  const nested = spec.keepsLayout ? context.dir : undefined;
+  const lines = [`FROM ${spec.runtimeImage}`, `WORKDIR ${appRoot(nested)}`];
+
+  if (spec.runtimePackages.length > 0) {
+    lines.push(`RUN apk add --no-cache ${spec.runtimePackages.join(" ")}`);
+  }
+
+  // Before the output, which changes every commit: what these install is a
+  // property of the image, not of the release being deployed
+  for (const step of spec.runtimeSteps) {
+    lines.push(`RUN ${step}`);
+  }
+
+  // The one copy nothing can be allowed to skip: an image without it starts,
+  // answers, and serves a 404 for everything the app was supposed to be
+  lines.push(copy(output, "/app", BUILDER_STAGE));
+
+  // Directories the output does not contain but the runtime needs, such as a
+  // generated client or the static assets a standalone build leaves behind.
+  // The two sides are rooted separately: dir is where the app was built, and
+  // nested is where the output put it
+  for (const entry of spec.carry) {
+    const { path, optional } = carried(entry);
+    const from = rootedAt(path, context.dir);
+    const to = rootedAt(destinationFor(path, spec.output), nested);
+
+    lines.push(optional ? copyOrSkip(from, to, BUILDER_STAGE) : copy(from, to, BUILDER_STAGE));
+  }
+
+  for (const [path, secret] of Object.entries(context.fileSecrets)) {
+    lines.push(
+      `RUN --mount=type=secret,id=${secret} cp /run/secrets/${secret} ${path}`,
+    );
+  }
+
+  lines.push(
+    `EXPOSE ${context.port}`,
+    `CMD ${JSON.stringify(spec.entrypoint)}`,
+  );
+
+  return lines;
+}
+
+function carried(entry: CarryPath) {
+  if (typeof entry === "string") return { path: entry, optional: false };
+  return { path: entry.path, optional: entry.optional };
+}
+
+// A COPY whose source does not exist fails the build, which is what a source
+// the build was supposed to produce should do
+function copy(from: string, to: string, stage?: string) {
+  return `COPY ${stage ? `--from=${stage} ` : ""}${from} ${to}`;
+}
+
+// Bracketing the last character makes the source a pattern, and a pattern
+// matching nothing is skipped: the only way a Dockerfile has to say "if it
+// exists". For a source whose absence is a fact about the repository
+function copyOrSkip(from: string, to: string, stage?: string) {
+  const path = from.replace(/\/+$/, "");
+  const last = path.slice(-1);
+
+  // An empty pattern matches nothing and would be skipped in silence, which is
+  // the one outcome this whole distinction exists to prevent
+  if (!last) return copy(from, to, stage);
+
+  return copy(`${path.slice(0, -1)}[${last}]`, to, stage);
+}
+
+// One mount per cache, repeated on every step, because a Dockerfile scopes a
+// cache to the RUN that asks for it rather than to the stage
+function cacheMounts(spec: BuildSpec, context: DockerfileContext) {
+  // Only the caches the topology derived an id for. It drops any whose target
+  // another already covers, and two mounts on one target is a build that does
+  // not start
+  return spec.caches
+    .filter((name) => name in context.caches)
+    .map((name) => {
+      const id = context.caches[name] ?? name;
+      return `--mount=type=cache,id=${id},target=${mountFor(name, context.dir)} `;
+    })
+    .join("");
+}
+
+function dependencyLayer(spec: BuildSpec, mounts: string) {
+  const { files, step, stripScripts = [] } = spec.dependencies ?? {
+    files: [],
+    step: "",
+  };
+
+  // A lockfile the repository does not have is not an error here. The install
+  // below is what decides that, and it says so in the package manager's words
+  const lines = files.map((file) => copyOrSkip(file, `/app/${file}`));
+
+  // This layer is the manifest and the lockfile, nothing else. A root prepare
+  // that installs git hooks or calls into scripts/ has neither
+  if (stripScripts.length > 0) {
+    lines.push(`RUN node --input-type=commonjs -e ${quote(strip(stripScripts))}`);
+  }
+
+  lines.push(`RUN ${mounts}${step}`);
+  return lines;
+}
+
+function strip(names: string[]) {
+  return [
+    'const fs = require("fs"), path = "/app/package.json";',
+    'const manifest = JSON.parse(fs.readFileSync(path, "utf8"));',
+    `for (const name of ${JSON.stringify(names)}) delete manifest.scripts?.[name];`,
+    "fs.writeFileSync(path, JSON.stringify(manifest, null, 2));",
+  ].join("");
+}
+
+// Only for a command passed as one argument to another. Single quotes are what
+// the shell strips, so a script containing double quotes survives
+function quote(command: string) {
+  return `'${command.replaceAll("'", `'\\''`)}'`;
+}