redkite 0.1.7 → 0.1.9

package/src/plugins/snapshot.ts

@@ -0,0 +1,44 @@
+import type { Plan } from "../pipeline.js";
+
+// What the two snapshot plugins share. A snapshot is a control-plane call
+// rather than anything the deploy host does, so it runs where the credentials
+// are, which is the process running redkite.
+
+// Hung before the swap and before any migration, so the thing being protected
+// from is still ahead of it. Steps run in the order the config lists them, so a
+// snapshot has to be written above the migrate it guards
+export type SnapshotPoint = `swap:before:${string}`;
+
+// Letters, digits and single hyphens, which is the intersection of what RDS
+// accepts as a snapshot identifier and what a point may be named
+export function slug(value: string) {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .replace(/-{2,}/g, "-");
+}
+
+// Sortable, second-resolution, and legal in every identifier below. Two
+// snapshots in one second would collide, and two deploys in one second is not
+// a thing a swap can do
+export function stamp(now: Date) {
+  return now.toISOString().replace(/[-:T]/g, "").slice(0, 14);
+}
+
+// Names the deploy it was taken for. A snapshot nobody can place is one nobody
+// will dare restore
+export function snapshotName(prefix: string, environment: string, now: Date) {
+  return slug(`${prefix}-${environment}-${stamp(now)}`);
+}
+
+// Read before the run starts, so a missing token is a config that fails while
+// the host is untouched rather than a deploy that stops with the swap ahead
+export function tokenFrom(name: string, plan: Plan) {
+  const value = process.env[name];
+  if (value) return value;
+
+  throw new Error(
+    `${name} is not set, and ${plan.config.project} snapshots its database before swapping`,
+  );
+}

package/src/presets/nodeApp.ts

@@ -1,4 +1,4 @@
-import type { BuildSpec, CarryPath, SourcemapSpec } from "../types.js";
+import type { BuildSpec, CarryPath } from "../types.js";
 
 type NodeAppOptions = {
   builder?: string;
@@ -12,7 +12,6 @@ type NodeAppOptions = {
   carry?: CarryPath[];
   submodules?: boolean;
   caches?: string[];
-  sourcemaps?: SourcemapSpec;
 };
 
 export function nodeApp(options: NodeAppOptions): BuildSpec {
@@ -41,6 +40,5 @@ export function nodeApp(options: NodeAppOptions): BuildSpec {
     aptPackages: ["git"],
     runtimePackages: ["curl"],
     runtimeSteps: [],
-    sourcemaps: options.sourcemaps,
   };
 }

package/src/recover.ts

@@ -0,0 +1,90 @@
+import { revert } from "./deploy.js";
+import { Docker } from "./docker.js";
+import { finalHost, type Host } from "./host.js";
+import { silent, type Log } from "./log.js";
+import { topologyFor, type Topology } from "./topology.js";
+import type { Deployment } from "./types.js";
+
+// Undoing a run that did not finish, from a process that is not it. A cancelled
+// job is killed rather than asked, so nothing it was holding survives to tidy
+// up: what the host is left in has to be readable from the host alone.
+
+export type RecoverOptions = {
+  config: Deployment;
+  environment: string;
+  host: Host;
+  log?: Log;
+};
+
+export type RolledBack = {
+  // Apps put back on the live address they were serving from
+  restored: string[];
+  // Apps that were already where they belong, which is most runs
+  untouched: string[];
+};
+
+// A retired container is the whole signal. One exists between the swap and the
+// cleanup that removes it, so finding one means a run moved an address and did
+// not get to say whether that worked
+export async function rollback(options: RecoverOptions): Promise<RolledBack> {
+  const { log, topology, docker } = opened(options);
+
+  const restored: string[] = [];
+  const untouched: string[] = [];
+
+  for (const app of topology.apps) {
+    if (!(await docker.container.exists(app.retired))) {
+      untouched.push(app.container);
+      continue;
+    }
+
+    log(`Putting ${app.container} back`);
+    await revert(docker, topology, app);
+    restored.push(app.container);
+  }
+
+  return { restored, untouched };
+}
+
+export type TakenDown = {
+  stopped: string[];
+};
+
+// Stopped rather than removed, so the next run adopts them where it left off.
+// For an environment that exists to be built against rather than served from,
+// which is the one a cancelled job should not leave running
+export async function down(options: RecoverOptions): Promise<TakenDown> {
+  const { log, topology, docker } = opened(options);
+  const stopped: string[] = [];
+
+  for (const name of every(topology)) {
+    if (!(await docker.container.isRunning(name))) continue;
+
+    log(`Stopping ${name}`);
+    await docker.container.stop(name);
+    stopped.push(name);
+  }
+
+  return { stopped };
+}
+
+// Everything this environment named, whether or not it was ever created. The
+// proxy is derived rather than listed, so it would be missed by walking the
+// config alone
+function every(topology: Topology) {
+  return [
+    ...topology.apps.flatMap((app) => [app.container, app.retired, app.failed]),
+    ...topology.services.map((service) => service.container),
+    topology.router.container,
+  ];
+}
+
+// Both of these exist because something was stopped, so neither may be refused
+// by that same stop
+function opened(options: RecoverOptions) {
+  return {
+    log: options.log ?? silent,
+    topology: topologyFor(options.config, options.environment),
+    docker: new Docker(finalHost(options.host)),
+  };
+}

package/src/secrets/store.ts

@@ -16,12 +16,11 @@ import type { SecretStore } from "./refs.js";
 
 const run = promisify(execFile);
 
+// A session the caller already holds, or what it takes to obtain one. Both
+// spellings in one object would be a pair of credentials nobody can tell apart
 export type BitwardenCredentials = {
-  clientId: string;
-  clientSecret: string;
-  password: string;
   detail?: (message: string) => void;
-};
+} & ({ session: string } | { clientId: string; clientSecret: string; password: string });
 
 // Used when bw is not already on PATH. Pinned, because an unpinned CLI is a
 // different program on a machine that has never run a deploy before
@@ -53,22 +52,7 @@ export async function bitwardenStore(
       maxBuffer: 32 * 1024 * 1024,
     });
 
-  detail("unlocking the vault");
-
-  // Already logged in is not an error, the session is what matters
-  await bw(["login", "--apikey"], {
-    BW_CLIENTID: credentials.clientId,
-    BW_CLIENTSECRET: credentials.clientSecret,
-  }).catch(() => undefined);
-
-  const unlocked = await bw(["unlock", "--passwordenv", "BW_PASSWORD", "--raw"], {
-    BW_PASSWORD: credentials.password,
-  }).catch((error: unknown) => {
-    throw new Error(`Could not unlock the Bitwarden vault: ${messageOf(error)}`);
-  });
-
-  const session = unlocked.stdout.trim();
-  if (!session) throw new Error("Bitwarden unlocked without a session");
+  const session = await unlock(bw, credentials, detail);
 
   // An item added since the last deploy is not in the local vault otherwise,
   // and bw get answers "not found" rather than fetching it
@@ -97,6 +81,41 @@ export async function bitwardenStore(
   };
 }
 
+type Bw = (args: string[], env?: Record<string, string>) => Promise<{ stdout: string }>;
+
+// A session handed in is one nobody had to obtain, which is the whole of what
+// a key in the environment buys: no api credentials, no master password, and
+// two fewer round trips before the first item is read
+async function unlock(
+  bw: Bw,
+  credentials: BitwardenCredentials,
+  detail: (message: string) => void,
+) {
+  if ("session" in credentials) {
+    detail("using the session it was given");
+    return credentials.session;
+  }
+
+  detail("unlocking the vault");
+
+  // Already logged in is not an error, the session is what matters
+  await bw(["login", "--apikey"], {
+    BW_CLIENTID: credentials.clientId,
+    BW_CLIENTSECRET: credentials.clientSecret,
+  }).catch(() => undefined);
+
+  const unlocked = await bw(["unlock", "--passwordenv", "BW_PASSWORD", "--raw"], {
+    BW_PASSWORD: credentials.password,
+  }).catch((error: unknown) => {
+    throw new Error(`Could not unlock the Bitwarden vault: ${messageOf(error)}`);
+  });
+
+  const session = unlocked.stdout.trim();
+  if (session) return session;
+
+  throw new Error("Bitwarden unlocked without a session");
+}
+
 type Cli = { file: string; prefix: string[] };
 
 // A machine with the CLI installed pays nothing. One without it installs the

package/src/source.ts

@@ -13,6 +13,12 @@ export type Source = {
   tree: string;
 };
 
+// What a clone is resolved against. A branch is tracked and a tag or a commit
+// is pinned, and which of the three it is decides where git is asked to look
+export type RefKind = "branch" | "tag" | "commit";
+
+export type Ref = { kind: RefKind; name: string };
+
 export type SourceRequest = {
   // Names the mirror and the checkout. Keyed per app and environment rather
   // than per repository, so two deploys never fetch into one directory at once
@@ -23,11 +29,19 @@ export type SourceRequest = {
   // What of a path goes into the build. Absent leaves it to the work tree's
   // own .gitignore, which is the only other thing that can say
   include?: string[];
-  branch: string;
+  ref: Ref;
   submodules: boolean;
   detail?: (message: string) => void;
 };
 
+// Where each kind lives. A tag is peeled, so an annotated one answers with the
+// commit it points at rather than with the tag object
+const REVISION: Record<RefKind, (name: string) => string> = {
+  branch: (name) => `refs/heads/${name}`,
+  tag: (name) => `refs/tags/${name}^{commit}`,
+  commit: (name) => `${name}^{commit}`,
+};
+
 // accept-new rather than no: a host key that changes is still worth refusing
 const GIT_SSH = "ssh -o StrictHostKeyChecking=accept-new";
 
@@ -110,8 +124,9 @@ async function clonedSource(
   const detail = request.detail ?? (() => {});
   const mirror = `${host.cache}/mirrors/${request.name}.git`;
   const path = `${host.cache}/source/${request.name}`;
+  const { kind, name } = request.ref;
 
-  detail(`fetching ${request.branch}`);
+  detail(`fetching ${name}`);
   await run(host, `updating the mirror of ${repo}`, [
     `if [ -d '${mirror}' ]; then`,
     `  git -C '${mirror}' remote set-url origin '${repo}'`,
@@ -122,13 +137,13 @@ async function clonedSource(
     "fi",
   ]);
 
-  const resolved = await run(host, `resolving ${request.branch}`, [
-    `git -C '${mirror}' rev-parse 'refs/heads/${request.branch}'`,
+  const resolved = await run(host, `resolving ${name}`, [
+    `git -C '${mirror}' rev-parse '${REVISION[kind](name)}'`,
   ]);
 
   const release = resolved.stdout.trim();
   if (!/^[0-9a-f]{40}$/.test(release)) {
-    throw new Error(`${repo} has no branch ${request.branch}`);
+    throw new Error(`${repo} has no ${kind} ${name}`);
   }
 
   detail(`checking out ${release.slice(0, 7)}`);

package/src/sshHost.ts

@@ -3,6 +3,7 @@ import { dirname } from "node:path";
 
 import type { Host, OnLine, Result } from "./host.js";
 import { quote, spawnCollect } from "./shell.js";
+import type { HostKeys } from "./types.js";
 
 // The deploy host, reached over ssh. The agent is forwarded rather than a key,
 // there is nothing to forward, and the agent travels with the connection so the
@@ -23,23 +24,40 @@ export type SshOptions = {
   shell?: Shell;
   directory?: string;
   cache?: string;
+  // How the host's key is checked. Defaults to accept-new
+  hostKeys?: HostKeys;
   // Aborting kills the ssh client. The command it was carrying keeps running on
   // the other machine, which is what stop is for
   signal?: AbortSignal;
 };
 
+// What ssh calls each of them. accept-new is the default rather than no: a
+// deploy hands over a vault's contents, and the last thing it should do that
+// to is a machine that answered to the address and nothing else
+const CHECKING: Record<HostKeys, string> = {
+  "accept-new": "accept-new",
+  strict: "yes",
+  off: "no",
+};
+
 // One TCP connection and one authentication for the whole deploy. Without it
 // every command pays a handshake, which is most of what a command costs
-const MULTIPLEX = [
-  "-o",
-  "ControlMaster=auto",
-  "-o",
-  "ControlPersist=60s",
-  "-o",
-  "StrictHostKeyChecking=no",
-  // The host clones from GitHub on our behalf rather than us shipping a tree
-  "-A",
-];
+function multiplex(hostKeys: HostKeys = "accept-new") {
+  return [
+    "-o",
+    "ControlMaster=auto",
+    "-o",
+    "ControlPersist=60s",
+    "-o",
+    `StrictHostKeyChecking=${CHECKING[hostKeys]}`,
+    // Never a prompt. Unknown under strict is a refusal with a reason, and a
+    // question nobody is there to answer is a deploy that hangs
+    "-o",
+    "BatchMode=yes",
+    // The host clones from GitHub on our behalf rather than us shipping a tree
+    "-A",
+  ];
+}
 
 export async function sshHost(bastion: string, options: SshOptions = {}): Promise<Host> {
   const control = `/tmp/redkite-${randomUUID().slice(0, 8)}.control`;
@@ -57,8 +75,10 @@ export async function sshHost(bastion: string, options: SshOptions = {}): Promis
     options.shell ??
     ((command: string) => spawnCollect("sh", ["-c", command], { signal }));
 
+  const flags = multiplex(options.hostKeys);
+
   const argv = (command: string) => [
-    ...MULTIPLEX,
+    ...flags,
     "-o",
     `ControlPath=${control}`,
     bastion,
@@ -111,6 +131,10 @@ export async function sshHost(bastion: string, options: SshOptions = {}): Promis
     pipe: async (local, remote) =>
       await shell(`${local} | ssh ${argv(remote).map(quote).join(" ")}`),
 
+    // Runs on the connection that is already open, and without the signal that
+    // killed whatever made it necessary
+    final: async (command) => await final(argv(command)),
+
     // Signalled where they are running, not here. Killing the client would
     // leave the build going with nothing left able to reach it
     stop: async (name) => {

package/src/steps.ts

@@ -1,4 +1,5 @@
 import { environmentOf } from "./config.js";
+import { envFlags } from "./environment.js";
 import type { Built, Plan, Step } from "./pipeline.js";
 import type { Topology } from "./topology.js";
 import type { AppSpec, StepNetwork } from "./types.js";
@@ -48,10 +49,16 @@ export function migrate(options: MigrateOptions): Step<`swap:before:${string}`>
       const image = builderOf(input, options.app);
       context.task.detail(`${options.app}: ${options.command}`);
 
+      // Nothing from the vault is in the image, so a migration reads its
+      // database url from the environment it is given rather than from a file
+      // the build left behind
+      const app = context.config.apps.find((item) => item.name === options.app);
+
       await context.docker.runOrThrow(
         [
           "run --rm",
           ...attachment(options.network ?? "host", context.topology),
+          ...(app ? await envFlags(app, context) : []),
           "--workdir /app",
           image,
           ...command,
@@ -90,12 +97,6 @@ export function builderOf(input: Built, name: string) {
   throw new Error(`${name} has no builder image, so nothing can be run in it`);
 }
 
-type SentryOptions = { stripFromImage: string };
-
-export function sentry(options: SentryOptions) {
-  return { provider: "sentry" as const, stripFromImage: options.stripFromImage };
-}
-
 export function routeOf(app: AppSpec) {
   return app.route;
 }

package/src/types.ts

@@ -1,4 +1,5 @@
 import type { AnyStep } from "./pipeline.js";
+import type { Plugin } from "./plugin.js";
 
 // The shapes a deploy config is written against. Nothing here knows about
 // Docker or git, so a config can be planned and asserted on without a host to
@@ -33,11 +34,26 @@ export type Environment = {
 // the alias the apps know it by
 export type StepNetwork = "host" | "deployment" | "none" | { named: string };
 
+// How the deploy host proves it is the machine it says it is, before a key,
+// an environment file or an image is handed to it
+export type HostKeys =
+  // Trusted the first time and refused if the key ever changes after that,
+  // which is what a host on a stable address wants and needs no setup
+  | "accept-new"
+  // Refused unless it is already in known_hosts. Nothing is taken on trust,
+  // at the cost of putting the key there before the first deploy
+  | "strict"
+  // Checked not at all. For a host whose address is handed out again and
+  // again, and only where nothing on the way to it can be listened to
+  | "off";
+
 export type DeployHost = {
   // SSH destination, user@address
   bastion: string;
   // Socket path on that machine. Local when absent, deploying to this one
   socket?: string;
+  // Defaults to accept-new
+  hostKeys?: HostKeys;
 };
 
 // One item in a store, named at the point of use rather than through a lookup
@@ -127,16 +143,6 @@ export type BuildSpec = {
   // Shell commands run in the runtime image, for what a package manager cannot
   // install. Above the output copy, so a new commit does not repeat them
   runtimeSteps: string[];
-  // Uploads source maps during the build, then deletes them from the image
-  sourcemaps?: SourcemapSpec;
-};
-
-export type SourcemapSpec = {
-  // Which service receives the maps, "sentry" is the only one today
-  provider: "sentry";
-  // Env var stripped from the shipped file, it is a build-time credential and
-  // has no reason to travel inside the image
-  stripFromImage: string;
 };
 
 // How a build is decided to work. The commands run in the builder image, which
@@ -193,6 +199,13 @@ export type AppSpec = {
   // Environment for the app. Several are merged in the order written, so a
   // shared ref can come first and a per-app one override it
   secrets?: SecretRefs;
+  // What this app is built from, overriding the environment's branch. At most
+  // one of the three. A branch is tracked, so every deploy takes whatever its
+  // head is by then; a tag or a commit is pinned, and the app stops moving
+  // until this line does. Ignored by an app built from a path
+  branch?: string;
+  tag?: string;
+  commit?: string;
   // Where the app sits in the repository, for a monorepo whose root is not it.
   // Build steps and the shipped command run there. The dependency install does
   // not: a workspace lockfile is resolved at the root for every package at once
@@ -237,7 +250,11 @@ export type Deployment = {
   // The applications. Everything derived, addresses, container names, nginx
   // upstreams and location blocks, is a function of this list
   apps: AppSpec[];
-  // Work injected into the run, from this file or from a plugin that answers
-  // with some. Addressed by where it runs, so nothing here has to be called
+  // Work injected into the run. Addressed by where it runs, so nothing here
+  // has to be called
   steps?: AnyStep[];
+  // Everything this deployment opts into: the vault that answers its secret
+  // refs, and whatever else brings steps of its own. Nothing a plugin carries
+  // happens until it is listed here, redkite's own vault included
+  plugins?: Plugin[];
 };

package/dist/secrets/bitwarden.d.ts

@@ -1,3 +0,0 @@
-import type { SecretRef } from "../types.js";
-export declare function bitwarden(id: string): SecretRef;
-//# sourceMappingURL=bitwarden.d.ts.map

package/dist/secrets/bitwarden.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"bitwarden.d.ts","sourceRoot":"","sources":["../../src/secrets/bitwarden.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAI7C,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,CAE/C"}

package/dist/secrets/bitwarden.js

@@ -1,6 +0,0 @@
-// Names an item in a Bitwarden vault. The id is not a secret, it is a pointer,
-// so it belongs in the config while the credentials arrive at deploy time
-export function bitwarden(id) {
-    return { provider: "bitwarden", id };
-}
-//# sourceMappingURL=bitwarden.js.map

package/dist/secrets/bitwarden.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"bitwarden.js","sourceRoot":"","sources":["../../src/secrets/bitwarden.ts"],"names":[],"mappings":"AAEA,+EAA+E;AAC/E,0EAA0E;AAC1E,MAAM,UAAU,SAAS,CAAC,EAAU;IAClC,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;AACvC,CAAC"}

package/src/secrets/bitwarden.ts

@@ -1,7 +0,0 @@
-import type { SecretRef } from "../types.js";
-
-// Names an item in a Bitwarden vault. The id is not a secret, it is a pointer,
-// so it belongs in the config while the credentials arrive at deploy time
-export function bitwarden(id: string): SecretRef {
-  return { provider: "bitwarden", id };
-}