redkite 0.1.10 → 0.1.12

package/src/cli/index.ts

@@ -1,4 +1,4 @@
-import { environmentOf } from "../config.js";
+import { environmentOf, withEnvironment } from "../config.js";
 import { deploy, verify } from "../deploy.js";
 import { Docker } from "../docker.js";
 import type { Host } from "../host.js";
@@ -17,6 +17,7 @@ import type { Deployment, DeployHost } from "../types.js";
 import { needsAgent, requireAgent } from "./agent.js";
 import { loadConfig, projectRoot } from "./config.js";
 import { loadDotenv } from "./dotenv.js";
+import { dumpCrash, recording } from "./crash.js";
 import { createLog, describeFailure } from "./log.js";
 
 // One command that reads the config and does everything under it: the agent,
@@ -42,7 +43,7 @@ of the project, and everything below it is derived.
 
 // Wraps a host rather than living inside one, so both implementations are
 // measured the same way and neither knows it is being timed
-function measured(host: Host, log?: Log) {
+function measured(host: Host, say?: (line: string) => void) {
   const totals = { commands: 0, commandMs: 0, files: 0 };
 
   const wrapped: Host = {
@@ -61,7 +62,7 @@ function measured(host: Host, log?: Log) {
 
       // The exit code matters: several of these are allowed to fail, and a
       // deploy reading verbose output is one where somebody wants to know which
-      log?.(`  $ ${command}  ${Date.now() - started}ms exit ${result.code}`);
+      say?.(`  $ ${command}  ${Date.now() - started}ms exit ${result.code}`);
       return result;
     },
 
@@ -211,7 +212,7 @@ function asLog(say: (message: string) => void): Log {
 }
 
 async function plan(environment: string, configPath?: string, read: string[] = []) {
-  const config = await loadConfig(configPath);
+  const config = withEnvironment(await loadConfig(configPath), environment);
   const topology = topologyFor(config, environment);
 
   const say = (message = "") => process.stdout.write(`${message}\n`);
@@ -549,7 +550,9 @@ async function run(
   configPath: string | undefined,
   options: RunOptions,
 ) {
-  const config = await loadConfig(configPath);
+  // Folded in before anything opens, so an environment naming an app that is
+  // not there is refused before a connection or a vault
+  const config = withEnvironment(await loadConfig(configPath), environment);
 
   // Fail on a missing environment before opening anything for it
   const topology = topologyFor(config, environment);
@@ -566,10 +569,37 @@ async function run(
   // failure path, and the finally below removes the scratch directory
   const stopping = new AbortController();
 
-  const log = createLog({
-    ...options,
-    onQuit: () => stop(),
-  });
+  // Everything the run says still reaches the view. The recording is what a
+  // crash log is written from, since the view keeps only each step's tail
+  const recorder = recording(
+    createLog({
+      ...options,
+      onQuit: () => stop(),
+    }),
+  );
+
+  const log = recorder.log;
+
+  // Before the view closes, so where the log went is among what it writes out.
+  // A log that cannot be written must not replace the failure it was recording
+  const dumped = async (outcome: string, error?: unknown) => {
+    try {
+      const path = await dumpCrash(config, recorder.transcript, {
+        project: config.project,
+        environment,
+        command: kind,
+        version: await version(),
+        argv: process.argv.slice(2),
+        outcome,
+        at: Date.now(),
+        error,
+      });
+
+      if (path) log.warn(`Crash logs written to ${path}`);
+    } catch (failure) {
+      log.warn(`Could not write the crash log: ${describeFailure(failure).split("\n")[0]}`);
+    }
+  };
 
   // Read by the wait below, so a press during it hardens what that is sending
   let hardest: "TERM" | "KILL" = "TERM";
@@ -600,7 +630,7 @@ async function run(
   try {
     const meter = measured(
       await hostFor(deployHost, log, stopping.signal),
-      options.verbose ? log : undefined,
+      options.verbose ? log : recorder.command,
     );
 
     host = meter.host;
@@ -631,15 +661,17 @@ async function run(
 
     log.fail(`Reverted ${result.reverted.join(", ")}`);
     process.exitCode = 1;
+    await dumped("reverted");
   } catch (error) {
     // Whatever the command in flight said about being killed is noise: what
-    // happened is that somebody asked for it to stop
+    // happened is that somebody asked for it to stop, which is not a crash
     if (stopping.signal.aborted) {
       log.fail("Stopped");
       process.exitCode = 130;
     } else {
       log.fail(describeFailure(error));
       process.exitCode = 1;
+      await dumped("failed", error);
     }
   } finally {
     // Nothing leaves while the build is still running. The signal a press

package/src/cli/log.ts

@@ -146,7 +146,7 @@ export function describeFailure(error: unknown) {
 
 const DEPTH = 8;
 
-function causes(error: Error) {
+export function causes(error: Error) {
   const chain: Error[] = [];
   let current: unknown = error;
 

package/src/config.ts

@@ -1,6 +1,8 @@
 import { PROXY } from "./services/proxy.js";
 import { assertSteps } from "./pipeline.js";
 import { pluginSteps } from "./plugin.js";
+import { listRefs } from "./secrets/refs.js";
+import type { AnyStep } from "./pipeline.js";
 import type { Deployment, Environment } from "./types.js";
 
 // The one selected, from the one place worth looking. A deployment that carries
@@ -10,6 +12,72 @@ export function environmentOf(config: Deployment, name: string) {
   return config.environment ?? config.environments?.[name];
 }
 
+// The selected environment's items folded into the apps they name, after each
+// app's own. Refs merge in order and the later key wins, so the deployment holds
+// what every environment shares and the environment holds what differs
+export function withEnvironment(config: Deployment, name: string): Deployment {
+  const environment = environmentOf(config, name);
+  if (!environment?.secrets && !environment?.files && !environment?.steps) return config;
+
+  const { secrets = {}, files = {}, steps, ...rest } = environment;
+  assertNamesApps(config, name, [...Object.keys(secrets), ...Object.keys(files)]);
+
+  const apps = config.apps.map((app) => {
+    const refs = secrets[app.name];
+    const paths = files[app.name];
+    if (!refs && !paths) return app;
+
+    return {
+      ...app,
+      secrets: refs ? [...listRefs(app.secrets), ...listRefs(refs)] : app.secrets,
+      files: paths ? { ...app.files, ...paths } : app.files,
+    };
+  });
+
+  const folded = { ...config, apps, steps: steps ? stepsWith(config, steps) : config.steps };
+
+  // What was folded in leaves the environment, so folding it again adds nothing
+  // and a caller that already did it can hand the result on to one that will
+  if (config.environment) return { ...folded, environment: rest };
+  return { ...folded, environments: { ...config.environments, [name]: rest } };
+}
+
+// A step at a point the deployment already fills replaces it for this
+// environment, where it stood. One only the environment has leads, the way a
+// plugin's does, so a snapshot added here still runs above the shared migration
+function stepsWith(config: Deployment, given: AnyStep[]) {
+  // Before the replacement map, which would otherwise keep the last of two and
+  // drop the other without a word
+  assertSteps(given);
+
+  const shared = config.steps ?? [];
+  const claimed = new Set(shared.map((step) => step.point));
+  const replacing = new Map(given.map((step) => [step.point, step]));
+
+  const steps = [
+    ...given.filter((step) => !claimed.has(step.point)),
+    ...shared.map((step) => replacing.get(step.point) ?? step),
+  ];
+
+  // A plugin fills points too, and one the environment lands on is the same
+  // collision defineDeployment refuses
+  assertSteps([...pluginSteps(config.plugins), ...steps]);
+  return steps;
+}
+
+// An environment file does not import the deployment, so an app name there
+// cannot be checked when it compiles. A typo would otherwise read nothing
+function assertNamesApps(config: Deployment, environment: string, names: string[]) {
+  const apps = new Set(config.apps.map((app) => app.name));
+  const unknown = names.filter((name) => !apps.has(name));
+  if (unknown.length === 0) return;
+
+  throw new Error(
+    `${environment} gives secrets to ${unknown.join(", ")}, which this deployment ` +
+      `has no app by that name for. Its apps are ${[...apps].join(", ")}`,
+  );
+}
+
 // Identity, but it pins the type so a missing health predicate fails to
 // compile. Environments are excluded rather than merely unused: each lives in
 // a redkite.<name>.config.ts of its own, and a second place to put one is a
@@ -31,6 +99,7 @@ export function defineDeployment<const T extends Deployment & { environments?: n
 // Identity, for the environment a redkite.<name>.config.ts holds. It pins the
 // type the same way defineDeployment does, so a missing subnet fails to compile
 export function defineEnvironment<const T extends Environment>(environment: T): T {
+  if (environment.steps) assertSteps(environment.steps);
   return environment;
 }
 

package/src/deploy.ts

@@ -1,6 +1,6 @@
-import { build, type BuildContext, type BuildResult } from "./build.js";
+import { build, refOf, sourceOf, type BuildContext, type BuildResult } from "./build.js";
 import { assertCheckable, runChecks } from "./checks.js";
-import { environmentOf } from "./config.js";
+import { environmentOf, withEnvironment } from "./config.js";
 import { Docker } from "./docker.js";
 import { envFileFor } from "./environment.js";
 import { healthcheck, type HealthDeps } from "./health.js";
@@ -26,6 +26,7 @@ import { pluginSteps } from "./plugin.js";
 import { readEnv, readRef, type SecretStores } from "./secrets/refs.js";
 import { ensureService } from "./services/ensure.js";
 import { plannedServices } from "./services/planned.js";
+import type { Source } from "./source.js";
 import { topologyFor, type AppTopology, type Topology } from "./topology.js";
 import type { AppSpec, Deployment } from "./types.js";
 
@@ -68,11 +69,12 @@ export async function verify(options: DeployOptions): Promise<Finished> {
 
 async function start(run: Run, options: DeployOptions): Promise<Finished> {
   const log = options.log ?? silent;
+  const config = withEnvironment(options.config, options.environment);
 
   const setting: Omit<Context, "task"> = {
-    config: options.config,
+    config,
     environment: options.environment,
-    topology: topologyFor(options.config, options.environment),
+    topology: topologyFor(config, options.environment),
     host: options.host,
     docker: new Docker(options.host),
     secrets: options.secrets,
@@ -82,7 +84,7 @@ async function start(run: Run, options: DeployOptions): Promise<Finished> {
 
   // A plugin's steps lead, so a snapshot listed as a plugin runs above the
   // migration the deployment writes after it
-  const added = [...pluginSteps(options.config.plugins), ...(options.config.steps ?? [])];
+  const added = [...pluginSteps(config.plugins), ...(config.steps ?? [])];
 
   return await runPipeline(run, merge(supplied(options), added), setting, options.signal);
 }
@@ -214,8 +216,14 @@ async function release(
     // Inside, because the swap is not over until this says so. A stop lands
     // here more often than anywhere else: it is the longest part, and by now
     // every address has already moved
-    if (!(await checkAll(context, health))) {
+    const unhealthy = await checkAll(context, health);
+
+    if (unhealthy.size > 0) {
       context.log.fail("Health checks failed, reverting");
+
+      // Before the revert, which gives the live name back to the container it
+      // retired. Asked after it, the logs would be the previous release's
+      await dumpLogs(context, apps, unhealthy);
       await Promise.all(apps.map((app) => revert(docker, topology, app)));
 
       return {
@@ -306,11 +314,46 @@ async function checkAll(context: Context, health: Omit<HealthDeps, "probe">) {
         log,
       };
 
-      return await healthcheck(target.container, target.port, app.health, deps);
+      const healthy = await healthcheck(target.container, target.port, app.health, deps);
+      return healthy ? undefined : target.container;
     }),
   );
 
-  return !results.includes(false);
+  // The containers that failed, since what gets written out for each depends on
+  // whether it was one of them
+  return new Set(results.filter((container): container is string => container !== undefined));
+}
+
+// Enough to hold a stack trace and what led up to it, not a day of access logs
+const LOG_TAIL = 200;
+
+// Every new container's, not only the ones that failed: a backend that never came
+// up is often explained by what the frontend says it could not reach. Each is a
+// step of its own, so the crash log gives it a file, and the one that failed its
+// check is marked failed, which is what puts its tail on the screen at the end
+async function dumpLogs(context: Context, apps: AppTopology[], unhealthy: Set<string>) {
+  await Promise.all(
+    apps.map(async (app) => {
+      const task = context.log.step(`Logs of ${app.name}`);
+      task.detail(`the last ${LOG_TAIL} lines of ${app.container}`);
+
+      const result = await context.docker.container.logs(app.container, LOG_TAIL);
+
+      // Said, and left there. The revert still has to run, and logs that could
+      // not be read are no reason to leave a failed release serving
+      if (result.code !== 0) {
+        task.fail(`could not read the logs of ${app.container}: ${result.stdout || result.stderr}`);
+        return;
+      }
+
+      const lines = result.stdout.split("\n").filter((line) => line.length > 0);
+      for (const line of lines) task.line(line);
+
+      const said = `${lines.length} lines from ${app.container}`;
+      if (unhealthy.has(app.container)) task.fail(`${said}, which failed its health check`);
+      else task.done(said);
+    }),
+  );
 }
 
 // The proxy is one of these, derived rather than listed. What each should be
@@ -346,6 +389,8 @@ async function buildAll(
 
   return await Promise.all(
     config.apps.map(async (app) => {
+      const placed = appOf(topology, app.name);
+      const source = await clone(app, placed, here ?? context.host, environment.branch, log);
       const task = log.step(`Building ${app.name}`);
 
       try {
@@ -373,7 +418,7 @@ async function buildAll(
           output: task.line,
         };
 
-        const result = await build(app, appOf(topology, app.name), buildContext);
+        const result = await build(app, placed, buildContext, source);
         task.done(`${result.release.slice(0, 7)}${result.cached ? " (held)" : ""}`);
 
         return { app, result };
@@ -385,6 +430,41 @@ async function buildAll(
   );
 }
 
+// A step of its own, ahead of the build that reads it. A wrong branch or an
+// unreachable repository shows here, and folded into the build it was a detail
+// that scrolled past without ever saying what it fetched
+async function clone(
+  app: AppSpec,
+  placed: AppTopology,
+  host: Host,
+  branch: string,
+  log: Log,
+): Promise<Source | undefined> {
+  // A directory is read where it is, not cloned, and the build still does that
+  if (!app.repo) return undefined;
+
+  const ref = refOf(app, branch);
+  const what = `${app.repo}, ${ref.kind} ${ref.name}`;
+  const task = log.step(`Cloning ${app.name}`);
+
+  try {
+    task.detail(what);
+    const source = await sourceOf(app, placed, {
+      host,
+      branch,
+      detail: task.detail,
+      output: task.line,
+    });
+
+    // Said again at the end, where it outlasts the details that replaced it
+    task.done(`${what} at ${source.release.slice(0, 7)}`);
+    return source;
+  } catch (error) {
+    task.fail(`${app.name} could not clone ${what}`);
+    throw error;
+  }
+}
+
 async function resolveFiles(app: AppSpec, stores: SecretStores) {
   const entries = await Promise.all(
     Object.entries(app.files ?? {}).map(

package/src/docker.ts

@@ -233,6 +233,12 @@ class DockerContainer {
     return RUNNING.has(await this.status(name));
   }
 
+  // Both streams, since a process that dies as it starts says why on stderr, and
+  // stamped so a line can be set against when the health check gave up
+  async logs(name: string, tail: number) {
+    return await this.docker.run(`container logs --tail ${tail} --timestamps ${name} 2>&1`);
+  }
+
   async start(name: string) {
     if (!(await this.exists(name))) {
       throw new Error(`Cannot start ${name}, it does not exist`);

package/src/steps.ts

@@ -1,4 +1,3 @@
-import { environmentOf } from "./config.js";
 import { envFlags } from "./environment.js";
 import type { Built, Plan, Step } from "./pipeline.js";
 import type { Topology } from "./topology.js";
@@ -30,10 +29,6 @@ type MigrateOptions = {
   // already reach whatever the app's environment file points at. A database
   // this deployment runs as a service is on "deployment" instead
   network?: StepNetwork;
-  // The machine this migration reaches its database through. The step runs on
-  // the deploy host, so a deployment that moved to another one is refused
-  // rather than pointed at a database it may not be able to see
-  through?: string;
 };
 
 // An ordinary step at an ordinary point. Hung before the swap, it runs while
@@ -43,7 +38,7 @@ export function migrate(options: MigrateOptions): Step<`swap:before:${string}`>
 
   return {
     point: `swap:before:migrate-${options.app}`,
-    check: (plan) => assertReachable(plan, options),
+    check: (plan) => assertApp(plan, options),
 
     run: async (input, context) => {
       const image = builderOf(input, options.app);
@@ -73,19 +68,9 @@ export function migrate(options: MigrateOptions): Step<`swap:before:${string}`>
 
 // Checked before the run starts, so a config naming an app that never kept a
 // builder fails without having touched the host
-function assertReachable(plan: Plan, options: MigrateOptions) {
-  const app = plan.config.apps.find((item) => item.name === options.app);
-  if (!app) throw new Error(`${options.app} names no app in this deployment`);
-
-  const bastion = environmentOf(plan.config, plan.environment)?.host?.bastion;
-  const expected = options.through;
-  if (!expected || expected === bastion) return;
-
-  throw new Error(
-    `${options.app} migrates through ${expected}, ` +
-      `but ${plan.environment} deploys to ${bastion ?? "this machine"}. ` +
-      "The step runs on the deploy host, so they have to be the same",
-  );
+function assertApp(plan: Plan, options: MigrateOptions) {
+  if (plan.config.apps.some((item) => item.name === options.app)) return;
+  throw new Error(`${options.app} names no app in this deployment`);
 }
 
 // A deployment may replace the build step, and a command hung on the pipeline

package/src/types.ts

@@ -26,6 +26,16 @@ export type Environment = {
   // derives. For a database or a legacy service that has no DNS the apps can
   // use, and whose address is what differs between environments
   extraHosts?: Record<string, string>;
+  // App name to the items this environment reads, after the app's own. Which
+  // item is usually what differs between staging and production, and the rest
+  // of what differs already lives here
+  secrets?: Record<string, SecretRefs>;
+  // App name to container path to item, laid over the app's own files
+  files?: Record<string, Record<string, SecretRef>>;
+  // Steps for this environment alone. One at a point the deployment already
+  // fills replaces it here, where it stood. One only this environment has runs
+  // ahead of the deployment's, so a snapshot lands above the migration
+  steps?: AnyStep[];
 };
 
 // Where a step's container is attached. "host" is the deploy host's own stack,
@@ -272,4 +282,13 @@ export type Deployment = {
   // refs, and whatever else brings steps of its own. Nothing a plugin carries
   // happens until it is listed here, redkite's own vault included
   plugins?: Plugin[];
+  // How redkite itself behaves, rather than anything it deploys
+  options?: DeploymentOptions;
+};
+
+export type DeploymentOptions = {
+  // A run that fails writes everything it said to
+  // /tmp/<project>/<environment>/crash-<time>/: the run's own log, and one file
+  // per step with its output in full. On unless this says false
+  crashLog?: boolean;
 };