redkite 0.1.11 → 0.1.12

package/src/cli/crash.ts

@@ -0,0 +1,286 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
+import type { Task } from "../log.js";
+import type { Deployment } from "../types.js";
+import { elapsed } from "./screen.js";
+import { causes } from "./log.js";
+import type { Viewer } from "./viewer.js";
+
+// What a failed run said, kept whole. The view trims a step to its last lines
+// and the alternate screen takes the rest with it, so the only complete record
+// of a run that went wrong is one written down while it happened.
+
+// Named rather than read from TMPDIR: the path is the one a person is told to
+// look in, and a runner that points TMPDIR somewhere else would hide it
+const ROOT = "/tmp";
+
+type Kind = "info" | "warn" | "fail" | "done" | "plan" | "step" | "command";
+
+type Entry = { at: number; kind: Kind; text: string };
+
+type Said = { at: number; kind: "detail" | "line"; text: string };
+
+type StepRecord = {
+  label: string;
+  started: number;
+  ended?: number;
+  state: "running" | "done" | "failed";
+  note?: string;
+  said: Said[];
+};
+
+export type Transcript = {
+  started: number;
+  entries: Entry[];
+  steps: StepRecord[];
+};
+
+export type Crash = {
+  project: string;
+  environment: string;
+  command: string;
+  version: string;
+  argv: string[];
+  outcome: string;
+  at: number;
+  error?: unknown;
+};
+
+// Every call reaches the wrapped log unchanged, so the view behaves as it did.
+// The transcript is a copy kept beside it, never a replacement for it
+export function recording(log: Viewer, now: () => number = Date.now) {
+  const transcript: Transcript = { started: now(), entries: [], steps: [] };
+  const note = (kind: Kind, text: string) => transcript.entries.push({ at: now(), kind, text });
+
+  const step = (label: string): Task => {
+    const record: StepRecord = { label, started: now(), state: "running", said: [] };
+    const task = log.step(label);
+
+    transcript.steps.push(record);
+    note("step", `${label} started`);
+
+    const settle = (state: "done" | "failed", message?: string) => {
+      record.state = state;
+      record.ended = now();
+      record.note = message;
+
+      const said = message ? `: ${message}` : "";
+      note("step", `${label} ${state}${said} (${elapsed(record.ended, record.started)})`);
+    };
+
+    return {
+      detail: (message) => {
+        record.said.push({ at: now(), kind: "detail", text: message });
+        task.detail(message);
+      },
+      line: (message) => {
+        record.said.push({ at: now(), kind: "line", text: message });
+        task.line(message);
+      },
+      done: (message) => {
+        settle("done", message);
+        task.done(message);
+      },
+      fail: (message) => {
+        settle("failed", message);
+        task.fail(message);
+      },
+    };
+  };
+
+  const recorded: Viewer = Object.assign(
+    (message: string) => {
+      note("info", message);
+      log(message);
+    },
+    {
+      plan: (points: string[]) => {
+        note("plan", points.join(", "));
+        log.plan?.(points);
+      },
+      warn: (message: string) => {
+        note("warn", message);
+        log.warn(message);
+      },
+      fail: (message: string) => {
+        note("fail", message);
+        log.fail(message);
+      },
+      done: (message: string) => {
+        note("done", message);
+        log.done(message);
+      },
+      step,
+      close: () => log.close(),
+    },
+  );
+
+  // A host command is only shown with --verbose, and is exactly what a crash log
+  // is read for, so it is kept whether it was shown or not
+  const command = (text: string) => note("command", text);
+
+  return { log: recorded, transcript, command };
+}
+
+// One directory per failed run, under the environment it failed in. Named from
+// the command line, so both segments are made safe to be a path first
+export function crashDirectory(project: string, environment: string, at: Date, root = ROOT) {
+  const moment = at.toISOString().replaceAll(":", "-");
+  return join(root, slug(project), slug(environment), `crash-${moment}`);
+}
+
+// The run's own log, and one file per step. A build's thousands of lines would
+// bury everything else in a single file, and a step's file is what gets opened
+// first. Numbered in the order they started, which also keeps two steps that
+// share a label from sharing a file
+export function crashFiles(transcript: Transcript, crash: Crash): Record<string, string> {
+  const width = Math.max(2, String(transcript.steps.length).length);
+
+  const named = transcript.steps.map((step, index) => ({
+    step,
+    file: `${pad(index + 1, width)}-${slug(step.label)}.log`,
+  }));
+
+  return {
+    "run.log": runLog(transcript, crash, named),
+    ...Object.fromEntries(
+      named.map(({ step, file }) => [file, stepLog(transcript, crash, step)]),
+    ),
+  };
+}
+
+// Nothing is written for a deployment that turned it off. Otherwise private to
+// whoever ran it: /tmp is shared by everyone on the machine, and a build's output
+// can say more than it meant to
+export async function dumpCrash(
+  config: Deployment,
+  transcript: Transcript,
+  crash: Crash,
+  root = ROOT,
+) {
+  if (config.options?.crashLog === false) return undefined;
+
+  const directory = crashDirectory(config.project, crash.environment, new Date(crash.at), root);
+  await mkdir(dirname(directory), { recursive: true, mode: 0o700 });
+
+  // Not recursive: a directory already there belongs to another run, and nothing
+  // of this one is written into it
+  await mkdir(directory, { mode: 0o700 });
+
+  for (const [name, contents] of Object.entries(crashFiles(transcript, crash))) {
+    await writeFile(join(directory, name), contents, { mode: 0o600, flag: "wx" });
+  }
+
+  return directory;
+}
+
+const MARK: Record<Kind, string> = {
+  info: "",
+  warn: "warn  ",
+  fail: "fail  ",
+  done: "done  ",
+  plan: "plan  ",
+  step: "step  ",
+  command: "",
+};
+
+type Named = { step: StepRecord; file: string };
+
+function runLog(transcript: Transcript, crash: Crash, named: Named[]) {
+  const since = sinceStart(transcript);
+
+  const index = named.map(({ step, file }) => {
+    const took = elapsed(step.ended ?? crash.at, step.started);
+    return `${file.padEnd(40)} ${step.state.padEnd(8)} ${took}`;
+  });
+
+  return lines([
+    "redkite crash log",
+    "",
+    `project      ${crash.project}`,
+    `environment  ${crash.environment}`,
+    `command      ${crash.command}`,
+    `outcome      ${crash.outcome}`,
+    `version      ${crash.version}`,
+    `argv         ${crash.argv.join(" ")}`,
+    `started      ${new Date(transcript.started).toISOString()}`,
+    `ended        ${new Date(crash.at).toISOString()}`,
+    "",
+    "== error",
+    ...errorLines(crash.error),
+    "",
+    "== steps, each in its own file beside this one",
+    ...(index.length > 0 ? index : ["none started"]),
+    "",
+    "== log",
+    ...transcript.entries.map(
+      (entry) => `${since(entry.at)}  ${MARK[entry.kind]}${indent(entry.text)}`,
+    ),
+  ]);
+}
+
+// Stamped from the start of the run, the same as run.log, so a line here can be
+// found among what the run said around it
+function stepLog(transcript: Transcript, crash: Crash, step: StepRecord) {
+  const since = sinceStart(transcript);
+  const note = step.note ? `: ${step.note}` : "";
+
+  return lines([
+    `step         ${step.label}`,
+    `state        ${step.state}${indent(note)}`,
+    `started      ${new Date(step.started).toISOString()}`,
+    `ended        ${step.ended ? new Date(step.ended).toISOString() : "still running when the run ended"}`,
+    `took         ${elapsed(step.ended ?? crash.at, step.started)}`,
+    "",
+    ...step.said.map(
+      (said) => `${since(said.at)}  ${said.kind === "detail" ? "· " : "| "}${indent(said.text)}`,
+    ),
+  ]);
+}
+
+// Whole, where the terminal gets the tail: the stack of every error in the
+// chain, since the step that threw is usually two causes down
+function errorLines(error: unknown) {
+  if (error === undefined) return ["none thrown"];
+  if (!(error instanceof Error)) return [String(error)];
+
+  return causes(error).flatMap((link, index) => [
+    ...(index > 0 ? ["", "caused by"] : []),
+    link.stack ?? link.message,
+  ]);
+}
+
+function sinceStart(transcript: Transcript) {
+  return (at: number) => `+${clock(at - transcript.started)}`;
+}
+
+function lines(rows: string[]) {
+  return `${rows.join("\n")}\n`;
+}
+
+// Continuation lines line up under the text rather than under the timestamp
+function indent(text: string) {
+  return text.replaceAll("\n", "\n              ");
+}
+
+function slug(text: string) {
+  const slugged = text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+
+  return slugged || "unnamed";
+}
+
+function clock(ms: number) {
+  const minutes = Math.floor(ms / 60_000);
+  const seconds = Math.floor(ms / 1000) % 60;
+  const millis = ms % 1000;
+
+  return `${pad(minutes, 2)}:${pad(seconds, 2)}.${pad(millis, 3)}`;
+}
+
+function pad(value: number, width: number) {
+  return String(value).padStart(width, "0");
+}

package/src/cli/index.ts

@@ -17,6 +17,7 @@ import type { Deployment, DeployHost } from "../types.js";
 import { needsAgent, requireAgent } from "./agent.js";
 import { loadConfig, projectRoot } from "./config.js";
 import { loadDotenv } from "./dotenv.js";
+import { dumpCrash, recording } from "./crash.js";
 import { createLog, describeFailure } from "./log.js";
 
 // One command that reads the config and does everything under it: the agent,
@@ -42,7 +43,7 @@ of the project, and everything below it is derived.
 
 // Wraps a host rather than living inside one, so both implementations are
 // measured the same way and neither knows it is being timed
-function measured(host: Host, log?: Log) {
+function measured(host: Host, say?: (line: string) => void) {
   const totals = { commands: 0, commandMs: 0, files: 0 };
 
   const wrapped: Host = {
@@ -61,7 +62,7 @@ function measured(host: Host, log?: Log) {
 
       // The exit code matters: several of these are allowed to fail, and a
       // deploy reading verbose output is one where somebody wants to know which
-      log?.(`  $ ${command}  ${Date.now() - started}ms exit ${result.code}`);
+      say?.(`  $ ${command}  ${Date.now() - started}ms exit ${result.code}`);
       return result;
     },
 
@@ -568,10 +569,37 @@ async function run(
   // failure path, and the finally below removes the scratch directory
   const stopping = new AbortController();
 
-  const log = createLog({
-    ...options,
-    onQuit: () => stop(),
-  });
+  // Everything the run says still reaches the view. The recording is what a
+  // crash log is written from, since the view keeps only each step's tail
+  const recorder = recording(
+    createLog({
+      ...options,
+      onQuit: () => stop(),
+    }),
+  );
+
+  const log = recorder.log;
+
+  // Before the view closes, so where the log went is among what it writes out.
+  // A log that cannot be written must not replace the failure it was recording
+  const dumped = async (outcome: string, error?: unknown) => {
+    try {
+      const path = await dumpCrash(config, recorder.transcript, {
+        project: config.project,
+        environment,
+        command: kind,
+        version: await version(),
+        argv: process.argv.slice(2),
+        outcome,
+        at: Date.now(),
+        error,
+      });
+
+      if (path) log.warn(`Crash logs written to ${path}`);
+    } catch (failure) {
+      log.warn(`Could not write the crash log: ${describeFailure(failure).split("\n")[0]}`);
+    }
+  };
 
   // Read by the wait below, so a press during it hardens what that is sending
   let hardest: "TERM" | "KILL" = "TERM";
@@ -602,7 +630,7 @@ async function run(
   try {
     const meter = measured(
       await hostFor(deployHost, log, stopping.signal),
-      options.verbose ? log : undefined,
+      options.verbose ? log : recorder.command,
     );
 
     host = meter.host;
@@ -633,15 +661,17 @@ async function run(
 
     log.fail(`Reverted ${result.reverted.join(", ")}`);
     process.exitCode = 1;
+    await dumped("reverted");
   } catch (error) {
     // Whatever the command in flight said about being killed is noise: what
-    // happened is that somebody asked for it to stop
+    // happened is that somebody asked for it to stop, which is not a crash
     if (stopping.signal.aborted) {
       log.fail("Stopped");
       process.exitCode = 130;
     } else {
       log.fail(describeFailure(error));
       process.exitCode = 1;
+      await dumped("failed", error);
     }
   } finally {
     // Nothing leaves while the build is still running. The signal a press

package/src/cli/log.ts

@@ -146,7 +146,7 @@ export function describeFailure(error: unknown) {
 
 const DEPTH = 8;
 
-function causes(error: Error) {
+export function causes(error: Error) {
   const chain: Error[] = [];
   let current: unknown = error;
 

package/src/deploy.ts

@@ -1,4 +1,4 @@
-import { build, type BuildContext, type BuildResult } from "./build.js";
+import { build, refOf, sourceOf, type BuildContext, type BuildResult } from "./build.js";
 import { assertCheckable, runChecks } from "./checks.js";
 import { environmentOf, withEnvironment } from "./config.js";
 import { Docker } from "./docker.js";
@@ -26,6 +26,7 @@ import { pluginSteps } from "./plugin.js";
 import { readEnv, readRef, type SecretStores } from "./secrets/refs.js";
 import { ensureService } from "./services/ensure.js";
 import { plannedServices } from "./services/planned.js";
+import type { Source } from "./source.js";
 import { topologyFor, type AppTopology, type Topology } from "./topology.js";
 import type { AppSpec, Deployment } from "./types.js";
 
@@ -215,8 +216,14 @@ async function release(
     // Inside, because the swap is not over until this says so. A stop lands
     // here more often than anywhere else: it is the longest part, and by now
     // every address has already moved
-    if (!(await checkAll(context, health))) {
+    const unhealthy = await checkAll(context, health);
+
+    if (unhealthy.size > 0) {
       context.log.fail("Health checks failed, reverting");
+
+      // Before the revert, which gives the live name back to the container it
+      // retired. Asked after it, the logs would be the previous release's
+      await dumpLogs(context, apps, unhealthy);
       await Promise.all(apps.map((app) => revert(docker, topology, app)));
 
       return {
@@ -307,11 +314,46 @@ async function checkAll(context: Context, health: Omit<HealthDeps, "probe">) {
         log,
       };
 
-      return await healthcheck(target.container, target.port, app.health, deps);
+      const healthy = await healthcheck(target.container, target.port, app.health, deps);
+      return healthy ? undefined : target.container;
     }),
   );
 
-  return !results.includes(false);
+  // The containers that failed, since what gets written out for each depends on
+  // whether it was one of them
+  return new Set(results.filter((container): container is string => container !== undefined));
+}
+
+// Enough to hold a stack trace and what led up to it, not a day of access logs
+const LOG_TAIL = 200;
+
+// Every new container's, not only the ones that failed: a backend that never came
+// up is often explained by what the frontend says it could not reach. Each is a
+// step of its own, so the crash log gives it a file, and the one that failed its
+// check is marked failed, which is what puts its tail on the screen at the end
+async function dumpLogs(context: Context, apps: AppTopology[], unhealthy: Set<string>) {
+  await Promise.all(
+    apps.map(async (app) => {
+      const task = context.log.step(`Logs of ${app.name}`);
+      task.detail(`the last ${LOG_TAIL} lines of ${app.container}`);
+
+      const result = await context.docker.container.logs(app.container, LOG_TAIL);
+
+      // Said, and left there. The revert still has to run, and logs that could
+      // not be read are no reason to leave a failed release serving
+      if (result.code !== 0) {
+        task.fail(`could not read the logs of ${app.container}: ${result.stdout || result.stderr}`);
+        return;
+      }
+
+      const lines = result.stdout.split("\n").filter((line) => line.length > 0);
+      for (const line of lines) task.line(line);
+
+      const said = `${lines.length} lines from ${app.container}`;
+      if (unhealthy.has(app.container)) task.fail(`${said}, which failed its health check`);
+      else task.done(said);
+    }),
+  );
 }
 
 // The proxy is one of these, derived rather than listed. What each should be
@@ -347,6 +389,8 @@ async function buildAll(
 
   return await Promise.all(
     config.apps.map(async (app) => {
+      const placed = appOf(topology, app.name);
+      const source = await clone(app, placed, here ?? context.host, environment.branch, log);
       const task = log.step(`Building ${app.name}`);
 
       try {
@@ -374,7 +418,7 @@ async function buildAll(
           output: task.line,
         };
 
-        const result = await build(app, appOf(topology, app.name), buildContext);
+        const result = await build(app, placed, buildContext, source);
         task.done(`${result.release.slice(0, 7)}${result.cached ? " (held)" : ""}`);
 
         return { app, result };
@@ -386,6 +430,41 @@ async function buildAll(
   );
 }
 
+// A step of its own, ahead of the build that reads it. A wrong branch or an
+// unreachable repository shows here, and folded into the build it was a detail
+// that scrolled past without ever saying what it fetched
+async function clone(
+  app: AppSpec,
+  placed: AppTopology,
+  host: Host,
+  branch: string,
+  log: Log,
+): Promise<Source | undefined> {
+  // A directory is read where it is, not cloned, and the build still does that
+  if (!app.repo) return undefined;
+
+  const ref = refOf(app, branch);
+  const what = `${app.repo}, ${ref.kind} ${ref.name}`;
+  const task = log.step(`Cloning ${app.name}`);
+
+  try {
+    task.detail(what);
+    const source = await sourceOf(app, placed, {
+      host,
+      branch,
+      detail: task.detail,
+      output: task.line,
+    });
+
+    // Said again at the end, where it outlasts the details that replaced it
+    task.done(`${what} at ${source.release.slice(0, 7)}`);
+    return source;
+  } catch (error) {
+    task.fail(`${app.name} could not clone ${what}`);
+    throw error;
+  }
+}
+
 async function resolveFiles(app: AppSpec, stores: SecretStores) {
   const entries = await Promise.all(
     Object.entries(app.files ?? {}).map(

package/src/docker.ts

@@ -233,6 +233,12 @@ class DockerContainer {
     return RUNNING.has(await this.status(name));
   }
 
+  // Both streams, since a process that dies as it starts says why on stderr, and
+  // stamped so a line can be set against when the health check gave up
+  async logs(name: string, tail: number) {
+    return await this.docker.run(`container logs --tail ${tail} --timestamps ${name} 2>&1`);
+  }
+
   async start(name: string) {
     if (!(await this.exists(name))) {
       throw new Error(`Cannot start ${name}, it does not exist`);

package/src/types.ts

@@ -282,4 +282,13 @@ export type Deployment = {
   // refs, and whatever else brings steps of its own. Nothing a plugin carries
   // happens until it is listed here, redkite's own vault included
   plugins?: Plugin[];
+  // How redkite itself behaves, rather than anything it deploys
+  options?: DeploymentOptions;
+};
+
+export type DeploymentOptions = {
+  // A run that fails writes everything it said to
+  // /tmp/<project>/<environment>/crash-<time>/: the run's own log, and one file
+  // per step with its output in full. On unless this says false
+  crashLog?: boolean;
 };