redkite 0.1.11 → 0.1.13

package/src/cli/agent.ts

@@ -21,14 +21,14 @@ function overSsh(repo: string) {
   return !/^(https?|git|file):\/\//.test(repo);
 }
 
-// A plain writer rather than a Log, because this runs before the view opens:
-// ssh-add may ask for a passphrase, and it cannot ask through a screen
-// something else is drawing
-export function requireAgent(warn: (message: string) => void) {
-  const existing = process.env["SSH_AUTH_SOCK"];
-  if (existing) return existing;
+export const AGENT_STARTED = "no SSH agent was running, so one was started";
 
-  warn("No SSH agent, starting one");
+// Runs before the view opens, because ssh-add may ask for a passphrase and it
+// cannot ask through a screen something else is drawing. Whether one was started
+// is answered rather than said, so the step that opens the connection says it
+export function requireAgent() {
+  const existing = process.env["SSH_AUTH_SOCK"];
+  if (existing) return { socket: existing, started: false };
 
   const output = execFileSync("ssh-agent", ["-s"], { encoding: "utf8" });
   const socket = output.match(/SSH_AUTH_SOCK=([^;]+);/)?.[1];
@@ -51,5 +51,5 @@ export function requireAgent(warn: (message: string) => void) {
     );
   }
 
-  return socket;
+  return { socket, started: true };
 }

package/src/cli/crash.ts

@@ -0,0 +1,287 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
+import { untagged, type Task } from "../log.js";
+import type { Deployment } from "../types.js";
+import { elapsed } from "./screen.js";
+import { causes } from "./log.js";
+import type { Viewer } from "./viewer.js";
+
+// What a failed run said, kept whole. The view trims a step to its last lines
+// and the alternate screen takes the rest with it, so the only complete record
+// of a run that went wrong is one written down while it happened.
+
+// Named rather than read from TMPDIR: the path is the one a person is told to
+// look in, and a runner that points TMPDIR somewhere else would hide it
+const ROOT = "/tmp";
+
+type Kind = "info" | "warn" | "fail" | "done" | "plan" | "step" | "command";
+
+type Entry = { at: number; kind: Kind; text: string };
+
+type Said = { at: number; kind: "detail" | "line"; text: string };
+
+type StepRecord = {
+  label: string;
+  started: number;
+  ended?: number;
+  state: "running" | "done" | "failed";
+  note?: string;
+  said: Said[];
+};
+
+export type Transcript = {
+  started: number;
+  entries: Entry[];
+  steps: StepRecord[];
+};
+
+export type Crash = {
+  project: string;
+  environment: string;
+  command: string;
+  version: string;
+  argv: string[];
+  outcome: string;
+  at: number;
+  error?: unknown;
+};
+
+// Every call reaches the wrapped log unchanged, so the view behaves as it did.
+// The transcript is a copy kept beside it, never a replacement for it
+export function recording(log: Viewer, now: () => number = Date.now) {
+  const transcript: Transcript = { started: now(), entries: [], steps: [] };
+  const note = (kind: Kind, text: string) => transcript.entries.push({ at: now(), kind, text });
+
+  const step = (label: string): Task => {
+    const record: StepRecord = { label, started: now(), state: "running", said: [] };
+    const task = log.step(label);
+
+    transcript.steps.push(record);
+    note("step", `${label} started`);
+
+    const settle = (state: "done" | "failed", message?: string) => {
+      record.state = state;
+      record.ended = now();
+      record.note = message;
+
+      const said = message ? `: ${message}` : "";
+      note("step", `${label} ${state}${said} (${elapsed(record.ended, record.started)})`);
+    };
+
+    return {
+      detail: (message) => {
+        record.said.push({ at: now(), kind: "detail", text: message });
+        task.detail(message);
+      },
+      line: (message) => {
+        record.said.push({ at: now(), kind: "line", text: message });
+        task.line(message);
+      },
+      done: (message) => {
+        settle("done", message);
+        task.done(message);
+      },
+      fail: (message) => {
+        settle("failed", message);
+        task.fail(message);
+      },
+    };
+  };
+
+  const recorded: Viewer = Object.assign(
+    (message: string) => {
+      note("info", message);
+      log(message);
+    },
+    {
+      plan: (points: string[]) => {
+        note("plan", points.join(", "));
+        log.plan?.(points);
+      },
+      warn: (message: string) => {
+        note("warn", message);
+        log.warn(message);
+      },
+      fail: (message: string) => {
+        note("fail", message);
+        log.fail(message);
+      },
+      done: (message: string) => {
+        note("done", message);
+        log.done(message);
+      },
+      step,
+      close: () => log.close(),
+    },
+  );
+
+  // A host command is only shown with --verbose, and is exactly what a crash log
+  // is read for, so it is kept whether it was shown or not
+  const command = (text: string) => note("command", text);
+
+  return { log: recorded, transcript, command };
+}
+
+// One directory per failed run, under the environment it failed in. Named from
+// the command line, so both segments are made safe to be a path first
+export function crashDirectory(project: string, environment: string, at: Date, root = ROOT) {
+  const moment = at.toISOString().replaceAll(":", "-");
+  return join(root, slug(project), slug(environment), `crash-${moment}`);
+}
+
+// The run's own log, and one file per step. A build's thousands of lines would
+// bury everything else in a single file, and a step's file is what gets opened
+// first. Numbered in the order they started, which also keeps two steps that
+// share a label from sharing a file
+export function crashFiles(transcript: Transcript, crash: Crash): Record<string, string> {
+  const width = Math.max(2, String(transcript.steps.length).length);
+
+  const named = transcript.steps.map((step, index) => ({
+    step,
+    file: `${pad(index + 1, width)}-${slug(step.label)}.log`,
+  }));
+
+  return {
+    "run.log": runLog(transcript, crash, named),
+    ...Object.fromEntries(
+      named.map(({ step, file }) => [file, stepLog(transcript, crash, step)]),
+    ),
+  };
+}
+
+// Nothing is written for a deployment that turned it off. Otherwise private to
+// whoever ran it: /tmp is shared by everyone on the machine, and a build's output
+// can say more than it meant to
+export async function dumpCrash(
+  config: Deployment,
+  transcript: Transcript,
+  crash: Crash,
+  root = ROOT,
+) {
+  if (config.options?.crashLog === false) return undefined;
+
+  const directory = crashDirectory(config.project, crash.environment, new Date(crash.at), root);
+  await mkdir(dirname(directory), { recursive: true, mode: 0o700 });
+
+  // Not recursive: a directory already there belongs to another run, and nothing
+  // of this one is written into it
+  await mkdir(directory, { mode: 0o700 });
+
+  for (const [name, contents] of Object.entries(crashFiles(transcript, crash))) {
+    await writeFile(join(directory, name), contents, { mode: 0o600, flag: "wx" });
+  }
+
+  return directory;
+}
+
+const MARK: Record<Kind, string> = {
+  info: "",
+  warn: "warn  ",
+  fail: "fail  ",
+  done: "done  ",
+  plan: "plan  ",
+  step: "step  ",
+  command: "",
+};
+
+type Named = { step: StepRecord; file: string };
+
+function runLog(transcript: Transcript, crash: Crash, named: Named[]) {
+  const since = sinceStart(transcript);
+
+  const index = named.map(({ step, file }) => {
+    const took = elapsed(step.ended ?? crash.at, step.started);
+    return `${file.padEnd(40)} ${step.state.padEnd(8)} ${took}`;
+  });
+
+  return lines([
+    "redkite crash log",
+    "",
+    `project      ${crash.project}`,
+    `environment  ${crash.environment}`,
+    `command      ${crash.command}`,
+    `outcome      ${crash.outcome}`,
+    `version      ${crash.version}`,
+    `argv         ${crash.argv.join(" ")}`,
+    `started      ${new Date(transcript.started).toISOString()}`,
+    `ended        ${new Date(crash.at).toISOString()}`,
+    "",
+    "== error",
+    ...errorLines(crash.error),
+    "",
+    "== steps, each in its own file beside this one",
+    ...(index.length > 0 ? index : ["none started"]),
+    "",
+    "== log",
+    ...transcript.entries.map(
+      (entry) => `${since(entry.at)}  ${MARK[entry.kind]}${indent(entry.text)}`,
+    ),
+  ]);
+}
+
+// Stamped from the start of the run, the same as run.log, so a line here can be
+// found among what the run said around it
+function stepLog(transcript: Transcript, crash: Crash, step: StepRecord) {
+  const since = sinceStart(transcript);
+  const note = step.note ? `: ${step.note}` : "";
+
+  return lines([
+    `step         ${step.label}`,
+    `state        ${step.state}${indent(note)}`,
+    `started      ${new Date(step.started).toISOString()}`,
+    `ended        ${step.ended ? new Date(step.ended).toISOString() : "still running when the run ended"}`,
+    `took         ${elapsed(step.ended ?? crash.at, step.started)}`,
+    "",
+    ...step.said.map(
+      (said) => `${since(said.at)}  ${said.kind === "detail" ? "· " : "| "}${indent(said.text)}`,
+    ),
+  ]);
+}
+
+// Whole, where the terminal gets the tail: the stack of every error in the
+// chain, since the step that threw is usually two causes down
+function errorLines(error: unknown) {
+  if (error === undefined) return ["none thrown"];
+  if (!(error instanceof Error)) return [String(error)];
+
+  return causes(error).flatMap((link, index) => [
+    ...(index > 0 ? ["", "caused by"] : []),
+    link.stack ?? link.message,
+  ]);
+}
+
+function sinceStart(transcript: Transcript) {
+  return (at: number) => `+${clock(at - transcript.started)}`;
+}
+
+function lines(rows: string[]) {
+  return `${rows.join("\n")}\n`;
+}
+
+// Continuation lines line up under the text rather than under the timestamp. A
+// file cannot draw a tag, so it gets the bracketed word
+function indent(text: string) {
+  return untagged(text).replaceAll("\n", "\n              ");
+}
+
+function slug(text: string) {
+  const slugged = text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+
+  return slugged || "unnamed";
+}
+
+function clock(ms: number) {
+  const minutes = Math.floor(ms / 60_000);
+  const seconds = Math.floor(ms / 1000) % 60;
+  const millis = ms % 1000;
+
+  return `${pad(minutes, 2)}:${pad(seconds, 2)}.${pad(millis, 3)}`;
+}
+
+function pad(value: number, width: number) {
+  return String(value).padStart(width, "0");
+}

package/src/cli/index.ts

@@ -14,9 +14,10 @@ import { sshHost } from "../sshHost.js";
 import { topologyFor, type Topology } from "../topology.js";
 import type { Deployment, DeployHost } from "../types.js";
 
-import { needsAgent, requireAgent } from "./agent.js";
+import { AGENT_STARTED, needsAgent, requireAgent } from "./agent.js";
 import { loadConfig, projectRoot } from "./config.js";
 import { loadDotenv } from "./dotenv.js";
+import { dumpCrash, recording } from "./crash.js";
 import { createLog, describeFailure } from "./log.js";
 
 // One command that reads the config and does everything under it: the agent,
@@ -42,7 +43,7 @@ of the project, and everything below it is derived.
 
 // Wraps a host rather than living inside one, so both implementations are
 // measured the same way and neither knows it is being timed
-function measured(host: Host, log?: Log) {
+function measured(host: Host, say?: (line: string) => void) {
   const totals = { commands: 0, commandMs: 0, files: 0 };
 
   const wrapped: Host = {
@@ -61,7 +62,7 @@ function measured(host: Host, log?: Log) {
 
       // The exit code matters: several of these are allowed to fail, and a
       // deploy reading verbose output is one where somebody wants to know which
-      log?.(`  $ ${command}  ${Date.now() - started}ms exit ${result.code}`);
+      say?.(`  $ ${command}  ${Date.now() - started}ms exit ${result.code}`);
       return result;
     },
 
@@ -180,7 +181,9 @@ async function recover(
   const deployHost = environmentOf(config, environment)?.host;
 
   const say = (message = "") => process.stdout.write(`${message}\n`);
-  if (needsAgent(config, environment)) requireAgent(say);
+  // Printed rather than drawn: a recovery has no view, and its connection
+  // step is silent
+  if (needsAgent(config, environment) && requireAgent().started) say(AGENT_STARTED);
 
   const host = await hostFor(deployHost, silent);
 
@@ -484,14 +487,26 @@ export function buildingHere(config: Deployment, environment: string): Deploymen
 
 // Without a bastion the containers are on this machine, and every command the
 // deploy issues is one this process can run itself
-async function hostFor(host: DeployHost | undefined, log: Log, signal?: AbortSignal) {
-  if (!host?.bastion) return await localHost({ signal });
+async function hostFor(
+  host: DeployHost | undefined,
+  log: Log,
+  signal?: AbortSignal,
+  startedAgent = false,
+) {
+  if (!host?.bastion) {
+    // No connection to say it in. A run on this machine that clones over ssh
+    // still started one, and that is still worth knowing
+    if (startedAgent) log.warn(AGENT_STARTED);
+    return await localHost({ signal });
+  }
 
   const task = log.step(`Opening a connection to ${host.bastion}`);
+  if (startedAgent) task.detail(AGENT_STARTED);
 
   try {
     const opened = await sshHost(host.bastion, { signal, hostKeys: host.hostKeys });
-    task.done();
+    // Kept on the row once the connection is open, not only while it opens
+    task.done(startedAgent ? AGENT_STARTED : undefined);
     return opened;
   } catch (error) {
     task.fail(`Could not reach ${host.bastion}`);
@@ -560,18 +575,43 @@ async function run(
   // The host clones the repositories over this, so it has to exist before the
   // connection that forwards it is opened. Before the view too: ssh-add asks
   // for a passphrase on the terminal, and by then the view owns it
-  if (needsAgent(config, environment)) {
-    requireAgent((message) => process.stderr.write(`${message}\n`));
-  }
+  const agent = needsAgent(config, environment) ? requireAgent() : undefined;
 
   // Whatever is in flight is killed, the pipeline unwinds through its own
   // failure path, and the finally below removes the scratch directory
   const stopping = new AbortController();
 
-  const log = createLog({
-    ...options,
-    onQuit: () => stop(),
-  });
+  // Everything the run says still reaches the view. The recording is what a
+  // crash log is written from, since the view keeps only each step's tail
+  const recorder = recording(
+    createLog({
+      ...options,
+      onQuit: () => stop(),
+    }),
+  );
+
+  const log = recorder.log;
+
+  // Before the view closes, so where the log went is among what it writes out.
+  // A log that cannot be written must not replace the failure it was recording
+  const dumped = async (outcome: string, error?: unknown) => {
+    try {
+      const path = await dumpCrash(config, recorder.transcript, {
+        project: config.project,
+        environment,
+        command: kind,
+        version: await version(),
+        argv: process.argv.slice(2),
+        outcome,
+        at: Date.now(),
+        error,
+      });
+
+      if (path) log.warn(`Crash logs written to ${path}`);
+    } catch (failure) {
+      log.warn(`Could not write the crash log: ${describeFailure(failure).split("\n")[0]}`);
+    }
+  };
 
   // Read by the wait below, so a press during it hardens what that is sending
   let hardest: "TERM" | "KILL" = "TERM";
@@ -601,8 +641,8 @@ async function run(
 
   try {
     const meter = measured(
-      await hostFor(deployHost, log, stopping.signal),
-      options.verbose ? log : undefined,
+      await hostFor(deployHost, log, stopping.signal, agent?.started),
+      options.verbose ? log : recorder.command,
     );
 
     host = meter.host;
@@ -633,15 +673,17 @@ async function run(
 
     log.fail(`Reverted ${result.reverted.join(", ")}`);
     process.exitCode = 1;
+    await dumped("reverted");
   } catch (error) {
     // Whatever the command in flight said about being killed is noise: what
-    // happened is that somebody asked for it to stop
+    // happened is that somebody asked for it to stop, which is not a crash
     if (stopping.signal.aborted) {
       log.fail("Stopped");
       process.exitCode = 130;
     } else {
       log.fail(describeFailure(error));
       process.exitCode = 1;
+      await dumped("failed", error);
     }
   } finally {
     // Nothing leaves while the build is still running. The signal a press

package/src/cli/log.ts

@@ -1,6 +1,6 @@
 import type { Task } from "../log.js";
 
-import { elapsed as took } from "./screen.js";
+import { paintTags, elapsed as took } from "./screen.js";
 import { createViewer, type Viewer } from "./viewer.js";
 
 // What a deploy prints. The engine's own trace is a build graph with span ids
@@ -48,7 +48,8 @@ function plainLog(lines: boolean) {
   const write = (stream: NodeJS.WriteStream, message: string, colour?: number) => {
     const painted = colours(stream);
     const stamp = painted ? paint(elapsed(), STAMP) : elapsed();
-    const body = painted && colour ? paint(message, colour) : message;
+    // A tag is drawn where there is colour and bracketed where there is not
+    const body = paintTags(message, painted, colour);
     const line = `${stamp} ${body}\n`;
 
     // Both streams share one terminal, so the block has to come down for a
@@ -146,7 +147,7 @@ export function describeFailure(error: unknown) {
 
 const DEPTH = 8;
 
-function causes(error: Error) {
+export function causes(error: Error) {
   const chain: Error[] = [];
   let current: unknown = error;
 

package/src/cli/screen.ts

@@ -1,3 +1,5 @@
+import { TAGS, untagged } from "../log.js";
+
 // The deploy as a list of collapsibles. A step is a title and the lines it
 // produced; the one running is open, the ones finished are shut, and the person
 // watching moves between them. The model and the renderer are pure, so what the
@@ -199,14 +201,53 @@ const RUNNING = 93;
 const DONE = 32;
 const FAILED = 91;
 const WARN = 33;
+// A tag is a blue label with rounded ends: the ends in blue on the terminal's own
+// ground, and the word white on blue between them
+const TAG_EDGE = 34;
+const TAG_BODY = "97;44";
+
+type Colour = number | string;
 
-type Piece = { text: string; colour?: number };
+type Piece = { text: string; colour?: Colour };
 
-function paint(text: string, colour: number | undefined, on: boolean) {
+function paint(text: string, colour: Colour | undefined, on: boolean) {
   if (!on || colour === undefined || text === "") return text;
   return `\u001b[${colour}m${text}\u001b[0m`;
 }
 
+// Text that may carry a tag, as pieces, so the row is measured before any escape
+// is added. The rounded ends are a column each, the same as the brackets drawn
+// in their place without colour, so a row is the same width either way
+function tagged(text: string, colour: Colour | undefined, on: boolean): Piece[] {
+  if (!on) return [{ text: untagged(text), colour }];
+
+  const pieces: Piece[] = [];
+  let at = 0;
+
+  for (const match of text.matchAll(TAGS)) {
+    if (match.index > at) pieces.push({ text: text.slice(at, match.index), colour });
+
+    pieces.push(
+      { text: "\u25d6", colour: TAG_EDGE },
+      { text: match[1] ?? "", colour: TAG_BODY },
+      { text: "\u25d7", colour: TAG_EDGE },
+    );
+
+    at = match.index + match[0].length;
+  }
+
+  if (at < text.length) pieces.push({ text: text.slice(at), colour });
+  return pieces;
+}
+
+// For what is written as a whole line rather than drawn into a row: the summary
+// left on the screen, and the plain log. The line's own colour resumes after
+export function paintTags(text: string, on: boolean, colour?: Colour) {
+  return tagged(text, colour, on)
+    .map((piece) => paint(piece.text, piece.colour, on))
+    .join("");
+}
+
 function widthOf(pieces: Piece[]) {
   return pieces.reduce((total, piece) => total + piece.text.length, 0);
 }
@@ -362,7 +403,7 @@ function said(model: Model, message: Message, colour: boolean) {
     [
       { text: gutter(model, message.at), colour: DIM },
       { text: "   " },
-      { text: message.text, colour: WARN },
+      ...tagged(message.text, WARN, colour),
     ],
     Math.max(24, model.columns),
     colour,
@@ -485,7 +526,7 @@ function title(model: Model, step: Step, index: number, colour: boolean) {
     { text: glyph, colour: stateColour(step) },
     { text: " " },
     { text: step.label, colour: labelColour(step, selected) },
-    { text: said, colour: DIM },
+    ...tagged(said, DIM, colour),
     { text: quiet(model, step), colour: WARN },
   ];
 

package/src/cli/viewer.ts

@@ -5,6 +5,7 @@ import {
   elapsed,
   emptyModel,
   keysOf,
+  paintTags,
   render,
   type Model,
   type Step,
@@ -113,7 +114,7 @@ export function createViewer(
     stream.write(ALTERNATE_OFF);
     // The alternate screen takes every frame with it, so the run has to be
     // written again on the screen the person keeps
-    for (const line of summary(model)) stream.write(`${line}\n`);
+    for (const line of summary(model, colour)) stream.write(`${line}\n`);
   };
 
   // What a second press means is not the view's to decide: the same key
@@ -191,18 +192,18 @@ export function createViewer(
 
 // One line per step and whatever was said outside them, which is the record a
 // person scrolls back to after the deploy is over
-function summary(model: Model): string[] {
+function summary(model: Model, colour: boolean): string[] {
   const rows = model.steps.map((step) => {
     const glyph = step.state === "done" ? "✔" : step.state === "failed" ? "✘" : "·";
     const said = step.note ? `: ${step.note}` : "";
     const took = elapsed(step.ended ?? model.now, step.started);
 
-    return `${glyph} ${step.label}${said} (${took})`;
+    return paintTags(`${glyph} ${step.label}${said} (${took})`, colour);
   });
 
   return [
     ...rows,
-    ...model.messages.map((message) => message.text),
+    ...model.messages.map((message) => paintTags(message.text, colour)),
     ...whatFailed(model),
   ];
 }