redkite 0.1.9 → 0.1.11

package/src/cli/screen.ts

@@ -14,6 +14,9 @@ export type Step = {
   note?: string;
   detail?: string;
   lines: string[];
+  // When it last said anything. A step that is working and one that is wedged
+  // look identical without it, which is the whole reason for the spinner
+  spoke?: number;
   expanded: boolean;
   // Opened by hand, so finishing does not shut it under the reader
   held: boolean;
@@ -21,14 +24,24 @@ export type Step = {
   offset: number;
 };
 
+// Said outside a step, carrying the moment it was said so its row stops
+// ticking the way a finished step's does
+export type Message = { text: string; at: number };
+
 export type Model = {
   steps: Step[];
-  messages: string[];
+  messages: Message[];
   cursor: number;
   // The cursor follows the newest step until the reader moves it
   following: boolean;
   // Nothing opens on its own, including the step that is running
   minimal: boolean;
+  // Long lines wrap rather than being cut. Off by default: one row per line is
+  // what keeps a frame countable, and this is for reading an error
+  wrapped: boolean;
+  // Every point the run will walk, known before it starts. What has not begun
+  // is drawn under what has, so the end is visible from the start
+  planned: string[];
   rows: number;
   columns: number;
   started: number;
@@ -43,6 +56,7 @@ export type Key =
   | "toggle"
   | "expand"
   | "collapse"
+  | "wrap"
   | "quit";
 
 export function emptyModel(rows: number, columns: number, now: number): Model {
@@ -52,6 +66,8 @@ export function emptyModel(rows: number, columns: number, now: number): Model {
     cursor: 0,
     following: true,
     minimal: false,
+    wrapped: false,
+    planned: [],
     rows,
     columns,
     started: now,
@@ -91,6 +107,7 @@ export function keyOf(data: string): Key | undefined {
   if (data === "\r" || data === "\n") return "toggle";
   if (data === "+" || data === "=") return "expand";
   if (data === "-" || data === "_") return "collapse";
+  if (data === "w") return "wrap";
   if (data === "q" || data === INTERRUPT) return "quit";
 
   return undefined;
@@ -106,6 +123,7 @@ export function apply(model: Model, key: Key): Model {
   }
 
   if (key === "first") return { ...model, cursor: 0, following: false };
+  if (key === "wrap") return { ...model, wrapped: !model.wrapped };
 
   if (key === "up") {
     if (step?.expanded && step.offset < Math.max(0, step.lines.length - 1)) {
@@ -223,50 +241,132 @@ const FLOOR = 3;
 // the view closes, and a list that only grows crowds every log off the screen
 const MESSAGES = 6;
 
+// Every key, inside eighty columns. One too wide is a row that wraps, and a
+// wrapped footer costs the frame a line it counted on
 export const HELP =
-  "\u2191\u2193 move \u00b7 enter open \u00b7 shift+\u2191 latest \u00b7 + open \u00b7 - collapse all \u00b7 q quit";
+  "\u2191\u2193 move \u00b7 enter open \u00b7 shift+\u2191 latest \u00b7 +/- all \u00b7 w wrap \u00b7 q quit";
+
+// Turns on the frame the clock is in, so the view is a pure function of now
+const SPINNER = "\u280b\u2819\u2839\u2838\u283c\u2834\u2826\u2827\u2807\u280f";
+const SPIN_MS = 80;
+
+// How long a running step may say nothing before the frame says so. Short
+// enough to catch a wedge, long enough that an ordinary pause is not one
+const QUIET_MS = 8000;
 
 export function render(model: Model, colour = false): string[] {
   const body = Math.max(1, model.rows - FOOTER);
   const rows: string[] = [];
   const messages = model.messages.slice(-MESSAGES);
+  const budget = Math.max(1, body - messages.length);
+
+  // What the step being read keeps whatever else wants the room. A frame of
+  // titles with no output says nothing the last line of output would not, so
+  // the list is what gives way rather than the log
+  const focused = model.steps[model.cursor];
+  const floor = focused?.expanded ? Math.min(FLOOR, rowsOf(model, focused)) : 0;
+
+  // More than the terminal has room for, so the list itself scrolls and only
+  // the step under the cursor keeps a log
+  if (model.steps.length + floor > budget) {
+    const room = Math.max(1, budget - floor);
+    const start = Math.max(0, Math.min(model.cursor - room + 1, model.steps.length - room));
+
+    for (const entry of ordered(model, messages)) {
+      if (rows.length >= budget) break;
+
+      if (entry.kind === "message") {
+        rows.push(said(model, entry.message, colour));
+        continue;
+      }
 
-  // More titles than the terminal has lines, so nothing can be open and the
-  // list itself is what scrolls
-  if (model.steps.length + messages.length > body) {
-    const start = Math.max(0, Math.min(model.cursor, model.steps.length - body));
+      if (entry.index < start) continue;
 
-    for (const [index, step] of model.steps.entries()) {
-      if (index >= start && rows.length < body) {
-        rows.push(title(model, step, index, colour));
-      }
+      rows.push(title(model, entry.step, entry.index, colour));
+      if (entry.index === model.cursor) rows.push(...logs(model, entry.step, floor, colour));
     }
 
-    return finish(rows, body, colour);
+    return finish(model, rows, body, colour);
   }
 
-  const shown = share(model, body - model.steps.length - messages.length);
+  const shown = share(model, budget - model.steps.length);
 
-  for (const [index, step] of model.steps.entries()) {
-    rows.push(title(model, step, index, colour));
-    rows.push(...logs(model, step, shown.get(index) ?? 0, colour));
+  for (const entry of ordered(model, messages)) {
+    if (entry.kind === "message") {
+      rows.push(said(model, entry.message, colour));
+      continue;
+    }
+
+    rows.push(title(model, entry.step, entry.index, colour));
+    rows.push(...logs(model, entry.step, shown.get(entry.index) ?? 0, colour));
   }
 
-  for (const message of messages) {
-    rows.push(
-      clipped(
-        [
-          { text: gutter(model), colour: DIM },
-          { text: "   " },
-          { text: message, colour: WARN },
-        ],
-        Math.max(24, model.columns),
-        colour,
-      ),
-    );
+  for (const point of remaining(model)) {
+    if (rows.length >= budget) break;
+    rows.push(waiting(model, point, colour));
   }
 
-  return finish(rows, body, colour);
+  return finish(model, rows, body, colour);
+}
+
+// Every point the run will walk that has not started. Drawn under what has, so
+// how much is left is visible from the first frame rather than at the end
+function remaining(model: Model) {
+  const started = new Set(model.steps.map((step) => step.label));
+  return model.planned.filter((point) => !started.has(point));
+}
+
+function waiting(model: Model, point: string, colour: boolean) {
+  const head: Piece[] = [
+    { text: " ".repeat(gutter(model).length), colour: DIM },
+    { text: "   " },
+    { text: "\u00b7", colour: DIM },
+    { text: " " },
+    { text: point, colour: DIM },
+  ];
+
+  return fit(model, head, "", colour);
+}
+
+type Row =
+  | { kind: "step"; at: number; step: Step; index: number }
+  | { kind: "message"; at: number; message: Message };
+
+// Everything the run has said, in the order it said it. Messages used to be
+// printed after every step, which put one from the first ten seconds below a
+// build still running half an hour later
+function ordered(model: Model, messages: Message[]): Row[] {
+  const steps: Row[] = model.steps.map((step, index) => ({
+    kind: "step",
+    at: step.started,
+    step,
+    index,
+  }));
+
+  const said: Row[] = messages.map((message) => ({
+    kind: "message",
+    at: message.at,
+    message,
+  }));
+
+  // A step announced in the same millisecond as a message leads it: the
+  // message is usually about what the step then went and did. Everything else
+  // ties to nothing, and a stable sort leaves it where it was
+  const rank = (row: Row) => (row.kind === "step" ? 0 : 1);
+
+  return [...steps, ...said].sort((a, b) => a.at - b.at || rank(a) - rank(b));
+}
+
+function said(model: Model, message: Message, colour: boolean) {
+  return clipped(
+    [
+      { text: gutter(model, message.at), colour: DIM },
+      { text: "   " },
+      { text: message.text, colour: WARN },
+    ],
+    Math.max(24, model.columns),
+    colour,
+  );
 }
 
 // The focused step is the one being read, so it is served first and every other
@@ -287,19 +387,19 @@ function share(model: Model, budget: number) {
     // printed nothing reserves nothing, and served last the focused step used
     // to be left with whatever the others happened not to want
     const glances = others.reduce(
-      (total, item) => total + Math.min(GLANCE, item.step.lines.length),
+      (total, item) => total + Math.min(GLANCE, rowsOf(model, item.step)),
       0,
     );
 
     const reserve = Math.min(glances, Math.max(0, left - FLOOR));
-    const take = Math.min(left - reserve, focused.step.lines.length);
+    const take = Math.min(left - reserve, rowsOf(model, focused.step));
 
     shown.set(focused.index, take);
     left -= take;
   }
 
   for (const item of others) {
-    const take = Math.min(GLANCE, item.step.lines.length, Math.max(0, left));
+    const take = Math.min(GLANCE, rowsOf(model, item.step), Math.max(0, left));
 
     shown.set(item.index, take);
     left -= take;
@@ -308,6 +408,15 @@ function share(model: Model, budget: number) {
   return shown;
 }
 
+// How many rows a step's log would fill. One per line until they are wrapped,
+// and then as many as each line needs
+function rowsOf(model: Model, step: Step) {
+  if (!model.wrapped) return step.lines.length;
+
+  const room = Math.max(8, model.columns - RULE.length);
+  return step.lines.reduce((total, line) => total + wrapped(line, room).length, 0);
+}
+
 const RULE = "        \u2502 ";
 
 // The newest lines, less whatever the reader has scrolled back past. One line
@@ -317,10 +426,26 @@ function logs(model: Model, step: Step, take: number, colour: boolean) {
 
   const end = Math.max(1, step.lines.length - step.offset);
   const room = Math.max(8, model.columns - RULE.length);
+  const lines = step.lines.slice(Math.max(0, end - take), end);
+
+  // Wrapped, one line is several rows, so what is taken is counted in rows
+  // after the wrapping rather than in lines before it
+  const rows = lines.flatMap((line) =>
+    model.wrapped ? wrapped(line, room) : [clip(line, room)],
+  );
 
-  return step.lines
-    .slice(Math.max(0, end - take), end)
-    .map((line) => `${paint(RULE, DIM, colour)}${clip(line, room)}`);
+  return rows.slice(-take).map((row) => `${paint(RULE, DIM, colour)}${row}`);
+}
+
+// Cut on width alone. A build's output is not prose, and breaking a path or a
+// stack frame on a space would put half of it where nothing can find it
+function wrapped(line: string, room: number) {
+  if (line.length <= room) return [line];
+
+  const rows: string[] = [];
+  for (let at = 0; at < line.length; at += room) rows.push(line.slice(at, at + room));
+
+  return rows;
 }
 
 // The ellipsis is the whole point: a line that was cut has to say so
@@ -329,36 +454,60 @@ export function clip(text: string, room: number) {
   return `${text.slice(0, room - 1)}\u2026`;
 }
 
-function finish(rows: string[], body: number, colour: boolean) {
+// Padded above rather than below, so the newest row sits against the footer
+// and the frame fills upwards the way a terminal's own output does. The footer
+// is cut like any other row: one too wide wraps, and a wrapped row costs two
+function finish(model: Model, rows: string[], body: number, colour: boolean) {
   const filled = rows.slice(0, body);
-  while (filled.length < body) filled.push("");
+  const blanks = Array.from({ length: Math.max(0, body - filled.length) }, () => "");
+  const help = clip(HELP, Math.max(24, model.columns));
 
-  return [...filled, "", paint(HELP, DIM, colour)];
+  return [...blanks, ...filled, "", paint(help, DIM, colour)];
 }
 
 function title(model: Model, step: Step, index: number, colour: boolean) {
   const selected = index === model.cursor;
-  const caret = selected ? "\u276f" : " ";
+  const running = step.state === "running";
   const arrow = step.expanded ? "\u25be" : "\u25b8";
   const glyph = step.state === "done" ? "\u2714" : step.state === "failed" ? "\u2718" : arrow;
 
   const said = step.note ? `: ${step.note}` : step.detail ? `  ${step.detail}` : "";
 
   const head: Piece[] = [
-    { text: gutter(model), colour: DIM },
+    // The run clock as this row stood: still moving while the step is, and
+    // stopped at the moment it finished
+    { text: gutter(model, step.ended), colour: DIM },
     { text: " " },
-    { text: caret, colour: selected ? CURSOR : undefined },
+    // Turning while the step is working, blank when it is not. A step that has
+    // stopped saying anything still turns, which is what quiet then says
+    { text: running ? spinner(model.now) : " ", colour: RUNNING },
     { text: " " },
     { text: glyph, colour: stateColour(step) },
     { text: " " },
     { text: step.label, colour: labelColour(step, selected) },
     { text: said, colour: DIM },
+    { text: quiet(model, step), colour: WARN },
   ];
 
   // Frozen at what it cost the moment it finished, still counting until then
   return fit(model, head, elapsed(step.ended ?? model.now, step.started), colour);
 }
 
+function spinner(now: number) {
+  return SPINNER[Math.floor(now / SPIN_MS) % SPINNER.length] ?? " ";
+}
+
+// How long a running step has said nothing. The difference between a build
+// that is working and one that is wedged, which nothing else on the row shows
+function quiet(model: Model, step: Step) {
+  if (step.state !== "running") return "";
+
+  const since = model.now - (step.spoke ?? step.started);
+  if (since < QUIET_MS) return "";
+
+  return `  quiet ${elapsed(model.now, step.spoke ?? step.started)}`;
+}
+
 function stateColour(step: Step) {
   if (step.state === "done") return DONE;
   if (step.state === "failed") return FAILED;
@@ -375,9 +524,10 @@ function labelColour(step: Step, selected: boolean) {
   return undefined;
 }
 
-// The stamp is the whole run, and it keeps counting for every row a step adds
-function gutter(model: Model) {
-  return `[${elapsed(model.now, model.started, true)}]`;
+// How far into the run this row belongs. Given a moment it stops there, which
+// is what keeps a finished row from ticking along with the one still running
+function gutter(model: Model, at?: number) {
+  return elapsed(at ?? model.now, model.started, true);
 }
 
 export function elapsed(now: number, started: number, clock = false) {

package/src/cli/viewer.ts

@@ -79,8 +79,13 @@ export function createViewer(
   input.resume();
   input.setEncoding("utf8");
 
+  const plan = (points: string[]) => change((current) => ({ ...current, planned: points }));
+
   const say = (message: string) =>
-    change((current) => ({ ...current, messages: [...current.messages, message] }));
+    change((current) => ({
+      ...current,
+      messages: [...current.messages, { text: message, at: Date.now() }],
+    }));
 
   const onKey = (data: string) => {
     for (const key of keysOf(data)) {
@@ -159,16 +164,23 @@ export function createViewer(
         expanded: item.held,
       }));
 
+    // Both mark the step as having spoken, which is what tells a working step
+    // from a wedged one on the row itself
     return {
-      detail: (message) => edit((item) => ({ ...item, detail: message })),
+      detail: (message) => edit((item) => ({ ...item, detail: message, spoke: Date.now() })),
       line: (message) =>
-        edit((item) => ({ ...item, lines: [...item.lines, message].slice(-KEPT) })),
+        edit((item) => ({
+          ...item,
+          spoke: Date.now(),
+          lines: [...item.lines, message].slice(-KEPT),
+        })),
       done: (message) => settle("done", message),
       fail: (message) => settle("failed", message),
     };
   };
 
   return Object.assign(say, {
+    plan,
     warn: say,
     fail: say,
     done: say,
@@ -188,5 +200,30 @@ function summary(model: Model): string[] {
     return `${glyph} ${step.label}${said} (${took})`;
   });
 
-  return [...rows, ...model.messages];
+  return [
+    ...rows,
+    ...model.messages.map((message) => message.text),
+    ...whatFailed(model),
+  ];
+}
+
+// The end of what the step that failed was saying. The reason a build stopped
+// is in its own output, and the alternate screen takes every frame with it
+const TAIL = 20;
+
+function whatFailed(model: Model) {
+  const rows: string[] = [];
+
+  for (const step of model.steps.filter((item) => item.state === "failed")) {
+    if (step.lines.length === 0) continue;
+
+    const kept = step.lines.slice(-TAIL);
+    const dropped = step.lines.length - kept.length;
+
+    rows.push("", `${step.label} said:`);
+    if (dropped > 0) rows.push(`  ... ${dropped} earlier lines`);
+    rows.push(...kept.map((line) => `  ${line}`));
+  }
+
+  return rows;
 }

package/src/config.ts

@@ -1,5 +1,8 @@
+import { PROXY } from "./services/proxy.js";
 import { assertSteps } from "./pipeline.js";
 import { pluginSteps } from "./plugin.js";
+import { listRefs } from "./secrets/refs.js";
+import type { AnyStep } from "./pipeline.js";
 import type { Deployment, Environment } from "./types.js";
 
 // The one selected, from the one place worth looking. A deployment that carries
@@ -9,6 +12,72 @@ export function environmentOf(config: Deployment, name: string) {
   return config.environment ?? config.environments?.[name];
 }
 
+// The selected environment's items folded into the apps they name, after each
+// app's own. Refs merge in order and the later key wins, so the deployment holds
+// what every environment shares and the environment holds what differs
+export function withEnvironment(config: Deployment, name: string): Deployment {
+  const environment = environmentOf(config, name);
+  if (!environment?.secrets && !environment?.files && !environment?.steps) return config;
+
+  const { secrets = {}, files = {}, steps, ...rest } = environment;
+  assertNamesApps(config, name, [...Object.keys(secrets), ...Object.keys(files)]);
+
+  const apps = config.apps.map((app) => {
+    const refs = secrets[app.name];
+    const paths = files[app.name];
+    if (!refs && !paths) return app;
+
+    return {
+      ...app,
+      secrets: refs ? [...listRefs(app.secrets), ...listRefs(refs)] : app.secrets,
+      files: paths ? { ...app.files, ...paths } : app.files,
+    };
+  });
+
+  const folded = { ...config, apps, steps: steps ? stepsWith(config, steps) : config.steps };
+
+  // What was folded in leaves the environment, so folding it again adds nothing
+  // and a caller that already did it can hand the result on to one that will
+  if (config.environment) return { ...folded, environment: rest };
+  return { ...folded, environments: { ...config.environments, [name]: rest } };
+}
+
+// A step at a point the deployment already fills replaces it for this
+// environment, where it stood. One only the environment has leads, the way a
+// plugin's does, so a snapshot added here still runs above the shared migration
+function stepsWith(config: Deployment, given: AnyStep[]) {
+  // Before the replacement map, which would otherwise keep the last of two and
+  // drop the other without a word
+  assertSteps(given);
+
+  const shared = config.steps ?? [];
+  const claimed = new Set(shared.map((step) => step.point));
+  const replacing = new Map(given.map((step) => [step.point, step]));
+
+  const steps = [
+    ...given.filter((step) => !claimed.has(step.point)),
+    ...shared.map((step) => replacing.get(step.point) ?? step),
+  ];
+
+  // A plugin fills points too, and one the environment lands on is the same
+  // collision defineDeployment refuses
+  assertSteps([...pluginSteps(config.plugins), ...steps]);
+  return steps;
+}
+
+// An environment file does not import the deployment, so an app name there
+// cannot be checked when it compiles. A typo would otherwise read nothing
+function assertNamesApps(config: Deployment, environment: string, names: string[]) {
+  const apps = new Set(config.apps.map((app) => app.name));
+  const unknown = names.filter((name) => !apps.has(name));
+  if (unknown.length === 0) return;
+
+  throw new Error(
+    `${environment} gives secrets to ${unknown.join(", ")}, which this deployment ` +
+      `has no app by that name for. Its apps are ${[...apps].join(", ")}`,
+  );
+}
+
 // Identity, but it pins the type so a missing health predicate fails to
 // compile. Environments are excluded rather than merely unused: each lives in
 // a redkite.<name>.config.ts of its own, and a second place to put one is a
@@ -30,6 +99,7 @@ export function defineDeployment<const T extends Deployment & { environments?: n
 // Identity, for the environment a redkite.<name>.config.ts holds. It pins the
 // type the same way defineDeployment does, so a missing subnet fails to compile
 export function defineEnvironment<const T extends Environment>(environment: T): T {
+  if (environment.steps) assertSteps(environment.steps);
   return environment;
 }
 
@@ -100,6 +170,16 @@ function assertUniqueNames(config: Deployment) {
   const names = [...config.apps, ...config.services].map((item) => item.name);
   const duplicate = names.find((name, i) => names.indexOf(name) !== i);
   if (duplicate) throw new Error(`Duplicate name in deployment: ${duplicate}`);
+
+  // The proxy is derived rather than listed, and it already has this name. A
+  // service claiming it is a second container on the first one's name, which
+  // nothing downstream can tell apart
+  if (!names.includes(PROXY)) return;
+
+  throw new Error(
+    `${PROXY} is the derived proxy, so a service cannot be called that. ` +
+      "What it runs and what goes in its server block is proxy: nginx({ … })",
+  );
 }
 
 // Two apps on the same route means one of them is unreachable, and which one

package/src/deploy.ts

@@ -1,6 +1,6 @@
 import { build, type BuildContext, type BuildResult } from "./build.js";
 import { assertCheckable, runChecks } from "./checks.js";
-import { environmentOf } from "./config.js";
+import { environmentOf, withEnvironment } from "./config.js";
 import { Docker } from "./docker.js";
 import { envFileFor } from "./environment.js";
 import { healthcheck, type HealthDeps } from "./health.js";
@@ -68,11 +68,12 @@ export async function verify(options: DeployOptions): Promise<Finished> {
 
 async function start(run: Run, options: DeployOptions): Promise<Finished> {
   const log = options.log ?? silent;
+  const config = withEnvironment(options.config, options.environment);
 
   const setting: Omit<Context, "task"> = {
-    config: options.config,
+    config,
     environment: options.environment,
-    topology: topologyFor(options.config, options.environment),
+    topology: topologyFor(config, options.environment),
     host: options.host,
     docker: new Docker(options.host),
     secrets: options.secrets,
@@ -82,7 +83,7 @@ async function start(run: Run, options: DeployOptions): Promise<Finished> {
 
   // A plugin's steps lead, so a snapshot listed as a plugin runs above the
   // migration the deployment writes after it
-  const added = [...pluginSteps(options.config.plugins), ...(options.config.steps ?? [])];
+  const added = [...pluginSteps(config.plugins), ...(config.steps ?? [])];
 
   return await runPipeline(run, merge(supplied(options), added), setting, options.signal);
 }
@@ -349,14 +350,26 @@ async function buildAll(
       const task = log.step(`Building ${app.name}`);
 
       try {
+        // Said before the reads rather than after them. Every ref is a call to
+        // the vault, and a step that has not spoken yet looks like one that is
+        // not doing anything
+        task.detail("reading its environment");
+        const env = await readEnv(app.secrets, context.secrets);
+
+        task.detail("reading the files it ships with");
+        const files = await resolveFiles(app, context.secrets);
+
         const buildContext: BuildContext = {
           host: here ?? context.host,
           docker: here ? new Docker(here) : context.docker,
           deliver: here && { host: context.host, docker: context.docker },
-          env: await readEnv(app.secrets, context.secrets),
-          files: await resolveFiles(app, context.secrets),
+          env,
+          files,
           branch: environment.branch,
           environment: context.environment,
+          // Forwarded to the deploy host by the connection, so what this
+          // process holds is what the build there can reach
+          agent: Boolean(process.env["SSH_AUTH_SOCK"]),
           detail: task.detail,
           output: task.line,
         };

package/src/docker.ts

@@ -15,6 +15,8 @@ export type BuildInvocation = {
   tags: string[];
   // Secret id to a path on the host holding its contents
   secrets?: Record<string, string>;
+  // Forwards the agent this shell holds into the build
+  ssh?: boolean;
   // Stops at the builder stage, for the image a step before the swap runs in
   target?: string;
 };
@@ -174,6 +176,9 @@ class DockerImage {
       ...Object.entries(spec.secrets ?? {}).map(
         ([id, path]) => `--secret id=${id},src=${path}`,
       ),
+      // Forwards this shell's agent into the build, which is how a dependency
+      // fetched over ssh is authenticated
+      ...(spec.ssh ? ["--ssh default"] : []),
       ...(spec.target ? [`--target ${spec.target}`] : []),
       spec.context,
     ].join(" ");