redkite 0.1.11 → 0.1.13

package/src/deploy.ts

@@ -1,4 +1,4 @@
-import { build, type BuildContext, type BuildResult } from "./build.js";
+import { build, refOf, sourceOf, type BuildContext, type BuildResult } from "./build.js";
 import { assertCheckable, runChecks } from "./checks.js";
 import { environmentOf, withEnvironment } from "./config.js";
 import { Docker } from "./docker.js";
@@ -26,6 +26,7 @@ import { pluginSteps } from "./plugin.js";
 import { readEnv, readRef, type SecretStores } from "./secrets/refs.js";
 import { ensureService } from "./services/ensure.js";
 import { plannedServices } from "./services/planned.js";
+import { describeRef, describeRepo, type Source } from "./source.js";
 import { topologyFor, type AppTopology, type Topology } from "./topology.js";
 import type { AppSpec, Deployment } from "./types.js";
 
@@ -215,8 +216,14 @@ async function release(
     // Inside, because the swap is not over until this says so. A stop lands
     // here more often than anywhere else: it is the longest part, and by now
     // every address has already moved
-    if (!(await checkAll(context, health))) {
+    const unhealthy = await checkAll(context, health);
+
+    if (unhealthy.size > 0) {
       context.log.fail("Health checks failed, reverting");
+
+      // Before the revert, which gives the live name back to the container it
+      // retired. Asked after it, the logs would be the previous release's
+      await dumpLogs(context, apps, unhealthy);
       await Promise.all(apps.map((app) => revert(docker, topology, app)));
 
       return {
@@ -298,20 +305,58 @@ async function checkAll(context: Context, health: Omit<HealthDeps, "probe">) {
   const results = await Promise.all(
     context.config.apps.map(async (app) => {
       const target = appOf(topology, app.name);
+
+      // A step of its own for each app, rather than lines said beside the swap:
+      // every attempt lands on it, and a crash log gives it a file
       const deps: HealthDeps = {
         ...health,
         probe: async (container, url) => {
           const result = await docker.run(`exec ${container} curl -s ${url}`);
           return { code: result.code, output: result.stdout };
         },
-        log,
+        task: log.step(`Health check of ${app.name}`),
       };
 
-      return await healthcheck(target.container, target.port, app.health, deps);
+      const healthy = await healthcheck(target.container, target.port, app.health, deps);
+      return healthy ? undefined : target.container;
     }),
   );
 
-  return !results.includes(false);
+  // The containers that failed, since what gets written out for each depends on
+  // whether it was one of them
+  return new Set(results.filter((container): container is string => container !== undefined));
+}
+
+// Enough to hold a stack trace and what led up to it, not a day of access logs
+const LOG_TAIL = 200;
+
+// Every new container's, not only the ones that failed: a backend that never came
+// up is often explained by what the frontend says it could not reach. Each is a
+// step of its own, so the crash log gives it a file, and the one that failed its
+// check is marked failed, which is what puts its tail on the screen at the end
+async function dumpLogs(context: Context, apps: AppTopology[], unhealthy: Set<string>) {
+  await Promise.all(
+    apps.map(async (app) => {
+      const task = context.log.step(`Logs of ${app.name}`);
+      task.detail(`the last ${LOG_TAIL} lines of ${app.container}`);
+
+      const result = await context.docker.container.logs(app.container, LOG_TAIL);
+
+      // Said, and left there. The revert still has to run, and logs that could
+      // not be read are no reason to leave a failed release serving
+      if (result.code !== 0) {
+        task.fail(`could not read the logs of ${app.container}: ${result.stdout || result.stderr}`);
+        return;
+      }
+
+      const lines = result.stdout.split("\n").filter((line) => line.length > 0);
+      for (const line of lines) task.line(line);
+
+      const said = `${lines.length} lines from ${app.container}`;
+      if (unhealthy.has(app.container)) task.fail(`${said}, which failed its health check`);
+      else task.done(said);
+    }),
+  );
 }
 
 // The proxy is one of these, derived rather than listed. What each should be
@@ -347,6 +392,8 @@ async function buildAll(
 
   return await Promise.all(
     config.apps.map(async (app) => {
+      const placed = appOf(topology, app.name);
+      const source = await clone(app, placed, here ?? context.host, environment.branch, log);
       const task = log.step(`Building ${app.name}`);
 
       try {
@@ -374,7 +421,7 @@ async function buildAll(
           output: task.line,
         };
 
-        const result = await build(app, appOf(topology, app.name), buildContext);
+        const result = await build(app, placed, buildContext, source);
         task.done(`${result.release.slice(0, 7)}${result.cached ? " (held)" : ""}`);
 
         return { app, result };
@@ -386,6 +433,41 @@ async function buildAll(
   );
 }
 
+// A step of its own, ahead of the build that reads it. A wrong branch or an
+// unreachable repository shows here, and folded into the build it was a detail
+// that scrolled past without ever saying what it fetched
+async function clone(
+  app: AppSpec,
+  placed: AppTopology,
+  host: Host,
+  branch: string,
+  log: Log,
+): Promise<Source | undefined> {
+  // A directory is read where it is, not cloned, and the build still does that
+  if (!app.repo) return undefined;
+
+  const ref = refOf(app, branch);
+  const what = `${describeRepo(app.repo)} ${describeRef(ref)}`;
+  const task = log.step(`Cloning ${app.name}`);
+
+  try {
+    task.detail(what);
+    const source = await sourceOf(app, placed, {
+      host,
+      branch,
+      detail: task.detail,
+      output: task.line,
+    });
+
+    // Said again at the end, where it outlasts the details that replaced it
+    task.done(`${what} -> ${source.release.slice(0, 7)}`);
+    return source;
+  } catch (error) {
+    task.fail(`${app.name} could not clone ${what}`);
+    throw error;
+  }
+}
+
 async function resolveFiles(app: AppSpec, stores: SecretStores) {
   const entries = await Promise.all(
     Object.entries(app.files ?? {}).map(

package/src/docker.ts

@@ -233,6 +233,12 @@ class DockerContainer {
     return RUNNING.has(await this.status(name));
   }
 
+  // Both streams, since a process that dies as it starts says why on stderr, and
+  // stamped so a line can be set against when the health check gave up
+  async logs(name: string, tail: number) {
+    return await this.docker.run(`container logs --tail ${tail} --timestamps ${name} 2>&1`);
+  }
+
   async start(name: string) {
     if (!(await this.exists(name))) {
       throw new Error(`Cannot start ${name}, it does not exist`);

package/src/health.ts

@@ -1,4 +1,4 @@
-import { silent, type Log } from "./log.js";
+import { silent, type Task } from "./log.js";
 import type { HealthSpec } from "./types.js";
 
 const RETRIES = 10;
@@ -16,7 +16,9 @@ export type Probe = (
 export type HealthDeps = {
   probe: Probe;
   sleep: (ms: number) => Promise<void>;
-  log?: Log;
+  // The step the check reports on. Every attempt is said on it, so a check that
+  // gave up shows what the container answered each time rather than only that
+  task?: Task;
 };
 
 // One loop for every app. The two copies it replaces had already drifted: the
@@ -30,22 +32,28 @@ export async function healthcheck(
 ): Promise<boolean> {
   const retries = spec.retries ?? RETRIES;
   const ceiling = spec.intervalMs ?? INTERVAL_MS;
-  const log = deps.log ?? silent;
+  const task = deps.task ?? silent.step("");
   const delay = spec.delayMs ?? DELAY_MS;
+  const url = `localhost:${port}${spec.path}`;
 
+  task.detail(`probing ${container} at ${url}`);
   if (delay > 0) await deps.sleep(delay);
 
   let backoff = FIRST_BACKOFF_MS;
 
   for (let attempt = 1; attempt <= retries; attempt += 1) {
-    const url = `localhost:${port}${spec.path}`;
     const { code, output } = await deps.probe(container, url);
+    const said = (message: string) => task.detail(`attempt ${attempt} of ${retries}: ${message}`);
 
-    if (code === 0 && passes(spec, output, log, container)) {
-      log.done(`${container} is healthy`);
+    if (code === 0 && passes(spec, output, said)) {
+      task.done(`${container} healthy after ${attempt} ${attempt === 1 ? "attempt" : "attempts"}`);
       return true;
     }
 
+    // Nothing answered at all, which is a container still starting or one that
+    // is listening somewhere other than where the check asks
+    if (code !== 0) said(`no answer on ${url}`);
+
     if (attempt === retries) break;
 
     // Doubling from a quarter second means a container that starts quickly
@@ -54,30 +62,29 @@ export async function healthcheck(
     backoff = Math.min(backoff * 2, ceiling);
   }
 
-  log.fail(`${container} failed its health check after ${retries} attempts`);
+  task.fail(`${container} unhealthy after ${retries} attempts`);
   return false;
 }
 
 // A body that parses but fails the predicate is a retry, not a verdict. The
 // container may still be warming up, and the caller has a retry budget for it
-function passes(
-  spec: HealthSpec,
-  output: string,
-  log: Log,
-  container: string,
-) {
+function passes(spec: HealthSpec, output: string, said: (message: string) => void) {
   let body: unknown;
 
   try {
     body = JSON.parse(output);
   } catch {
-    log.warn(`${container} returned a body that is not JSON: ${output}`);
+    said(`answered with something that is not JSON: ${output}`);
+    return false;
+  }
+
+  if (typeof body !== "object" || body === null) {
+    said(`answered with JSON that is not an object: ${output}`);
     return false;
   }
 
-  if (typeof body !== "object" || body === null) return false;
   if (spec.expect(body as Record<string, unknown>)) return true;
 
-  log.warn(`${container} is not healthy yet: ${output}`);
+  said(`not healthy yet: ${output}`);
   return false;
 }

package/src/log.ts

@@ -1,8 +1,26 @@
 // The shape progress is reported through. The library only knows this much, so
 // a deploy driven from somewhere without a terminal supplies its own.
 
+// A short label a terminal can draw as a tag, such as the host a repository
+// lives on. Marked with private-use characters, which nothing a command prints
+// uses, so a log finds one without guessing at brackets somebody typed
+const TAG_OPEN = "\uE000";
+const TAG_CLOSE = "\uE001";
+
+export const TAGS = /\uE000([^\uE000\uE001]*)\uE001/g;
+
+export function tag(label: string) {
+  return `${TAG_OPEN}${label}${TAG_CLOSE}`;
+}
+
+// For a log that has no way to draw one: a file, a pipe, a CI log
+export function untagged(text: string) {
+  return text.replaceAll(TAGS, "[$1]");
+}
+
 // Work that is still running. Announced when it starts, because the thing worth
-// knowing during a four minute build is which step is the four minutes
+// knowing during a four minute build is which step is the four minutes. What it
+// says may carry a tag, which a log that draws plain text passes through untagged
 export type Task = {
   // What the step is doing right now, replacing whatever it said before
   detail(message: string): void;

package/src/source.ts

@@ -1,4 +1,5 @@
 import type { Host } from "./host.js";
+import { tag } from "./log.js";
 
 // Getting the source onto the machine that builds it. A repository the host
 // clones for itself over the forwarded agent, so no tree crosses the wire: a
@@ -45,6 +46,29 @@ const REVISION: Record<RefKind, (name: string) => string> = {
   commit: (name) => `${name}^{commit}`,
 };
 
+// Hosts common enough to be worth a tag in place of their name
+const HOSTS: Record<string, string> = { "github.com": "GH" };
+
+// user@host:path, or scheme://user@host/path, with the .git every clone URL ends
+// in. Anything that is not one of those, a directory for instance, has no host
+const REMOTE = /^(?:[a-z][a-z0-9+.-]*:\/\/)?(?:[^@/]+@)?([^:/]+)[:/](.+?)(?:\.git)?\/?$/;
+
+// How a repository is named on screen: the host as a tag where it is a known
+// one, and the path without the .git that says nothing a person needs
+export function describeRepo(repo: string) {
+  const match = REMOTE.exec(repo);
+  if (!match?.[1] || !match[2]) return repo.replace(/\.git$/, "");
+
+  const known = HOSTS[match[1]];
+  return known ? `${tag(known)} ${match[2]}` : `${match[1]}:${match[2]}`;
+}
+
+// A branch is what is tracked nearly always, so it goes by its name alone. A tag
+// or a commit is a pin, and says which kind it is
+export function describeRef(ref: Ref) {
+  return ref.kind === "branch" ? ref.name : `${ref.kind} ${ref.name}`;
+}
+
 // accept-new rather than no: a host key that changes is still worth refusing
 const GIT_SSH = "ssh -o StrictHostKeyChecking=accept-new";
 

package/src/types.ts

@@ -282,4 +282,13 @@ export type Deployment = {
   // refs, and whatever else brings steps of its own. Nothing a plugin carries
   // happens until it is listed here, redkite's own vault included
   plugins?: Plugin[];
+  // How redkite itself behaves, rather than anything it deploys
+  options?: DeploymentOptions;
+};
+
+export type DeploymentOptions = {
+  // A run that fails writes everything it said to
+  // /tmp/<project>/<environment>/crash-<time>/: the run's own log, and one file
+  // per step with its output in full. On unless this says false
+  crashLog?: boolean;
 };