redkite 0.1.9 → 0.1.11

package/src/services/planned.ts

@@ -1,7 +1,7 @@
 import { createHash } from "node:crypto";
 
 import type { Docker } from "../docker.js";
-import { LISTEN_PORT, renderNginx } from "../nginx.js";
+import { LISTEN_PORT, proxyService, renderProxy } from "./proxy.js";
 import type { Run } from "../pipeline.js";
 import type { ServiceTopology, Topology } from "../topology.js";
 import type { Deployment, ServiceSpec } from "../types.js";
@@ -36,10 +36,10 @@ export function plannedServices(
   if (run === "verify") return rest;
 
   const proxy: PlannedService = {
-    spec: router(config),
+    spec: proxyService(config),
     service: topology.router,
     files: {
-      "/etc/nginx/conf.d/default.conf": renderNginx(topology, config.maxBodySize),
+      "/etc/nginx/conf.d/default.conf": renderProxy(topology, config.proxy),
     },
     publish: topology.publicPort,
   };
@@ -47,16 +47,6 @@ export function plannedServices(
   return [proxy, ...rest];
 }
 
-// Not something a deployment lists. Apps carry routes, routes need a proxy to
-// resolve them, and the one that renders them is this
-export function router(config: Deployment): ServiceSpec {
-  return {
-    name: "nginx",
-    image: config.proxyImage ?? "nginx:stable",
-    restart: "always",
-  };
-}
-
 // Everything a recreate would change, and nothing a deploy resolves. Secrets
 // are named by their refs rather than their values: whether the config is the
 // one running is a question a plan should answer without unlocking a vault

package/src/services/proxy.ts

@@ -0,0 +1,126 @@
+import type { AppTopology, Topology } from "../topology.js";
+import type { Deployment, ProxySpec, ServiceSpec } from "../types.js";
+
+// The proxy every route is resolved by: what it is, and the configuration it
+// runs. Both halves here, because a deployment does not list this service and
+// the only thing that decides either is the app list.
+//
+// Header policy is set once rather than per location, which is what let the two
+// blocks drift apart and lose X-Forwarded-For on one of them.
+
+const FORWARDING = [
+  "proxy_set_header Host $http_host;",
+  "proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;",
+  "proxy_set_header X-Forwarded-Host $http_host;",
+  "proxy_set_header X-Forwarded-Proto $scheme;",
+];
+
+const FAILOVER = [
+  "proxy_connect_timeout 2s;",
+  "proxy_read_timeout 30s;",
+  "proxy_next_upstream error timeout http_502 http_503;",
+  "proxy_next_upstream_tries 2;",
+];
+
+// The port the proxy listens on inside its container. The published port maps
+// onto this one, so both sides have to agree and only one of them may say it
+export const LISTEN_PORT = 3000;
+
+// What the derived proxy is called. A service by this name would be a second
+// container claiming the first one's, so the config surface refuses it
+export const PROXY = "nginx";
+
+// Not something a deployment lists. Apps carry routes, routes need something
+// to resolve them, and this says what that something is
+export function proxyService(config: Deployment): ServiceSpec {
+  return {
+    name: PROXY,
+    image: config.proxy?.image ?? "nginx:stable",
+    restart: "always",
+  };
+}
+
+export function renderProxy(topology: Topology, proxy: ProxySpec = {}) {
+  const upstreams = topology.apps.map((app) => upstream(app)).join("\n\n");
+  // Longest route first, so "/api/" is not shadowed by the "/" catch-all
+  const ordered = [...topology.apps].sort((a, b) => b.route.length - a.route.length);
+  const locations = ordered.map((app) => location(app, proxy.location ?? [])).join("\n\n");
+
+  assertNotDerived(proxy.server ?? []);
+
+  const server = settled([
+    `merge_slashes off;`,
+    `client_max_body_size ${proxy.maxBodySize ?? "1M"};`,
+    ...(proxy.server ?? []),
+  ]);
+
+  return `resolver 127.0.0.1 valid=5s;
+
+${upstreams}
+
+server {
+    listen ${LISTEN_PORT};
+${indented(server, 4)}
+${locations}
+}
+`;
+}
+
+// nginx refuses a second proxy_read_timeout but expects several
+// proxy_set_header, so what identifies a line is the directive and, for the
+// ones that carry a name, the name after it
+const NAMED = new Set(["add_header", "proxy_set_header", "proxy_hide_header", "set"]);
+
+function directive(line: string) {
+  const [name = "", argument = ""] = line.trim().split(/\s+/);
+  return NAMED.has(name) ? `${name} ${argument}` : name;
+}
+
+// The last of each wins. A line written by hand replaces the one redkite would
+// have put there rather than repeating it, which nginx would refuse outright
+function settled(lines: string[]) {
+  const kept = new Map<string, string>();
+
+  for (const line of lines) {
+    if (!line.trim()) continue;
+    kept.set(directive(line), line.trim());
+  }
+
+  return [...kept.values()];
+}
+
+// The published port maps onto this one, so both sides have to agree and only
+// one of them may say it
+function assertNotDerived(lines: string[]) {
+  const listening = lines.find((line) => directive(line) === "listen");
+  if (!listening) return;
+
+  throw new Error(
+    `The proxy's server block says ${listening.trim()}, and the port it listens ` +
+      "on inside its container is what publicPort is published onto",
+  );
+}
+
+function indented(lines: string[], by: number) {
+  return lines.map((line) => " ".repeat(by) + line).join("\n");
+}
+
+function upstream(app: AppTopology) {
+  return `upstream ${app.name} {
+    server ${app.container}:${app.port} max_fails=1 fail_timeout=2s;
+    server ${app.retired}:${app.port} backup;
+}`;
+}
+
+function location(app: AppTopology, added: string[]) {
+  // A trailing slash on proxy_pass strips the location prefix. The catch-all
+  // must not carry one, a mounted route must
+  const target = app.route === "/" ? `http://${app.name}` : `http://${app.name}/`;
+
+  // Last, so one of these replaces what redkite set rather than repeating it
+  const body = settled([`proxy_pass ${target};`, ...FAILOVER, ...FORWARDING, ...added]);
+
+  return `    location ${app.route} {
+${indented(body, 8)}
+    }`;
+}

package/src/source.ts

@@ -32,6 +32,9 @@ export type SourceRequest = {
   ref: Ref;
   submodules: boolean;
   detail?: (message: string) => void;
+  // Every line git writes. A first clone of a large repository is the slowest
+  // part of a build and it used to say nothing at all while it happened
+  output?: (line: string) => void;
 };
 
 // Where each kind lives. A tag is peeled, so an annotated one answers with the
@@ -135,7 +138,7 @@ async function clonedSource(
     `  mkdir -p '${host.cache}/mirrors'`,
     `  git clone --mirror '${repo}' '${mirror}'`,
     "fi",
-  ]);
+  ], request.output);
 
   const resolved = await run(host, `resolving ${name}`, [
     `git -C '${mirror}' rev-parse '${REVISION[kind](name)}'`,
@@ -160,7 +163,7 @@ async function clonedSource(
     // Leaves the submodules alone: they are tracked, and clean only removes
     // what is not
     `git -C '${path}' clean -ffdx`,
-  ]);
+  ], request.output);
 
   if (request.submodules) {
     // --remote follows the branch named in .gitmodules rather than the commit
@@ -169,15 +172,21 @@ async function clonedSource(
     await run(host, "updating submodules", [
       `git -C '${path}' submodule sync --recursive`,
       `git -C '${path}' submodule update --init --remote --recursive`,
-    ]);
+    ], request.output);
   }
 
   return { release, tree: path };
 }
 
-async function run(host: Host, what: string, script: string[]) {
+async function run(
+  host: Host,
+  what: string,
+  script: string[],
+  output?: (line: string) => void,
+) {
   const result = await host.sh(
     [`export GIT_SSH_COMMAND='${GIT_SSH}'`, "set -e", ...script].join("\n"),
+    output,
   );
 
   if (result.code === 0) return result;

package/src/steps.ts

@@ -1,4 +1,3 @@
-import { environmentOf } from "./config.js";
 import { envFlags } from "./environment.js";
 import type { Built, Plan, Step } from "./pipeline.js";
 import type { Topology } from "./topology.js";
@@ -30,10 +29,6 @@ type MigrateOptions = {
   // already reach whatever the app's environment file points at. A database
   // this deployment runs as a service is on "deployment" instead
   network?: StepNetwork;
-  // Written when the step reached its database through a forward. It runs on
-  // the deploy host now, which is the machine that bastion named, so nothing
-  // is forwarded and the deploy refuses a bastion that is anything else
-  tunnel?: { bastion: string; from: string; alias?: string; port?: number };
 };
 
 // An ordinary step at an ordinary point. Hung before the swap, it runs while
@@ -43,7 +38,7 @@ export function migrate(options: MigrateOptions): Step<`swap:before:${string}`>
 
   return {
     point: `swap:before:migrate-${options.app}`,
-    check: (plan) => assertReachable(plan, options),
+    check: (plan) => assertApp(plan, options),
 
     run: async (input, context) => {
       const image = builderOf(input, options.app);
@@ -73,19 +68,9 @@ export function migrate(options: MigrateOptions): Step<`swap:before:${string}`>
 
 // Checked before the run starts, so a config naming an app that never kept a
 // builder fails without having touched the host
-function assertReachable(plan: Plan, options: MigrateOptions) {
-  const app = plan.config.apps.find((item) => item.name === options.app);
-  if (!app) throw new Error(`${options.app} names no app in this deployment`);
-
-  const bastion = environmentOf(plan.config, plan.environment)?.host?.bastion;
-  const tunnel = options.tunnel;
-  if (!tunnel || tunnel.bastion === bastion) return;
-
-  throw new Error(
-    `${options.app} tunnels its migration through ${tunnel.bastion}, ` +
-      `but ${plan.environment} deploys to ${bastion ?? "this machine"}. ` +
-      "The step runs on the deploy host, so they have to be the same",
-  );
+function assertApp(plan: Plan, options: MigrateOptions) {
+  if (plan.config.apps.some((item) => item.name === options.app)) return;
+  throw new Error(`${options.app} names no app in this deployment`);
 }
 
 // A deployment may replace the build step, and a command hung on the pipeline

package/src/types.ts

@@ -26,6 +26,16 @@ export type Environment = {
   // derives. For a database or a legacy service that has no DNS the apps can
   // use, and whose address is what differs between environments
   extraHosts?: Record<string, string>;
+  // App name to the items this environment reads, after the app's own. Which
+  // item is usually what differs between staging and production, and the rest
+  // of what differs already lives here
+  secrets?: Record<string, SecretRefs>;
+  // App name to container path to item, laid over the app's own files
+  files?: Record<string, Record<string, SecretRef>>;
+  // Steps for this environment alone. One at a point the deployment already
+  // fills replaces it here, where it stood. One only this environment has runs
+  // ahead of the deployment's, so a snapshot lands above the migration
+  steps?: AnyStep[];
 };
 
 // Where a step's container is attached. "host" is the deploy host's own stack,
@@ -68,6 +78,22 @@ export type SecretRef = {
 // One entry, or several merged in order. A key set by a later ref wins
 export type SecretRefs = SecretRef | SecretRef[];
 
+// The proxy every app's route is resolved by. Written with nginx(), because
+// what redkite derives is the server block and the upstreams and what this
+// says is everything around them
+export type ProxySpec = {
+  // Image it runs, pinned rather than floating
+  image?: string;
+  // Largest request body it accepts before answering 413
+  maxBodySize?: string;
+  // Lines put inside the server block, above the locations. For what belongs
+  // to the whole server: a redirect, a rate limit zone, an error page
+  server?: string[];
+  // Lines put inside every location, below what redkite sets, so one of these
+  // replaces a header rather than being replaced by it
+  location?: string[];
+};
+
 export type ServiceSpec = {
   // Identifies the service, and becomes part of its container name
   name: string;
@@ -239,10 +265,9 @@ export type Deployment = {
   // them: the thing that differs between staging and production is a file, not
   // a key several levels down a literal
   environments?: Record<string, Environment>;
-  // Largest request body the proxy accepts before answering 413
-  maxBodySize?: string;
-  // Image the derived proxy runs, pinned rather than floating
-  proxyImage?: string;
+  // The derived proxy, which is not something a deployment lists: apps carry
+  // routes, routes need something to resolve them, so there is exactly one
+  proxy?: ProxySpec;
   // Long-lived containers shared by the apps, not rebuilt on every deploy.
   // The proxy is not one of them: apps with routes imply exactly one, so it is
   // derived rather than listed

package/dist/nginx.d.ts

@@ -1,4 +0,0 @@
-import type { Topology } from "./topology.js";
-export declare const LISTEN_PORT = 3000;
-export declare function renderNginx(topology: Topology, maxBodySize?: string): string;
-//# sourceMappingURL=nginx.d.ts.map

package/dist/nginx.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"nginx.d.ts","sourceRoot":"","sources":["../src/nginx.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAe,QAAQ,EAAE,MAAM,eAAe,CAAC;AAsB3D,eAAO,MAAM,WAAW,OAAO,CAAC;AAEhC,wBAAgB,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,MAAM,UAoBnE"}

package/dist/nginx.js

@@ -1,53 +0,0 @@
-// The nginx configuration is a function of the app list. Header policy is set
-// here once rather than per location, which is what let the two blocks drift
-// apart and lose X-Forwarded-For on one of them.
-const FORWARDING = [
-    "proxy_set_header Host $http_host;",
-    "proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;",
-    "proxy_set_header X-Forwarded-Host $http_host;",
-    "proxy_set_header X-Forwarded-Proto $scheme;",
-];
-const FAILOVER = [
-    "proxy_connect_timeout 2s;",
-    "proxy_read_timeout 30s;",
-    "proxy_next_upstream error timeout http_502 http_503;",
-    "proxy_next_upstream_tries 2;",
-];
-// The port the proxy listens on inside its container. The published port maps
-// onto this one, so both sides have to agree and only one of them may say it
-export const LISTEN_PORT = 3000;
-export function renderNginx(topology, maxBodySize) {
-    const upstreams = topology.apps.map((app) => upstream(app)).join("\n\n");
-    // Longest route first, so "/api/" is not shadowed by the "/" catch-all
-    const ordered = [...topology.apps].sort((a, b) => b.route.length - a.route.length);
-    const locations = ordered.map((app) => location(app)).join("\n\n");
-    return `resolver 127.0.0.1 valid=5s;
-
-${upstreams}
-
-server {
-    listen ${LISTEN_PORT};
-    merge_slashes off;
-
-    client_max_body_size ${maxBodySize ?? "1M"};
-
-${locations}
-}
-`;
-}
-function upstream(app) {
-    return `upstream ${app.name} {
-    server ${app.container}:${app.port} max_fails=1 fail_timeout=2s;
-    server ${app.retired}:${app.port} backup;
-}`;
-}
-function location(app) {
-    // A trailing slash on proxy_pass strips the location prefix. The catch-all
-    // must not carry one, a mounted route must
-    const target = app.route === "/" ? `http://${app.name}` : `http://${app.name}/`;
-    const body = [`proxy_pass ${target};`, ...FAILOVER, ...FORWARDING];
-    return `    location ${app.route} {
-${body.map((line) => `        ${line}`).join("\n")}
-    }`;
-}
-//# sourceMappingURL=nginx.js.map

package/dist/nginx.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"nginx.js","sourceRoot":"","sources":["../src/nginx.ts"],"names":[],"mappings":"AAEA,8EAA8E;AAC9E,6EAA6E;AAC7E,iDAAiD;AAEjD,MAAM,UAAU,GAAG;IACjB,mCAAmC;IACnC,8DAA8D;IAC9D,+CAA+C;IAC/C,6CAA6C;CAC9C,CAAC;AAEF,MAAM,QAAQ,GAAG;IACf,2BAA2B;IAC3B,yBAAyB;IACzB,sDAAsD;IACtD,8BAA8B;CAC/B,CAAC;AAEF,8EAA8E;AAC9E,6EAA6E;AAC7E,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC;AAEhC,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,WAAoB;IAElE,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzE,uEAAuE;IACvE,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACnF,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEnE,OAAO;;EAEP,SAAS;;;aAGE,WAAW;;;2BAGG,WAAW,IAAI,IAAI;;EAE5C,SAAS;;CAEV,CAAC;AACF,CAAC;AAED,SAAS,QAAQ,CAAC,GAAgB;IAChC,OAAO,YAAY,GAAG,CAAC,IAAI;aAChB,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,IAAI;aACzB,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,IAAI;EAClC,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,GAAgB;IAChC,2EAA2E;IAC3E,2CAA2C;IAC3C,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;IAChF,MAAM,IAAI,GAAG,CAAC,cAAc,MAAM,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,UAAU,CAAC,CAAC;IAEnE,OAAO,gBAAgB,GAAG,CAAC,KAAK;EAChC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;MAC5C,CAAC;AACP,CAAC"}

package/src/nginx.ts

@@ -1,63 +0,0 @@
-import type { AppTopology, Topology } from "./topology.js";
-
-// The nginx configuration is a function of the app list. Header policy is set
-// here once rather than per location, which is what let the two blocks drift
-// apart and lose X-Forwarded-For on one of them.
-
-const FORWARDING = [
-  "proxy_set_header Host $http_host;",
-  "proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;",
-  "proxy_set_header X-Forwarded-Host $http_host;",
-  "proxy_set_header X-Forwarded-Proto $scheme;",
-];
-
-const FAILOVER = [
-  "proxy_connect_timeout 2s;",
-  "proxy_read_timeout 30s;",
-  "proxy_next_upstream error timeout http_502 http_503;",
-  "proxy_next_upstream_tries 2;",
-];
-
-// The port the proxy listens on inside its container. The published port maps
-// onto this one, so both sides have to agree and only one of them may say it
-export const LISTEN_PORT = 3000;
-
-export function renderNginx(topology: Topology, maxBodySize?: string) {
-
-  const upstreams = topology.apps.map((app) => upstream(app)).join("\n\n");
-  // Longest route first, so "/api/" is not shadowed by the "/" catch-all
-  const ordered = [...topology.apps].sort((a, b) => b.route.length - a.route.length);
-  const locations = ordered.map((app) => location(app)).join("\n\n");
-
-  return `resolver 127.0.0.1 valid=5s;
-
-${upstreams}
-
-server {
-    listen ${LISTEN_PORT};
-    merge_slashes off;
-
-    client_max_body_size ${maxBodySize ?? "1M"};
-
-${locations}
-}
-`;
-}
-
-function upstream(app: AppTopology) {
-  return `upstream ${app.name} {
-    server ${app.container}:${app.port} max_fails=1 fail_timeout=2s;
-    server ${app.retired}:${app.port} backup;
-}`;
-}
-
-function location(app: AppTopology) {
-  // A trailing slash on proxy_pass strips the location prefix. The catch-all
-  // must not carry one, a mounted route must
-  const target = app.route === "/" ? `http://${app.name}` : `http://${app.name}/`;
-  const body = [`proxy_pass ${target};`, ...FAILOVER, ...FORWARDING];
-
-  return `    location ${app.route} {
-${body.map((line) => `        ${line}`).join("\n")}
-    }`;
-}