redkite 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +161 -12
- package/dist/build.d.ts +3 -0
- package/dist/build.d.ts.map +1 -1
- package/dist/build.js +18 -2
- package/dist/build.js.map +1 -1
- package/dist/cli/agent.js +2 -2
- package/dist/cli/agent.js.map +1 -1
- package/dist/cli/config.d.ts +1 -0
- package/dist/cli/config.d.ts.map +1 -1
- package/dist/cli/config.js +82 -16
- package/dist/cli/config.js.map +1 -1
- package/dist/cli/dotenv.d.ts +2 -0
- package/dist/cli/dotenv.d.ts.map +1 -0
- package/dist/cli/dotenv.js +30 -0
- package/dist/cli/dotenv.js.map +1 -0
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +11 -5
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/screen.d.ts +10 -3
- package/dist/cli/screen.d.ts.map +1 -1
- package/dist/cli/screen.js +153 -36
- package/dist/cli/screen.js.map +1 -1
- package/dist/cli/viewer.d.ts.map +1 -1
- package/dist/cli/viewer.js +36 -4
- package/dist/cli/viewer.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +12 -0
- package/dist/config.js.map +1 -1
- package/dist/deploy.d.ts.map +1 -1
- package/dist/deploy.js +12 -2
- package/dist/deploy.js.map +1 -1
- package/dist/docker.d.ts +1 -0
- package/dist/docker.d.ts.map +1 -1
- package/dist/docker.js +3 -0
- package/dist/docker.js.map +1 -1
- package/dist/dockerfile.d.ts +1 -0
- package/dist/dockerfile.d.ts.map +1 -1
- package/dist/dockerfile.js +16 -3
- package/dist/dockerfile.js.map +1 -1
- package/dist/environment.d.ts +2 -0
- package/dist/environment.d.ts.map +1 -1
- package/dist/environment.js +56 -1
- package/dist/environment.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/log.d.ts +1 -0
- package/dist/log.d.ts.map +1 -1
- package/dist/log.js +1 -0
- package/dist/log.js.map +1 -1
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +3 -0
- package/dist/pipeline.js.map +1 -1
- package/dist/plugins/bitwarden.d.ts.map +1 -1
- package/dist/plugins/bitwarden.js +25 -6
- package/dist/plugins/bitwarden.js.map +1 -1
- package/dist/presets/nextApp.d.ts.map +1 -1
- package/dist/presets/nextApp.js +5 -4
- package/dist/presets/nextApp.js.map +1 -1
- package/dist/presets/nodeApp.d.ts.map +1 -1
- package/dist/presets/nodeApp.js +15 -6
- package/dist/presets/nodeApp.js.map +1 -1
- package/dist/secrets/manager.d.ts +9 -0
- package/dist/secrets/manager.d.ts.map +1 -0
- package/dist/secrets/manager.js +130 -0
- package/dist/secrets/manager.js.map +1 -0
- package/dist/secrets/store.d.ts.map +1 -1
- package/dist/secrets/store.js +46 -12
- package/dist/secrets/store.js.map +1 -1
- package/dist/services/ensure.js +1 -1
- package/dist/services/ensure.js.map +1 -1
- package/dist/services/index.d.ts +2 -1
- package/dist/services/index.d.ts.map +1 -1
- package/dist/services/index.js +11 -0
- package/dist/services/index.js.map +1 -1
- package/dist/services/planned.d.ts +0 -1
- package/dist/services/planned.d.ts.map +1 -1
- package/dist/services/planned.js +3 -12
- package/dist/services/planned.js.map +1 -1
- package/dist/services/proxy.d.ts +7 -0
- package/dist/services/proxy.d.ts.map +1 -0
- package/dist/services/proxy.js +103 -0
- package/dist/services/proxy.js.map +1 -0
- package/dist/source.d.ts +7 -1
- package/dist/source.d.ts.map +1 -1
- package/dist/source.js +17 -9
- package/dist/source.js.map +1 -1
- package/dist/steps.d.ts +1 -6
- package/dist/steps.d.ts.map +1 -1
- package/dist/steps.js +3 -3
- package/dist/steps.js.map +1 -1
- package/dist/types.d.ts +10 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/build.ts +22 -2
- package/src/cli/agent.ts +2 -2
- package/src/cli/config.ts +96 -15
- package/src/cli/dotenv.ts +35 -0
- package/src/cli/index.ts +11 -5
- package/src/cli/screen.ts +193 -43
- package/src/cli/viewer.ts +41 -4
- package/src/config.ts +19 -0
- package/src/deploy.ts +14 -2
- package/src/docker.ts +5 -0
- package/src/dockerfile.ts +23 -3
- package/src/environment.ts +71 -1
- package/src/index.ts +2 -1
- package/src/log.ts +4 -0
- package/src/pipeline.ts +4 -0
- package/src/plugins/bitwarden.ts +38 -11
- package/src/presets/nextApp.ts +5 -4
- package/src/presets/nodeApp.ts +15 -6
- package/src/secrets/manager.ts +172 -0
- package/src/secrets/store.ts +51 -16
- package/src/services/ensure.ts +1 -1
- package/src/services/index.ts +13 -1
- package/src/services/planned.ts +3 -13
- package/src/services/proxy.ts +126 -0
- package/src/source.ts +33 -9
- package/src/steps.ts +7 -7
- package/src/types.ts +26 -4
- package/dist/nginx.d.ts +0 -4
- package/dist/nginx.d.ts.map +0 -1
- package/dist/nginx.js +0 -53
- package/dist/nginx.js.map +0 -1
- package/src/nginx.ts +0 -63
package/src/cli/viewer.ts
CHANGED
|
@@ -79,8 +79,13 @@ export function createViewer(
|
|
|
79
79
|
input.resume();
|
|
80
80
|
input.setEncoding("utf8");
|
|
81
81
|
|
|
82
|
+
const plan = (points: string[]) => change((current) => ({ ...current, planned: points }));
|
|
83
|
+
|
|
82
84
|
const say = (message: string) =>
|
|
83
|
-
change((current) => ({
|
|
85
|
+
change((current) => ({
|
|
86
|
+
...current,
|
|
87
|
+
messages: [...current.messages, { text: message, at: Date.now() }],
|
|
88
|
+
}));
|
|
84
89
|
|
|
85
90
|
const onKey = (data: string) => {
|
|
86
91
|
for (const key of keysOf(data)) {
|
|
@@ -159,16 +164,23 @@ export function createViewer(
|
|
|
159
164
|
expanded: item.held,
|
|
160
165
|
}));
|
|
161
166
|
|
|
167
|
+
// Both mark the step as having spoken, which is what tells a working step
|
|
168
|
+
// from a wedged one on the row itself
|
|
162
169
|
return {
|
|
163
|
-
detail: (message) => edit((item) => ({ ...item, detail: message })),
|
|
170
|
+
detail: (message) => edit((item) => ({ ...item, detail: message, spoke: Date.now() })),
|
|
164
171
|
line: (message) =>
|
|
165
|
-
edit((item) => ({
|
|
172
|
+
edit((item) => ({
|
|
173
|
+
...item,
|
|
174
|
+
spoke: Date.now(),
|
|
175
|
+
lines: [...item.lines, message].slice(-KEPT),
|
|
176
|
+
})),
|
|
166
177
|
done: (message) => settle("done", message),
|
|
167
178
|
fail: (message) => settle("failed", message),
|
|
168
179
|
};
|
|
169
180
|
};
|
|
170
181
|
|
|
171
182
|
return Object.assign(say, {
|
|
183
|
+
plan,
|
|
172
184
|
warn: say,
|
|
173
185
|
fail: say,
|
|
174
186
|
done: say,
|
|
@@ -188,5 +200,30 @@ function summary(model: Model): string[] {
|
|
|
188
200
|
return `${glyph} ${step.label}${said} (${took})`;
|
|
189
201
|
});
|
|
190
202
|
|
|
191
|
-
return [
|
|
203
|
+
return [
|
|
204
|
+
...rows,
|
|
205
|
+
...model.messages.map((message) => message.text),
|
|
206
|
+
...whatFailed(model),
|
|
207
|
+
];
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// The end of what the step that failed was saying. The reason a build stopped
|
|
211
|
+
// is in its own output, and the alternate screen takes every frame with it
|
|
212
|
+
const TAIL = 20;
|
|
213
|
+
|
|
214
|
+
function whatFailed(model: Model) {
|
|
215
|
+
const rows: string[] = [];
|
|
216
|
+
|
|
217
|
+
for (const step of model.steps.filter((item) => item.state === "failed")) {
|
|
218
|
+
if (step.lines.length === 0) continue;
|
|
219
|
+
|
|
220
|
+
const kept = step.lines.slice(-TAIL);
|
|
221
|
+
const dropped = step.lines.length - kept.length;
|
|
222
|
+
|
|
223
|
+
rows.push("", `${step.label} said:`);
|
|
224
|
+
if (dropped > 0) rows.push(` ... ${dropped} earlier lines`);
|
|
225
|
+
rows.push(...kept.map((line) => ` ${line}`));
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
return rows;
|
|
192
229
|
}
|
package/src/config.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { PROXY } from "./services/proxy.js";
|
|
1
2
|
import { assertSteps } from "./pipeline.js";
|
|
2
3
|
import { pluginSteps } from "./plugin.js";
|
|
3
4
|
import type { Deployment, Environment } from "./types.js";
|
|
@@ -65,6 +66,14 @@ function assertOneSource(config: Deployment) {
|
|
|
65
66
|
throw new Error(`${app.name} says what to include, but is cloned rather than built from a path`);
|
|
66
67
|
}
|
|
67
68
|
|
|
69
|
+
const refs = ["branch", "tag", "commit"].filter((kind) => app[kind as "branch"]);
|
|
70
|
+
|
|
71
|
+
if (refs.length > 1) {
|
|
72
|
+
throw new Error(
|
|
73
|
+
`${app.name} names ${refs.join(" and ")} to build from, and they are different commits`,
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
|
|
68
77
|
if (app.include?.length === 0) {
|
|
69
78
|
throw new Error(`${app.name} includes nothing, so there would be no build context`);
|
|
70
79
|
}
|
|
@@ -92,6 +101,16 @@ function assertUniqueNames(config: Deployment) {
|
|
|
92
101
|
const names = [...config.apps, ...config.services].map((item) => item.name);
|
|
93
102
|
const duplicate = names.find((name, i) => names.indexOf(name) !== i);
|
|
94
103
|
if (duplicate) throw new Error(`Duplicate name in deployment: ${duplicate}`);
|
|
104
|
+
|
|
105
|
+
// The proxy is derived rather than listed, and it already has this name. A
|
|
106
|
+
// service claiming it is a second container on the first one's name, which
|
|
107
|
+
// nothing downstream can tell apart
|
|
108
|
+
if (!names.includes(PROXY)) return;
|
|
109
|
+
|
|
110
|
+
throw new Error(
|
|
111
|
+
`${PROXY} is the derived proxy, so a service cannot be called that. ` +
|
|
112
|
+
"What it runs and what goes in its server block is proxy: nginx({ … })",
|
|
113
|
+
);
|
|
95
114
|
}
|
|
96
115
|
|
|
97
116
|
// Two apps on the same route means one of them is unreachable, and which one
|
package/src/deploy.ts
CHANGED
|
@@ -349,14 +349,26 @@ async function buildAll(
|
|
|
349
349
|
const task = log.step(`Building ${app.name}`);
|
|
350
350
|
|
|
351
351
|
try {
|
|
352
|
+
// Said before the reads rather than after them. Every ref is a call to
|
|
353
|
+
// the vault, and a step that has not spoken yet looks like one that is
|
|
354
|
+
// not doing anything
|
|
355
|
+
task.detail("reading its environment");
|
|
356
|
+
const env = await readEnv(app.secrets, context.secrets);
|
|
357
|
+
|
|
358
|
+
task.detail("reading the files it ships with");
|
|
359
|
+
const files = await resolveFiles(app, context.secrets);
|
|
360
|
+
|
|
352
361
|
const buildContext: BuildContext = {
|
|
353
362
|
host: here ?? context.host,
|
|
354
363
|
docker: here ? new Docker(here) : context.docker,
|
|
355
364
|
deliver: here && { host: context.host, docker: context.docker },
|
|
356
|
-
env
|
|
357
|
-
files
|
|
365
|
+
env,
|
|
366
|
+
files,
|
|
358
367
|
branch: environment.branch,
|
|
359
368
|
environment: context.environment,
|
|
369
|
+
// Forwarded to the deploy host by the connection, so what this
|
|
370
|
+
// process holds is what the build there can reach
|
|
371
|
+
agent: Boolean(process.env["SSH_AUTH_SOCK"]),
|
|
360
372
|
detail: task.detail,
|
|
361
373
|
output: task.line,
|
|
362
374
|
};
|
package/src/docker.ts
CHANGED
|
@@ -15,6 +15,8 @@ export type BuildInvocation = {
|
|
|
15
15
|
tags: string[];
|
|
16
16
|
// Secret id to a path on the host holding its contents
|
|
17
17
|
secrets?: Record<string, string>;
|
|
18
|
+
// Forwards the agent this shell holds into the build
|
|
19
|
+
ssh?: boolean;
|
|
18
20
|
// Stops at the builder stage, for the image a step before the swap runs in
|
|
19
21
|
target?: string;
|
|
20
22
|
};
|
|
@@ -174,6 +176,9 @@ class DockerImage {
|
|
|
174
176
|
...Object.entries(spec.secrets ?? {}).map(
|
|
175
177
|
([id, path]) => `--secret id=${id},src=${path}`,
|
|
176
178
|
),
|
|
179
|
+
// Forwards this shell's agent into the build, which is how a dependency
|
|
180
|
+
// fetched over ssh is authenticated
|
|
181
|
+
...(spec.ssh ? ["--ssh default"] : []),
|
|
177
182
|
...(spec.target ? [`--target ${spec.target}`] : []),
|
|
178
183
|
spec.context,
|
|
179
184
|
].join(" ");
|
package/src/dockerfile.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { dirname } from "node:path";
|
|
2
|
+
|
|
1
3
|
import { appRoot, destinationFor, mountFor, rootedAt } from "./layout.js";
|
|
2
4
|
import type { BuildSpec, CarryPath } from "./types.js";
|
|
3
5
|
|
|
@@ -20,6 +22,10 @@ export type DockerfileContext = {
|
|
|
20
22
|
envSecret: string;
|
|
21
23
|
// Container path to secret id, for credentials that have to be files
|
|
22
24
|
fileSecrets: Record<string, string>;
|
|
25
|
+
// Whether an agent is there to forward. A dependency fetched over ssh during
|
|
26
|
+
// the install needs one, and asking for a mount nothing is behind fails the
|
|
27
|
+
// build outright
|
|
28
|
+
agent?: boolean;
|
|
23
29
|
// Where the app sits in the repository, when it is not the repository. The
|
|
24
30
|
// build steps and the shipped command run there, and every /app path the
|
|
25
31
|
// spec names is read against it
|
|
@@ -73,7 +79,18 @@ function builderStage(spec: BuildSpec, context: DockerfileContext) {
|
|
|
73
79
|
lines.push(`ENV NODE_ENV=${context.environment}`);
|
|
74
80
|
lines.push("ENV NEXT_TELEMETRY_DISABLED=1");
|
|
75
81
|
|
|
76
|
-
|
|
82
|
+
// accept-new rather than no, the same as the checkout on the host. Without a
|
|
83
|
+
// known_hosts of its own a build has nothing to compare against, and a first
|
|
84
|
+
// sight it refuses is a dependency it cannot fetch
|
|
85
|
+
if (context.agent) {
|
|
86
|
+
lines.push(`ENV GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new"`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// The agent reaches the install and the steps, which is where a manifest's
|
|
90
|
+
// git dependencies are resolved
|
|
91
|
+
const agent = context.agent ? "--mount=type=ssh " : "";
|
|
92
|
+
|
|
93
|
+
if (spec.dependencies) lines.push(...dependencyLayer(spec, `${agent}${mounts}`));
|
|
77
94
|
|
|
78
95
|
// Submodules are already in the context: the checkout on the host resolved
|
|
79
96
|
// them, so nothing in the build needs an agent or a .git directory
|
|
@@ -93,7 +110,7 @@ function builderStage(spec: BuildSpec, context: DockerfileContext) {
|
|
|
93
110
|
// written exactly as it would be typed. Quoting it made the whole command one
|
|
94
111
|
// word, and the shell went looking for a program by that name
|
|
95
112
|
for (const step of spec.steps) {
|
|
96
|
-
lines.push(`RUN ${env}${mounts}${step}`);
|
|
113
|
+
lines.push(`RUN ${env}${agent}${mounts}${step}`);
|
|
97
114
|
}
|
|
98
115
|
|
|
99
116
|
return lines;
|
|
@@ -134,9 +151,12 @@ function runtimeStage(spec: BuildSpec, context: DockerfileContext) {
|
|
|
134
151
|
lines.push(optional ? copyOrSkip(from, to, BUILDER_STAGE) : copy(from, to, BUILDER_STAGE));
|
|
135
152
|
}
|
|
136
153
|
|
|
154
|
+
// The directory first: cp does not make one, and a credential belongs
|
|
155
|
+
// somewhere of its own more often than beside the code
|
|
137
156
|
for (const [path, secret] of Object.entries(context.fileSecrets)) {
|
|
138
157
|
lines.push(
|
|
139
|
-
`RUN --mount=type=secret,id=${secret}
|
|
158
|
+
`RUN --mount=type=secret,id=${secret} mkdir -p ${dirname(path)} && ` +
|
|
159
|
+
`cp /run/secrets/${secret} ${path}`,
|
|
140
160
|
);
|
|
141
161
|
}
|
|
142
162
|
|
package/src/environment.ts
CHANGED
|
@@ -14,7 +14,7 @@ export async function envFileFor(app: AppSpec, context: Context) {
|
|
|
14
14
|
if (!app.secrets) return undefined;
|
|
15
15
|
|
|
16
16
|
const contents = await readEnv(app.secrets, context.secrets);
|
|
17
|
-
return await context.host.write(`apps/${app.name}/env`, contents);
|
|
17
|
+
return await context.host.write(`apps/${app.name}/env`, dockerEnv(contents));
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
// The flag, or nothing at all for an app that names no secrets
|
|
@@ -22,3 +22,73 @@ export async function envFlags(app: AppSpec, context: Context) {
|
|
|
22
22
|
const path = await envFileFor(app, context);
|
|
23
23
|
return path ? [`--env-file ${path}`] : [];
|
|
24
24
|
}
|
|
25
|
+
|
|
26
|
+
// A vault holds dotenv, and docker's env file is not dotenv: a quote is part of
|
|
27
|
+
// the value there, so DATABASE_URL="postgres://..." reaches the process with the
|
|
28
|
+
// quote still on it and every url parser rejects the scheme
|
|
29
|
+
export function dockerEnv(contents: string) {
|
|
30
|
+
const lines = Object.entries(parseEnv(contents)).map(([key, value]) => {
|
|
31
|
+
// The format is one variable per line, so there is no spelling of this that
|
|
32
|
+
// docker would read back. Better to name the variable than to truncate it
|
|
33
|
+
if (value.includes("\n")) {
|
|
34
|
+
throw new Error(
|
|
35
|
+
`${key} spans more than one line, which a docker env file cannot carry. ` +
|
|
36
|
+
"Pass it as a file secret instead",
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return `${key}=${value}`;
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
return lines.length > 0 ? `${lines.join("\n")}\n` : "";
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Enough of the format to carry a credential: KEY=value, quotes stripped,
|
|
47
|
+
// blank lines and comments skipped. A quoted value may run past the end of its
|
|
48
|
+
// line, because a key or a certificate is written the way it was generated
|
|
49
|
+
export function parseEnv(contents: string) {
|
|
50
|
+
const values: Record<string, string> = {};
|
|
51
|
+
const lines = contents.split("\n");
|
|
52
|
+
|
|
53
|
+
for (let at = 0; at < lines.length; at += 1) {
|
|
54
|
+
const text = (lines[at] ?? "").trim();
|
|
55
|
+
if (!text || text.startsWith("#")) continue;
|
|
56
|
+
|
|
57
|
+
const split = text.indexOf("=");
|
|
58
|
+
if (split < 1) continue;
|
|
59
|
+
|
|
60
|
+
const key = text.slice(0, split).replace(/^export\s+/, "").trim();
|
|
61
|
+
const start = text.slice(split + 1).trim();
|
|
62
|
+
const edge = start[0];
|
|
63
|
+
|
|
64
|
+
if (edge !== '"' && edge !== "'") {
|
|
65
|
+
values[key] = start;
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (start.length > 1 && start.endsWith(edge)) {
|
|
70
|
+
values[key] = start.slice(1, -1);
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// The quote never closed on its own line. Take whole lines until one closes
|
|
75
|
+
// it, so a pem arrives whole rather than as its first line
|
|
76
|
+
const held = [start.slice(1)];
|
|
77
|
+
|
|
78
|
+
while (at + 1 < lines.length) {
|
|
79
|
+
at += 1;
|
|
80
|
+
const next = (lines[at] ?? "").trimEnd();
|
|
81
|
+
|
|
82
|
+
if (next.endsWith(edge)) {
|
|
83
|
+
held.push(next.slice(0, -1));
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
held.push(next);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
values[key] = held.join("\n");
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return values;
|
|
94
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -9,18 +9,19 @@ export * from "./host.js";
|
|
|
9
9
|
export * from "./layout.js";
|
|
10
10
|
export * from "./localHost.js";
|
|
11
11
|
export * from "./log.js";
|
|
12
|
-
export * from "./nginx.js";
|
|
13
12
|
export * from "./pipeline.js";
|
|
14
13
|
export * from "./plugin.js";
|
|
15
14
|
export * from "./plugins/index.js";
|
|
16
15
|
export * from "./recover.js";
|
|
17
16
|
export * from "./presets/index.js";
|
|
17
|
+
export * from "./secrets/manager.js";
|
|
18
18
|
export * from "./secrets/refs.js";
|
|
19
19
|
export * from "./secrets/store.js";
|
|
20
20
|
export * from "./shell.js";
|
|
21
21
|
export * from "./services/ensure.js";
|
|
22
22
|
export * from "./services/index.js";
|
|
23
23
|
export * from "./services/planned.js";
|
|
24
|
+
export * from "./services/proxy.js";
|
|
24
25
|
export * from "./source.js";
|
|
25
26
|
export * from "./sshHost.js";
|
|
26
27
|
export * from "./steps.js";
|
package/src/log.ts
CHANGED
|
@@ -14,6 +14,9 @@ export type Task = {
|
|
|
14
14
|
};
|
|
15
15
|
|
|
16
16
|
export type Log = ((message: string) => void) & {
|
|
17
|
+
// Every point the run will walk, said once before the first of them starts.
|
|
18
|
+
// A view that knows the whole list can show what has not begun
|
|
19
|
+
plan?(points: string[]): void;
|
|
17
20
|
// Something the run survived, but that the person should know happened
|
|
18
21
|
warn(message: string): void;
|
|
19
22
|
// The reason a deploy is about to stop, or did
|
|
@@ -31,6 +34,7 @@ const NOTHING: Task = {
|
|
|
31
34
|
};
|
|
32
35
|
|
|
33
36
|
export const silent: Log = Object.assign(() => {}, {
|
|
37
|
+
plan: () => {},
|
|
34
38
|
warn: () => {},
|
|
35
39
|
fail: () => {},
|
|
36
40
|
done: () => {},
|
package/src/pipeline.ts
CHANGED
|
@@ -207,6 +207,10 @@ export async function runPipeline(
|
|
|
207
207
|
const ordered = sequence(steps, RUNS[run]);
|
|
208
208
|
const plan: Plan = { config: setting.config, environment: setting.environment };
|
|
209
209
|
|
|
210
|
+
// What the run will walk, said before any of it does. Nothing depends on it
|
|
211
|
+
// being heard: a log without a view has nothing to do with the list
|
|
212
|
+
setting.log.plan?.(ordered.map((step) => step.point));
|
|
213
|
+
|
|
210
214
|
// Every check before any step, so a run that is going to fail on a config
|
|
211
215
|
// mistake fails before it has created a network or built an image. Nothing
|
|
212
216
|
// wraps what a check throws: it is about the config, and the config is what
|
package/src/plugins/bitwarden.ts
CHANGED
|
@@ -1,26 +1,34 @@
|
|
|
1
1
|
import { definePlugin, type OpenContext, type Plugin } from "../plugin.js";
|
|
2
|
+
import { secretsManagerStore } from "../secrets/manager.js";
|
|
2
3
|
import { bitwardenStore } from "../secrets/store.js";
|
|
3
4
|
import type { SecretRef } from "../types.js";
|
|
4
5
|
|
|
5
6
|
// Redkite's own vault, and a plugin like any other: a deployment that does not
|
|
6
7
|
// register it cannot resolve a bitwarden ref, and says so before it builds.
|
|
8
|
+
//
|
|
9
|
+
// Bitwarden is two services rather than one. Secrets Manager holds secrets for
|
|
10
|
+
// machines and opens with an access token, which is what a deploy wants. The
|
|
11
|
+
// password manager holds a person's vault and wants a session obtained by
|
|
12
|
+
// unlocking it. The ids are different, the CLIs are different, and giving one
|
|
13
|
+
// service's credential to the other is how a deploy ends up waiting on a
|
|
14
|
+
// password prompt nobody can see.
|
|
7
15
|
|
|
8
|
-
// The one variable
|
|
9
|
-
// job that unlocked once and reuses it across several deploys is the case
|
|
16
|
+
// The one variable either side reads, so a job sets one thing whichever it is
|
|
10
17
|
const KEY = "BW_KEY";
|
|
11
18
|
|
|
12
19
|
export type BitwardenOptions = {
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
20
|
+
// Which Bitwarden this deployment reads, and what opens it.
|
|
21
|
+
//
|
|
22
|
+
// true, the default, is Secrets Manager: BW_KEY is the access token. A
|
|
23
|
+
// string is that token given directly, for a config naming its own variable.
|
|
24
|
+
//
|
|
25
|
+
// false is the password manager: BW_KEY is a session from bw unlock --raw,
|
|
26
|
+
// and without one the api credentials obtain one.
|
|
18
27
|
secrets?: boolean | string;
|
|
19
28
|
};
|
|
20
29
|
|
|
21
30
|
function vault(options: BitwardenOptions = {}): Plugin {
|
|
22
31
|
const secrets = options.secrets ?? true;
|
|
23
|
-
if (secrets === false) return definePlugin({ name: "bitwarden" });
|
|
24
32
|
|
|
25
33
|
return definePlugin({
|
|
26
34
|
name: "bitwarden",
|
|
@@ -28,8 +36,26 @@ function vault(options: BitwardenOptions = {}): Plugin {
|
|
|
28
36
|
});
|
|
29
37
|
}
|
|
30
38
|
|
|
31
|
-
async function open(secrets:
|
|
32
|
-
|
|
39
|
+
async function open(secrets: boolean | string, context: OpenContext) {
|
|
40
|
+
if (secrets === false) return await passwords(context);
|
|
41
|
+
|
|
42
|
+
const token = typeof secrets === "string" ? secrets : process.env[KEY];
|
|
43
|
+
|
|
44
|
+
if (!token) {
|
|
45
|
+
throw new Error(
|
|
46
|
+
`${KEY} is not set, and this deployment reads from Bitwarden Secrets ` +
|
|
47
|
+
"Manager. It is an access token, not a password. A deployment reading " +
|
|
48
|
+
"the password manager instead says bitwarden({ secrets: false })",
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return await secretsManagerStore({ token, detail: context.detail });
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// The personal vault. A session skips the login and the unlock, which is two
|
|
56
|
+
// fewer round trips and the only way a job without a master password gets in
|
|
57
|
+
async function passwords(context: OpenContext) {
|
|
58
|
+
const session = process.env[KEY];
|
|
33
59
|
if (session) return await bitwardenStore({ session, detail: context.detail });
|
|
34
60
|
|
|
35
61
|
return await bitwardenStore({
|
|
@@ -45,7 +71,8 @@ function required(name: string) {
|
|
|
45
71
|
if (value) return value;
|
|
46
72
|
|
|
47
73
|
throw new Error(
|
|
48
|
-
`Neither ${KEY} nor ${name} is set, and this deployment reads its
|
|
74
|
+
`Neither ${KEY} nor ${name} is set, and this deployment reads its ` +
|
|
75
|
+
"environment from the Bitwarden password manager",
|
|
49
76
|
);
|
|
50
77
|
}
|
|
51
78
|
|
package/src/presets/nextApp.ts
CHANGED
|
@@ -57,10 +57,11 @@ export function nextApp(options: NextAppOptions = {}): BuildSpec {
|
|
|
57
57
|
// whatever the repository put there, and plenty of apps have none
|
|
58
58
|
carry: ["/app/.next/static", { path: "/app/public", optional: true }],
|
|
59
59
|
entrypoint: start("node server.js"),
|
|
60
|
-
// app
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
|
|
60
|
+
// next-app is Next's own build cache, which only ever makes a build
|
|
61
|
+
// slower when it is dropped. node_modules is not here for the same
|
|
62
|
+
// reason it is not in nodeApp: a dropped mount and a kept layer is a
|
|
63
|
+
// build that cannot find what the manifest lists
|
|
64
|
+
caches: ["yarn", "npm", "next-app"],
|
|
64
65
|
}),
|
|
65
66
|
preset: "nextApp",
|
|
66
67
|
// The standalone tree traces from the workspace root, so an app in a
|
package/src/presets/nodeApp.ts
CHANGED
|
@@ -31,13 +31,22 @@ export function nodeApp(options: NodeAppOptions): BuildSpec {
|
|
|
31
31
|
output: options.output,
|
|
32
32
|
carry: options.carry ?? [],
|
|
33
33
|
entrypoint: options.entrypoint,
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
//
|
|
37
|
-
|
|
34
|
+
// The package manager's own cache, and nothing else. node_modules used to
|
|
35
|
+
// be a mount too, and BuildKit drops a cache mount whenever it likes while
|
|
36
|
+
// keeping the layer that filled it: the install then does not re-run and
|
|
37
|
+
// the modules are simply gone. What that looks like is a build saying it
|
|
38
|
+
// cannot find a dependency the manifest plainly lists.
|
|
39
|
+
//
|
|
40
|
+
// Installed into the layer instead, they are there for every step below,
|
|
41
|
+
// for a migration run in this image afterwards, and for a check. The
|
|
42
|
+
// manager's cache still makes the install itself fast
|
|
43
|
+
caches: options.caches ?? ["yarn", "npm"],
|
|
38
44
|
submodules: options.submodules ?? false,
|
|
39
|
-
// The checkout arrives complete, so this is what a build step itself needs
|
|
40
|
-
|
|
45
|
+
// The checkout arrives complete, so this is what a build step itself needs.
|
|
46
|
+
// openssh-client is for the dependencies it resolves rather than the
|
|
47
|
+
// repository: a git+ssh entry in a manifest is fetched during the install,
|
|
48
|
+
// and git cannot do that without an ssh to run
|
|
49
|
+
aptPackages: ["git", "openssh-client"],
|
|
41
50
|
runtimePackages: ["curl"],
|
|
42
51
|
runtimeSteps: [],
|
|
43
52
|
};
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { chmodSync, existsSync } from "node:fs";
|
|
3
|
+
import { mkdir, rm, writeFile } from "node:fs/promises";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
|
|
7
|
+
import type { SecretStore } from "./refs.js";
|
|
8
|
+
|
|
9
|
+
// Bitwarden Secrets Manager, which is not the password manager: different
|
|
10
|
+
// service, different CLI, different credential. An access token opens a set of
|
|
11
|
+
// projects rather than a personal vault, which is what a deploy wants, and it
|
|
12
|
+
// is the whole of the authentication: nothing to log in to and nothing to
|
|
13
|
+
// unlock, so there is no session to obtain or lose.
|
|
14
|
+
|
|
15
|
+
// Not an npm package. The password manager's CLI is one and redkite installs
|
|
16
|
+
// it with npm; this one is a released binary, and the only thing on npm under
|
|
17
|
+
// its name belongs to somebody else entirely. Pinned for the same reason the
|
|
18
|
+
// other is: an unpinned CLI is a different program on a machine that has never
|
|
19
|
+
// run a deploy before
|
|
20
|
+
const VERSION = "2.1.0";
|
|
21
|
+
const RELEASES = "https://github.com/bitwarden/sdk-sm/releases/download";
|
|
22
|
+
const CLIS = join(homedir(), ".cache", "redkite", "cli");
|
|
23
|
+
|
|
24
|
+
// What the release calls each platform. Anything not here has to bring its own
|
|
25
|
+
const TARGETS: Record<string, string> = {
|
|
26
|
+
"linux-x64": "x86_64-unknown-linux-gnu",
|
|
27
|
+
"linux-arm64": "aarch64-unknown-linux-gnu",
|
|
28
|
+
"darwin-x64": "x86_64-apple-darwin",
|
|
29
|
+
"darwin-arm64": "aarch64-apple-darwin",
|
|
30
|
+
"win32-x64": "x86_64-pc-windows-msvc",
|
|
31
|
+
"win32-arm64": "aarch64-pc-windows-msvc",
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export type ManagerCredentials = {
|
|
35
|
+
// The access token, which is the only credential this service has
|
|
36
|
+
token: string;
|
|
37
|
+
detail?: (message: string) => void;
|
|
38
|
+
// Injected so the argv this builds is asserted on rather than trusted
|
|
39
|
+
bws?: Bws;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export type Bws = (args: string[], env: Record<string, string>) => Promise<string>;
|
|
43
|
+
|
|
44
|
+
export async function secretsManagerStore(
|
|
45
|
+
credentials: ManagerCredentials,
|
|
46
|
+
): Promise<SecretStore> {
|
|
47
|
+
const detail = credentials.detail ?? (() => {});
|
|
48
|
+
const bws = credentials.bws ?? (await resolve(detail));
|
|
49
|
+
|
|
50
|
+
// Fetched once each. A second read of the same item during a deploy is the
|
|
51
|
+
// same answer, and this avoids a process per call site
|
|
52
|
+
const cache = new Map<string, Promise<string>>();
|
|
53
|
+
|
|
54
|
+
const fetch = async (id: string) => {
|
|
55
|
+
detail(`reading ${id.slice(0, 8)}`);
|
|
56
|
+
|
|
57
|
+
const answer = await bws(["secret", "get", id, "--output", "json"], {
|
|
58
|
+
BWS_ACCESS_TOKEN: credentials.token,
|
|
59
|
+
}).catch((error: unknown) => {
|
|
60
|
+
throw new Error(`Could not read ${id} from Bitwarden Secrets Manager: ${messageOf(error)}`);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
return valueOf(answer, id);
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
return {
|
|
67
|
+
read: async (id) => {
|
|
68
|
+
const pending = cache.get(id) ?? fetch(id);
|
|
69
|
+
cache.set(id, pending);
|
|
70
|
+
return await pending;
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// The secret's value, which is the whole of what a ref points at. Everything
|
|
76
|
+
// else the call answers with is about the item rather than in it
|
|
77
|
+
function valueOf(answer: string, id: string) {
|
|
78
|
+
try {
|
|
79
|
+
const parsed = JSON.parse(answer) as { value?: unknown };
|
|
80
|
+
if (typeof parsed.value === "string") return parsed.value;
|
|
81
|
+
} catch {
|
|
82
|
+
// Falls through to the same refusal as a shape without a value
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
throw new Error(`Bitwarden Secrets Manager answered for ${id} without a value`);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async function resolve(detail: (message: string) => void): Promise<Bws> {
|
|
89
|
+
const file = await binary(detail);
|
|
90
|
+
return async (args, env) => await run(file, args, env);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function binary(detail: (message: string) => void) {
|
|
94
|
+
const override = process.env["REDKITE_BWS_BIN"];
|
|
95
|
+
if (override) return override;
|
|
96
|
+
|
|
97
|
+
const onPath = await run("bws", ["--version"], {}).then(
|
|
98
|
+
() => true,
|
|
99
|
+
() => false,
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
if (onPath) return "bws";
|
|
103
|
+
|
|
104
|
+
const directory = join(CLIS, `bws-${VERSION}`);
|
|
105
|
+
const installed = join(directory, process.platform === "win32" ? "bws.exe" : "bws");
|
|
106
|
+
if (existsSync(installed)) return installed;
|
|
107
|
+
|
|
108
|
+
return await install(directory, installed, detail);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Downloaded once per machine per pinned version, the way the other CLI is
|
|
112
|
+
// installed once. The archive is a zip with one file in it, and unzip is the
|
|
113
|
+
// one thing on every platform that reads one
|
|
114
|
+
async function install(directory: string, installed: string, detail: (m: string) => void) {
|
|
115
|
+
const target = TARGETS[`${process.platform}-${process.arch}`];
|
|
116
|
+
|
|
117
|
+
if (!target) {
|
|
118
|
+
throw new Error(
|
|
119
|
+
`Bitwarden publishes no bws for ${process.platform}-${process.arch}. ` +
|
|
120
|
+
"Install it yourself and point REDKITE_BWS_BIN at it",
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const url = `${RELEASES}/bws-v${VERSION}/bws-${target}-${VERSION}.zip`;
|
|
125
|
+
detail(`installing bws ${VERSION}, once for this machine`);
|
|
126
|
+
|
|
127
|
+
await mkdir(directory, { recursive: true });
|
|
128
|
+
const archive = join(directory, "bws.zip");
|
|
129
|
+
|
|
130
|
+
const answer = await globalThis.fetch(url);
|
|
131
|
+
if (!answer.ok) throw new Error(`Could not download bws from ${url}: ${answer.status}`);
|
|
132
|
+
|
|
133
|
+
await writeFile(archive, Buffer.from(await answer.arrayBuffer()));
|
|
134
|
+
await run("unzip", ["-o", "-q", archive, "-d", directory], {}).catch((error: unknown) => {
|
|
135
|
+
throw new Error(`Could not unpack bws: ${messageOf(error)}`);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
await rm(archive, { force: true });
|
|
139
|
+
|
|
140
|
+
// The zip carries no mode anything trusts, so it arrives unrunnable
|
|
141
|
+
chmodSync(installed, 0o755);
|
|
142
|
+
|
|
143
|
+
return installed;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Nothing on stdin, and the output kept exactly as it came. The same reasoning
|
|
147
|
+
// as the password manager's runner: a question nobody can answer is a deploy
|
|
148
|
+
// that waits for ever
|
|
149
|
+
function run(file: string, args: string[], env: Record<string, string>) {
|
|
150
|
+
return new Promise<string>((resolve_, reject) => {
|
|
151
|
+
const child = spawn(file, args, {
|
|
152
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
153
|
+
env: { ...process.env, ...env },
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
let stdout = "";
|
|
157
|
+
let stderr = "";
|
|
158
|
+
|
|
159
|
+
child.stdout.on("data", (chunk: Buffer) => (stdout += chunk.toString()));
|
|
160
|
+
child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString()));
|
|
161
|
+
child.on("error", reject);
|
|
162
|
+
|
|
163
|
+
child.on("close", (code) => {
|
|
164
|
+
if (code === 0) return resolve_(stdout);
|
|
165
|
+
reject(new Error(stderr.trim() || stdout.trim() || `${file} exited ${code}`));
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function messageOf(error: unknown) {
|
|
171
|
+
return error instanceof Error ? error.message : String(error);
|
|
172
|
+
}
|