rollbridge 0.1.70 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -63
- package/TODO.md +0 -13
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +65 -387
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +22 -212
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -435
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +5 -119
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +21 -125
- package/test/owner-replacement.test.js +14 -175
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +72 -737
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
|
@@ -1,297 +0,0 @@
|
|
|
1
|
-
// @ts-check
|
|
2
|
-
|
|
3
|
-
import fs from "node:fs/promises"
|
|
4
|
-
import os from "node:os"
|
|
5
|
-
import path from "node:path"
|
|
6
|
-
import {describe, expect, test} from "@velocious/testing"
|
|
7
|
-
import {normalizeConfig} from "../src/config.js"
|
|
8
|
-
import {readComposeLedger, rollComposeUnit, runComposeCommand} from "../src/compose-unit.js"
|
|
9
|
-
|
|
10
|
-
// Real Docker Compose proof. It is opt-in because CI lanes are not guaranteed a reachable
|
|
11
|
-
// daemon: ROLLBRIDGE_COMPOSE_DOCKER_TEST=1 plus DOCKER_HOST (or the local socket) enables it.
|
|
12
|
-
// A published port is bound by the machine running the daemon, so when DOCKER_HOST points
|
|
13
|
-
// elsewhere the readiness URL has to name that machine: ROLLBRIDGE_COMPOSE_DOCKER_READY_HOST=dind.
|
|
14
|
-
const enabled = process.env.ROLLBRIDGE_COMPOSE_DOCKER_TEST === "1"
|
|
15
|
-
const readyHost = process.env.ROLLBRIDGE_COMPOSE_DOCKER_READY_HOST ?? "127.0.0.1"
|
|
16
|
-
const portBase = 20000 + (process.pid % 4000)
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* @typedef {import("../src/config.js").RollbridgeConfig & {compose: import("../src/config.js").ComposeUnitConfig[]}} ComposeTestConfig
|
|
20
|
-
*/
|
|
21
|
-
|
|
22
|
-
describe("compose-docker", () => {
|
|
23
|
-
|
|
24
|
-
const PROJECT = `rollbridge-compose-proof-${process.pid}`
|
|
25
|
-
const V1 = "rollbridge-proof-gateway:v1"
|
|
26
|
-
const V2 = "rollbridge-proof-gateway:v2"
|
|
27
|
-
const V3 = "rollbridge-proof-gateway:v3-unhealthy"
|
|
28
|
-
|
|
29
|
-
// The gateway records what it does into /work/timeline, a shared volume, so the ordering survives
|
|
30
|
-
// container replacement. Drain and settle markers are container-local, so every container has to
|
|
31
|
-
// drain and settle for itself before a roll is allowed to recreate it.
|
|
32
|
-
const DOCKERFILE = [
|
|
33
|
-
"FROM alpine:3.20",
|
|
34
|
-
"ARG VERSION=v1",
|
|
35
|
-
"ARG FAIL_ACTIVATION=0",
|
|
36
|
-
"ENV VERSION=$VERSION FAIL_ACTIVATION=$FAIL_ACTIVATION SETTLE_SECONDS=2",
|
|
37
|
-
"RUN apk add --no-cache busybox-extras && mkdir -p /srv /work && printf '%s\\n' \\",
|
|
38
|
-
" '#!/bin/sh' \\",
|
|
39
|
-
" 'set -eu' \\",
|
|
40
|
-
" 'log() { printf \"%s %s\\n\" \"$VERSION\" \"$1\" >> /work/timeline; }' \\",
|
|
41
|
-
" 'if [ \"$FAIL_ACTIVATION\" = 1 ]; then log start; exec sleep 3600; fi' \\",
|
|
42
|
-
" 'echo ready > /srv/ready' \\",
|
|
43
|
-
" '/usr/sbin/httpd -p 7421 -h /srv &' \\",
|
|
44
|
-
" 'log start' \\",
|
|
45
|
-
" 'while :; do' \\",
|
|
46
|
-
" ' if [ -f /tmp/draining ]; then log drain; break; fi' \\",
|
|
47
|
-
" ' log admit' \\",
|
|
48
|
-
" ' sleep 0.3' \\",
|
|
49
|
-
" 'done' \\",
|
|
50
|
-
" 'sleep \"$SETTLE_SECONDS\"' \\",
|
|
51
|
-
" 'touch /tmp/settled' \\",
|
|
52
|
-
" 'log settled' \\",
|
|
53
|
-
" 'while :; do sleep 1; done' \\",
|
|
54
|
-
" > /usr/local/bin/app && chmod +x /usr/local/bin/app",
|
|
55
|
-
"ENTRYPOINT [\"/bin/sh\", \"/usr/local/bin/app\"]",
|
|
56
|
-
"HEALTHCHECK --interval=1s --timeout=2s --retries=40 CMD wget -q -O /dev/null http://127.0.0.1:7421/ready || exit 1",
|
|
57
|
-
""
|
|
58
|
-
].join("\n")
|
|
59
|
-
|
|
60
|
-
/**
|
|
61
|
-
* @param {string} projectDirectory - Directory to write the Compose project into.
|
|
62
|
-
* @param {number} hostPort - Host port the gateway publishes.
|
|
63
|
-
* @returns {Promise<string[]>} Compose global arguments for the project.
|
|
64
|
-
*/
|
|
65
|
-
async function writeComposeProject(projectDirectory, hostPort) {
|
|
66
|
-
await fs.writeFile(path.join(projectDirectory, "gateway.compose.yml"), [
|
|
67
|
-
`name: ${PROJECT}`,
|
|
68
|
-
"services:",
|
|
69
|
-
" gateway:",
|
|
70
|
-
` image: ${V1}`,
|
|
71
|
-
" init: true",
|
|
72
|
-
" ports:",
|
|
73
|
-
` - "${hostPort}:7421"`,
|
|
74
|
-
" volumes:",
|
|
75
|
-
" - gateway-work:/work",
|
|
76
|
-
"volumes:",
|
|
77
|
-
" gateway-work:",
|
|
78
|
-
""
|
|
79
|
-
].join("\n"))
|
|
80
|
-
|
|
81
|
-
return ["docker", "compose", "--project-name", PROJECT, "-f", path.join(projectDirectory, "gateway.compose.yml")]
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
/**
|
|
85
|
-
* @param {string[]} composeArgs - Compose global arguments.
|
|
86
|
-
* @returns {Promise<string[]>} Timeline lines recorded by the gateway containers.
|
|
87
|
-
*/
|
|
88
|
-
async function readTimeline(composeArgs) {
|
|
89
|
-
const result = await runComposeCommand({args: [...composeArgs, "exec", "-T", "gateway", "cat", "/work/timeline"], timeoutMs: 30000})
|
|
90
|
-
|
|
91
|
-
if (result.code !== 0) throw new Error(`could not read the gateway timeline: ${result.stderr.trim()}`)
|
|
92
|
-
|
|
93
|
-
return result.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
/**
|
|
97
|
-
* @param {string[]} lines - Timeline lines.
|
|
98
|
-
* @param {string} line - Line to locate.
|
|
99
|
-
* @returns {number} Index of the last occurrence, or -1 when the line is absent.
|
|
100
|
-
*/
|
|
101
|
-
function lastLine(lines, line) {
|
|
102
|
-
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
103
|
-
if (lines[index] === line) return index
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
return -1
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/**
|
|
110
|
-
* @param {string[]} composeArgs - Compose global arguments.
|
|
111
|
-
* @returns {Promise<string>} Image of the container Compose currently lists for the gateway.
|
|
112
|
-
*/
|
|
113
|
-
async function liveImage(composeArgs) {
|
|
114
|
-
const listed = await runComposeCommand({args: [...composeArgs, "ps", "-q", "gateway"], timeoutMs: 15000})
|
|
115
|
-
const inspected = await runComposeCommand({args: ["docker", "inspect", "--format", "{{.Config.Image}}", listed.stdout.trim()], timeoutMs: 15000})
|
|
116
|
-
|
|
117
|
-
return inspected.stdout.trim()
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* @param {string[]} composeArgs - Compose global arguments.
|
|
122
|
-
* @returns {Promise<void>} Resolves once the gateway container reports healthy.
|
|
123
|
-
*/
|
|
124
|
-
async function waitHealthy(composeArgs) {
|
|
125
|
-
const deadline = Date.now() + 60000
|
|
126
|
-
|
|
127
|
-
while (Date.now() < deadline) {
|
|
128
|
-
const listed = await runComposeCommand({args: [...composeArgs, "ps", "-q", "gateway"], timeoutMs: 15000})
|
|
129
|
-
const containerId = listed.stdout.trim().split("\n").filter(Boolean).at(-1)
|
|
130
|
-
|
|
131
|
-
if (containerId !== undefined) {
|
|
132
|
-
const inspected = await runComposeCommand({args: ["docker", "inspect", "--format", "{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}", containerId], timeoutMs: 15000})
|
|
133
|
-
const [state, health] = inspected.stdout.trim().split("|")
|
|
134
|
-
|
|
135
|
-
if (state === "running" && (health === "" || health === "healthy")) return
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
await new Promise((resolve) => setTimeout(resolve, 500))
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
throw new Error("the proof gateway never became healthy")
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/**
|
|
145
|
-
* @param {string} projectDirectory - Build context.
|
|
146
|
-
* @param {string} tag - Image tag to build.
|
|
147
|
-
* @param {string} version - Version the container logs into the timeline.
|
|
148
|
-
* @param {boolean} [unhealthy] - Build a container that never passes its healthcheck.
|
|
149
|
-
* @returns {Promise<void>} Resolves once the image is built.
|
|
150
|
-
*/
|
|
151
|
-
async function buildImage(projectDirectory, tag, version, unhealthy = false) {
|
|
152
|
-
const result = await runComposeCommand({
|
|
153
|
-
args: ["docker", "build", "-t", tag, "--build-arg", `VERSION=${version}`, "--build-arg", `FAIL_ACTIVATION=${unhealthy ? "1" : "0"}`, "-f", path.join(projectDirectory, "Dockerfile"), projectDirectory],
|
|
154
|
-
timeoutMs: 300000
|
|
155
|
-
})
|
|
156
|
-
|
|
157
|
-
if (result.code !== 0) throw new Error(`docker build failed for ${tag}: ${result.stderr.trim()}`)
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
test("rolls a real Compose service drain-first, retains the previous tag, and rolls back when activation fails", {timeoutMs: 300000}, async () => {
|
|
161
|
-
if (!enabled) return
|
|
162
|
-
|
|
163
|
-
const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-docker-"))
|
|
164
|
-
const hostPort = portBase
|
|
165
|
-
const composeArgs = await writeComposeProject(projectDirectory, hostPort)
|
|
166
|
-
|
|
167
|
-
await fs.writeFile(path.join(projectDirectory, "Dockerfile"), DOCKERFILE)
|
|
168
|
-
await buildImage(projectDirectory, V1, "v1")
|
|
169
|
-
await buildImage(projectDirectory, V2, "v2")
|
|
170
|
-
await buildImage(projectDirectory, V3, "v3", true)
|
|
171
|
-
|
|
172
|
-
const config = /** @type {ComposeTestConfig} */ (normalizeConfig({
|
|
173
|
-
application: "compose-docker-proof",
|
|
174
|
-
compose: [{
|
|
175
|
-
activation: {intervalMs: 500, readyUrl: `http://${readyHost}:${hostPort}/ready`, timeoutMs: 60000},
|
|
176
|
-
composeFile: "gateway.compose.yml",
|
|
177
|
-
drain: {command: `${composeArgs.join(" ")} exec -T gateway touch /tmp/draining`, timeoutMs: 30000},
|
|
178
|
-
id: "gateway",
|
|
179
|
-
image: V2,
|
|
180
|
-
inFlight: {command: `${composeArgs.join(" ")} exec -T gateway test -f /tmp/settled`, intervalMs: 500, timeoutMs: 30000},
|
|
181
|
-
projectDirectory,
|
|
182
|
-
recreateTimeoutMs: 120000,
|
|
183
|
-
service: "gateway",
|
|
184
|
-
stopGracePeriodMs: 10000
|
|
185
|
-
}],
|
|
186
|
-
composeStatePath: path.join(projectDirectory, "compose.state.json"),
|
|
187
|
-
control: {path: path.join(projectDirectory, "rollbridge.sock")},
|
|
188
|
-
processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
189
|
-
proxy: {host: "127.0.0.1", port: 8182}
|
|
190
|
-
}))
|
|
191
|
-
const unit = config.compose[0]
|
|
192
|
-
|
|
193
|
-
try {
|
|
194
|
-
const started = await runComposeCommand({args: [...composeArgs, "up", "-d", "--force-recreate", "gateway"], timeoutMs: 120000})
|
|
195
|
-
|
|
196
|
-
if (started.code !== 0) throw new Error(`compose up failed: ${started.stderr.trim()}`)
|
|
197
|
-
|
|
198
|
-
await waitHealthy(composeArgs)
|
|
199
|
-
|
|
200
|
-
const rolled = await rollComposeUnit({config, runCommand: runComposeCommand, unit})
|
|
201
|
-
const afterRoll = await readTimeline(composeArgs)
|
|
202
|
-
|
|
203
|
-
expect(rolled.status).toBe("rolled")
|
|
204
|
-
expect(rolled.stage).toBe("activation")
|
|
205
|
-
expect(rolled.image).toBe(V2)
|
|
206
|
-
expect(rolled.previousImage).toBe(V1)
|
|
207
|
-
expect(await liveImage(composeArgs)).toBe(V2)
|
|
208
|
-
|
|
209
|
-
// Quiesce, then the in-flight work settling, then the new image starting — in that order.
|
|
210
|
-
const v1Drain = afterRoll.indexOf("v1 drain")
|
|
211
|
-
const v1Settled = afterRoll.indexOf("v1 settled")
|
|
212
|
-
const v2Start = afterRoll.indexOf("v2 start")
|
|
213
|
-
|
|
214
|
-
expect(v1Drain >= 0).toBe(true)
|
|
215
|
-
expect(v1Settled > v1Drain).toBe(true)
|
|
216
|
-
expect(v2Start > v1Settled).toBe(true)
|
|
217
|
-
expect(afterRoll.slice(v1Drain + 1, v2Start).filter((line) => line === "v1 admit")).toEqual([])
|
|
218
|
-
expect((await readComposeLedger({config, unit})).generations.map((generation) => [generation.image, generation.status])).toEqual([[V2, "activated"]])
|
|
219
|
-
|
|
220
|
-
// Roll to an image whose healthcheck never passes: the previous tag must be serving again.
|
|
221
|
-
const rolledBack = await rollComposeUnit({config, image: V3, runCommand: runComposeCommand, unit})
|
|
222
|
-
const afterRollback = await readTimeline(composeArgs)
|
|
223
|
-
|
|
224
|
-
expect(rolledBack.status).toBe("rolled_back")
|
|
225
|
-
expect(rolledBack.stage).toBe("rollback")
|
|
226
|
-
expect(rolledBack.rollback).toMatchObject({image: V2, status: "rolled_back"})
|
|
227
|
-
expect(rolledBack.previousImage).toBe(V3)
|
|
228
|
-
|
|
229
|
-
const v2Drain = afterRollback.indexOf("v2 drain")
|
|
230
|
-
const v2Settled = afterRollback.indexOf("v2 settled")
|
|
231
|
-
const v3Start = afterRollback.indexOf("v3 start")
|
|
232
|
-
const v2Serving = lastLine(afterRollback, "v2 start")
|
|
233
|
-
|
|
234
|
-
expect(v2Drain > v2Start).toBe(true)
|
|
235
|
-
expect(v2Settled > v2Drain).toBe(true)
|
|
236
|
-
expect(v3Start > v2Settled).toBe(true)
|
|
237
|
-
expect(v2Serving > v3Start).toBe(true)
|
|
238
|
-
expect(afterRollback.slice(v2Drain + 1, v3Start).filter((line) => line === "v2 admit")).toEqual([])
|
|
239
|
-
|
|
240
|
-
expect(await liveImage(composeArgs)).toBe(V2)
|
|
241
|
-
await waitHealthy(composeArgs)
|
|
242
|
-
|
|
243
|
-
const ready = await runComposeCommand({args: [...composeArgs, "exec", "-T", "gateway", "wget", "-q", "-O", "-", "http://127.0.0.1:7421/ready"], timeoutMs: 15000})
|
|
244
|
-
|
|
245
|
-
expect(ready.stdout.trim()).toBe("ready")
|
|
246
|
-
expect((await readComposeLedger({config, unit})).generations.map((generation) => [generation.image, generation.status])).toEqual([[V2, "activated"], [V2, "rolled_back"]])
|
|
247
|
-
} finally {
|
|
248
|
-
await runComposeCommand({args: [...composeArgs, "down", "-v"], timeoutMs: 120000})
|
|
249
|
-
await runComposeCommand({args: ["docker", "rmi", "-f", V1, V2, V3], timeoutMs: 60000})
|
|
250
|
-
await fs.rm(projectDirectory, {force: true, recursive: true})
|
|
251
|
-
}
|
|
252
|
-
})
|
|
253
|
-
|
|
254
|
-
test("refuses to roll a real Compose service that publishes the Rollbridge proxy port", {timeoutMs: 180000}, async () => {
|
|
255
|
-
if (!enabled) return
|
|
256
|
-
|
|
257
|
-
const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-proxy-guard-"))
|
|
258
|
-
const proxyPort = portBase + 1
|
|
259
|
-
const composeArgs = await writeComposeProject(projectDirectory, proxyPort)
|
|
260
|
-
|
|
261
|
-
await fs.writeFile(path.join(projectDirectory, "Dockerfile"), DOCKERFILE)
|
|
262
|
-
await buildImage(projectDirectory, V1, "v1")
|
|
263
|
-
|
|
264
|
-
const config = /** @type {ComposeTestConfig} */ (normalizeConfig({
|
|
265
|
-
application: "compose-docker-proof",
|
|
266
|
-
compose: [{
|
|
267
|
-
activation: {intervalMs: 500, timeoutMs: 30000},
|
|
268
|
-
composeFile: "gateway.compose.yml",
|
|
269
|
-
id: "gateway",
|
|
270
|
-
image: V1,
|
|
271
|
-
projectDirectory,
|
|
272
|
-
service: "gateway",
|
|
273
|
-
stopGracePeriodMs: 10000
|
|
274
|
-
}],
|
|
275
|
-
composeStatePath: path.join(projectDirectory, "compose.state.json"),
|
|
276
|
-
control: {path: path.join(projectDirectory, "rollbridge.sock")},
|
|
277
|
-
processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
278
|
-
proxy: {host: "127.0.0.1", port: proxyPort}
|
|
279
|
-
}))
|
|
280
|
-
|
|
281
|
-
try {
|
|
282
|
-
const started = await runComposeCommand({args: [...composeArgs, "up", "-d", "--force-recreate", "gateway"], timeoutMs: 120000})
|
|
283
|
-
|
|
284
|
-
if (started.code !== 0) throw new Error(`compose up failed: ${started.stderr.trim()}`)
|
|
285
|
-
|
|
286
|
-
await waitHealthy(composeArgs)
|
|
287
|
-
|
|
288
|
-
await expect(rollComposeUnit({config, runCommand: runComposeCommand, unit: config.compose[0]})).rejects.toThrow(new RegExp(`publishes port ${proxyPort}, the Rollbridge proxy port`))
|
|
289
|
-
|
|
290
|
-
expect(await liveImage(composeArgs)).toBe(V1)
|
|
291
|
-
} finally {
|
|
292
|
-
await runComposeCommand({args: [...composeArgs, "down", "-v"], timeoutMs: 120000})
|
|
293
|
-
await runComposeCommand({args: ["docker", "rmi", "-f", V1], timeoutMs: 60000})
|
|
294
|
-
await fs.rm(projectDirectory, {force: true, recursive: true})
|
|
295
|
-
}
|
|
296
|
-
})
|
|
297
|
-
})
|