rollbridge 0.1.73 → 0.1.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-bounded-control-client-close.md +4 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +69 -3
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/control-client.js +32 -5
- package/src/daemon.js +448 -76
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +92 -9
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +452 -39
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/control-protocol.test.js +22 -0
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +149 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +151 -20
- package/test/owner-replacement.test.js +176 -15
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +447 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises"
|
|
4
|
+
import os from "node:os"
|
|
5
|
+
import path from "node:path"
|
|
6
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
7
|
+
import {normalizeConfig} from "../src/config.js"
|
|
8
|
+
import {composeUnitStatus, readComposeLedger, rollbackComposeUnit, rollComposeUnit} from "../src/compose-unit.js"
|
|
9
|
+
|
|
10
|
+
describe("compose-unit", () => {
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @typedef {{code: number, signal: string | null, stderr: string, stdout: string}} CommandResult
|
|
14
|
+
* @typedef {{args?: string[], command?: string}} RecordedCall
|
|
15
|
+
* @typedef {{containerId: string, health: string, image: string | null, ports: {HostPort: string, containerPort: string}[], recreateImages: string[], state: string}} DockerState
|
|
16
|
+
* @typedef {{healthFor?: (image: string) => string, image: string, ports?: {HostPort: string, containerPort: string}[], recreateError?: string, shell?: (command: string) => CommandResult}} DockerWorldOptions
|
|
17
|
+
* @typedef {{calls: RecordedCall[], composeFile: string, projectDirectory: string, runCommand: import("../src/compose-unit.js").CommandRunner, state: DockerState}} DockerWorld
|
|
18
|
+
* @typedef {import("../src/config.js").RollbridgeConfig & {compose: import("../src/config.js").ComposeUnitConfig[]}} ComposeTestConfig
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
const CURRENT_IMAGE = "registry.local/gateway:1.4.0"
|
|
22
|
+
const TARGET_IMAGE = "registry.local/gateway:2.0.0"
|
|
23
|
+
const OLD_IMAGE = "registry.local/gateway:1.0.0"
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Stand-in for the docker/docker-compose CLI: it records every invocation and mutates a small
|
|
27
|
+
* container model, so a roll can be proven stage by stage without a Docker daemon.
|
|
28
|
+
* @param {DockerWorldOptions} options - Initial container state and per-command behaviour.
|
|
29
|
+
* @returns {Promise<DockerWorld>} World with its recorded calls and bound command runner.
|
|
30
|
+
*/
|
|
31
|
+
async function createDockerWorld(options) {
|
|
32
|
+
const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-unit-"))
|
|
33
|
+
const composeFile = path.join(projectDirectory, "gateway.compose.yml")
|
|
34
|
+
|
|
35
|
+
await fs.writeFile(composeFile, `services:\n gateway:\n image: ${CURRENT_IMAGE}\n`)
|
|
36
|
+
|
|
37
|
+
/** @type {DockerState} */
|
|
38
|
+
const state = {
|
|
39
|
+
containerId: "container-1",
|
|
40
|
+
health: "healthy",
|
|
41
|
+
image: options.image,
|
|
42
|
+
ports: options.ports ?? [],
|
|
43
|
+
recreateImages: [],
|
|
44
|
+
state: "running"
|
|
45
|
+
}
|
|
46
|
+
/** @type {RecordedCall[]} */
|
|
47
|
+
const calls = []
|
|
48
|
+
|
|
49
|
+
/** @type {import("../src/compose-unit.js").CommandRunner} */
|
|
50
|
+
const runCommand = async ({args, command}) => {
|
|
51
|
+
if (command !== undefined) {
|
|
52
|
+
calls.push({command})
|
|
53
|
+
|
|
54
|
+
return options.shell?.(command) ?? {code: 0, signal: null, stderr: "", stdout: ""}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
calls.push({args})
|
|
58
|
+
|
|
59
|
+
const [executable, ...rest] = args ?? []
|
|
60
|
+
|
|
61
|
+
if (executable === "docker" && rest[0] === "compose") {
|
|
62
|
+
if (rest.includes("ps")) return {code: 0, signal: null, stderr: "", stdout: `${state.containerId}\n`}
|
|
63
|
+
|
|
64
|
+
if (rest.includes("up")) {
|
|
65
|
+
const overridePath = rest[rest.findIndex((token) => token.endsWith(".override.yml"))]
|
|
66
|
+
const override = await fs.readFile(overridePath, "utf8")
|
|
67
|
+
const image = /** @type {string[]} */ (override.match(/image: "([^"]+)"/))[1]
|
|
68
|
+
|
|
69
|
+
// A compose recreate that the daemon refuses does not touch the running container, which is
|
|
70
|
+
// what the real-daemon proof measures; the stand-in keeps the same observable state.
|
|
71
|
+
if (options.recreateError !== undefined) {
|
|
72
|
+
return {code: 1, signal: null, stderr: `${options.recreateError} (image ${image})`, stdout: ""}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
state.recreateImages.push(image)
|
|
76
|
+
state.image = image
|
|
77
|
+
state.containerId = `container-${state.recreateImages.length + 1}`
|
|
78
|
+
state.health = options.healthFor?.(image) ?? "healthy"
|
|
79
|
+
|
|
80
|
+
return {code: 0, signal: null, stderr: "", stdout: ""}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (executable === "docker" && rest[0] === "inspect") {
|
|
85
|
+
const portJson = state.ports.length === 0
|
|
86
|
+
? "null"
|
|
87
|
+
: JSON.stringify(Object.fromEntries(state.ports.map((port) => [`${port.containerPort}/tcp`, [{HostIp: "0.0.0.0", HostPort: port.HostPort}]])))
|
|
88
|
+
|
|
89
|
+
return {code: 0, signal: null, stderr: "", stdout: `${state.image}|${state.state}|${state.health}|${portJson}|null\n`}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return {code: 127, signal: null, stderr: `unexpected command: ${(args ?? []).join(" ")}`, stdout: ""}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return {calls, composeFile, projectDirectory, runCommand, state}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* @param {string} projectDirectory - Directory holding the Compose file.
|
|
100
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} [composeOverrides] - Unit field overrides.
|
|
101
|
+
* @returns {ComposeTestConfig} Config with one Compose unit.
|
|
102
|
+
*/
|
|
103
|
+
function buildConfig(projectDirectory, composeOverrides = {}) {
|
|
104
|
+
return /** @type {ComposeTestConfig} */ (normalizeConfig({
|
|
105
|
+
application: "compose-test",
|
|
106
|
+
compose: [{
|
|
107
|
+
activation: {intervalMs: 20, timeoutMs: 200},
|
|
108
|
+
drain: {command: "quiesce {{service}}", timeoutMs: 500},
|
|
109
|
+
id: "gateway",
|
|
110
|
+
image: TARGET_IMAGE,
|
|
111
|
+
inFlight: {command: "await in-flight {{service}}", intervalMs: 20, timeoutMs: 200},
|
|
112
|
+
projectDirectory,
|
|
113
|
+
service: "gateway",
|
|
114
|
+
stopGracePeriodMs: 30000,
|
|
115
|
+
...composeOverrides
|
|
116
|
+
}],
|
|
117
|
+
composeStatePath: path.join(projectDirectory, "compose.state.json"),
|
|
118
|
+
control: {path: path.join(projectDirectory, "rollbridge.sock")},
|
|
119
|
+
processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
120
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
121
|
+
}))
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* @param {string} prefix - Command prefix that should fail.
|
|
126
|
+
* @param {string} stderr - Failure output.
|
|
127
|
+
* @returns {(command: string) => CommandResult} Shell handler failing every command with this prefix.
|
|
128
|
+
*/
|
|
129
|
+
function failingShell(prefix, stderr) {
|
|
130
|
+
return (command) => command.startsWith(prefix)
|
|
131
|
+
? {code: 1, signal: null, stderr, stdout: ""}
|
|
132
|
+
: {code: 0, signal: null, stderr: "", stdout: ""}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
test("rolls one Compose unit drain-first and retains the previous image", async () => {
|
|
136
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE})
|
|
137
|
+
const config = buildConfig(world.projectDirectory)
|
|
138
|
+
const unit = config.compose[0]
|
|
139
|
+
|
|
140
|
+
try {
|
|
141
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
142
|
+
|
|
143
|
+
expect(result.status).toBe("rolled")
|
|
144
|
+
expect(result.stage).toBe("activation")
|
|
145
|
+
expect(result.image).toBe(TARGET_IMAGE)
|
|
146
|
+
expect(result.previousImage).toBe(CURRENT_IMAGE)
|
|
147
|
+
expect(result.generations.map((generation) => [generation.image, generation.status])).toEqual([[TARGET_IMAGE, "activated"]])
|
|
148
|
+
expect(world.state.recreateImages).toEqual([TARGET_IMAGE])
|
|
149
|
+
|
|
150
|
+
const commands = world.calls.map((call) => call.command).filter(Boolean)
|
|
151
|
+
|
|
152
|
+
expect(commands).toEqual(["quiesce gateway", "await in-flight gateway"])
|
|
153
|
+
|
|
154
|
+
const recreateArgs = /** @type {string[]} */ (world.calls.find((call) => call.args?.includes("up"))?.args)
|
|
155
|
+
|
|
156
|
+
expect(recreateArgs.slice(-5)).toEqual(["up", "-d", "--no-deps", "--force-recreate", "gateway"])
|
|
157
|
+
expect(recreateArgs).toContain("--no-deps")
|
|
158
|
+
|
|
159
|
+
const overridePath = recreateArgs[recreateArgs.findIndex((token) => token.endsWith(".override.yml"))]
|
|
160
|
+
|
|
161
|
+
expect(overridePath.startsWith(world.projectDirectory)).toBeTruthy()
|
|
162
|
+
|
|
163
|
+
const ledger = await readComposeLedger({config, unit})
|
|
164
|
+
|
|
165
|
+
expect(ledger.generations[0].previousImage).toBe(CURRENT_IMAGE)
|
|
166
|
+
expect(ledger.generations[0].releaseId).toBe(result.releaseId)
|
|
167
|
+
await expect(fs.stat(overridePath)).rejects.toThrow()
|
|
168
|
+
} finally {
|
|
169
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
170
|
+
}
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
test("the generated override pins the target image and the stop grace period", async () => {
|
|
174
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE})
|
|
175
|
+
const config = buildConfig(world.projectDirectory)
|
|
176
|
+
/** @type {string | undefined} */
|
|
177
|
+
let overrideContents
|
|
178
|
+
|
|
179
|
+
try {
|
|
180
|
+
/** @type {import("../src/compose-unit.js").CommandRunner} */
|
|
181
|
+
const recordingRunner = async (request) => {
|
|
182
|
+
if (request.args?.includes("up")) {
|
|
183
|
+
overrideContents = await fs.readFile(request.args[request.args.findIndex((token) => token.endsWith(".override.yml"))], "utf8")
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
return await world.runCommand(request)
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const result = await rollComposeUnit({config, runCommand: recordingRunner, unit: config.compose[0]})
|
|
190
|
+
|
|
191
|
+
expect(result.status).toBe("rolled")
|
|
192
|
+
expect(overrideContents).toMatch(/image: "registry\.local\/gateway:2\.0\.0"/)
|
|
193
|
+
expect(overrideContents).toMatch(/stop_grace_period: 30s/)
|
|
194
|
+
} finally {
|
|
195
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
196
|
+
}
|
|
197
|
+
})
|
|
198
|
+
|
|
199
|
+
test("a unit that cannot be quiesced fails the roll and is never recreated", async () => {
|
|
200
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE, shell: failingShell("quiesce", "gateway refused to drain: drain socket timeout")})
|
|
201
|
+
const config = buildConfig(world.projectDirectory)
|
|
202
|
+
const unit = config.compose[0]
|
|
203
|
+
|
|
204
|
+
try {
|
|
205
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
206
|
+
|
|
207
|
+
expect(result.status).toBe("failed")
|
|
208
|
+
expect(result.stage).toBe("drain")
|
|
209
|
+
expect(result.error).toMatch(/could not be quiesced/)
|
|
210
|
+
expect(result.error).toMatch(/gateway refused to drain/)
|
|
211
|
+
expect(world.state.recreateImages).toEqual([])
|
|
212
|
+
expect(world.state.image).toBe(CURRENT_IMAGE)
|
|
213
|
+
expect(world.state.containerId).toBe("container-1")
|
|
214
|
+
expect(world.calls.some((call) => call.args?.includes("up"))).toBe(false)
|
|
215
|
+
expect((await readComposeLedger({config, unit})).generations).toEqual([])
|
|
216
|
+
} finally {
|
|
217
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
218
|
+
}
|
|
219
|
+
})
|
|
220
|
+
|
|
221
|
+
test("in-flight work that never settles fails the roll and keeps the current container", async () => {
|
|
222
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE, shell: failingShell("await", "3 handoffs still running")})
|
|
223
|
+
const config = buildConfig(world.projectDirectory, {inFlight: {command: "await in-flight {{service}}", intervalMs: 20, timeoutMs: 60}})
|
|
224
|
+
|
|
225
|
+
try {
|
|
226
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})
|
|
227
|
+
|
|
228
|
+
expect(result.status).toBe("failed")
|
|
229
|
+
expect(result.stage).toBe("in_flight")
|
|
230
|
+
expect(result.error).toMatch(/still had in-flight work after 60ms/)
|
|
231
|
+
expect(result.error).toMatch(/3 handoffs still running/)
|
|
232
|
+
expect(world.state.recreateImages).toEqual([])
|
|
233
|
+
expect(world.state.image).toBe(CURRENT_IMAGE)
|
|
234
|
+
} finally {
|
|
235
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
236
|
+
}
|
|
237
|
+
})
|
|
238
|
+
|
|
239
|
+
test("in-flight work that settles after a retry lets the roll continue", async () => {
|
|
240
|
+
let attempts = 0
|
|
241
|
+
const world = await createDockerWorld({
|
|
242
|
+
image: CURRENT_IMAGE,
|
|
243
|
+
shell: (command) => {
|
|
244
|
+
if (!command.startsWith("await")) return {code: 0, signal: null, stderr: "", stdout: ""}
|
|
245
|
+
|
|
246
|
+
attempts += 1
|
|
247
|
+
|
|
248
|
+
return attempts < 3
|
|
249
|
+
? {code: 1, signal: null, stderr: "1 handoff still running", stdout: ""}
|
|
250
|
+
: {code: 0, signal: null, stderr: "", stdout: ""}
|
|
251
|
+
}
|
|
252
|
+
})
|
|
253
|
+
const config = buildConfig(world.projectDirectory, {inFlight: {command: "await in-flight {{service}}", intervalMs: 20, timeoutMs: 500}})
|
|
254
|
+
|
|
255
|
+
try {
|
|
256
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})
|
|
257
|
+
|
|
258
|
+
expect(result.status).toBe("rolled")
|
|
259
|
+
expect(attempts).toBe(3)
|
|
260
|
+
expect(world.state.recreateImages).toEqual([TARGET_IMAGE])
|
|
261
|
+
} finally {
|
|
262
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
263
|
+
}
|
|
264
|
+
})
|
|
265
|
+
|
|
266
|
+
test("a refused recreate fails the roll and leaves the previous container serving", async () => {
|
|
267
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE, recreateError: "Error response from daemon: No such image: registry.local/gateway:2.0.0"})
|
|
268
|
+
const config = buildConfig(world.projectDirectory)
|
|
269
|
+
const unit = config.compose[0]
|
|
270
|
+
|
|
271
|
+
try {
|
|
272
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
273
|
+
|
|
274
|
+
expect(result.status).toBe("failed")
|
|
275
|
+
expect(result.stage).toBe("recreate")
|
|
276
|
+
expect(result.image).toBe(CURRENT_IMAGE)
|
|
277
|
+
expect(result.previousImage).toBe(CURRENT_IMAGE)
|
|
278
|
+
expect(result.rollback).toBeUndefined()
|
|
279
|
+
expect(result.error).toMatch(/Compose recreate failed for unit "gateway" \(image registry\.local\/gateway:2\.0\.0, previous registry\.local\/gateway:1\.4\.0\)/)
|
|
280
|
+
expect(result.error).toMatch(/No such image/)
|
|
281
|
+
expect(world.state.recreateImages).toEqual([])
|
|
282
|
+
expect(world.state.image).toBe(CURRENT_IMAGE)
|
|
283
|
+
expect(world.state.containerId).toBe("container-1")
|
|
284
|
+
expect(world.state.state).toBe("running")
|
|
285
|
+
expect(world.calls.filter((call) => call.args?.includes("up"))).toHaveLength(1)
|
|
286
|
+
expect((await readComposeLedger({config, unit})).generations).toEqual([])
|
|
287
|
+
expect(await fs.stat(path.join(world.projectDirectory, ".rollbridge-gateway.override.yml")).then(() => true, () => false)).toBe(false)
|
|
288
|
+
} finally {
|
|
289
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
290
|
+
}
|
|
291
|
+
})
|
|
292
|
+
|
|
293
|
+
test("a candidate image that never activates is rolled back to the previous image", async () => {
|
|
294
|
+
const world = await createDockerWorld({healthFor: (image) => image === TARGET_IMAGE ? "unhealthy" : "healthy", image: CURRENT_IMAGE})
|
|
295
|
+
const config = buildConfig(world.projectDirectory, {activation: {intervalMs: 20, timeoutMs: 120}})
|
|
296
|
+
|
|
297
|
+
try {
|
|
298
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})
|
|
299
|
+
|
|
300
|
+
expect(result.status).toBe("rolled_back")
|
|
301
|
+
expect(result.stage).toBe("rollback")
|
|
302
|
+
expect(result.error).toMatch(/did not activate within 120ms/)
|
|
303
|
+
expect(result.rollback).toMatchObject({image: CURRENT_IMAGE, status: "rolled_back"})
|
|
304
|
+
expect(world.state.recreateImages).toEqual([TARGET_IMAGE, CURRENT_IMAGE])
|
|
305
|
+
expect(world.state.image).toBe(CURRENT_IMAGE)
|
|
306
|
+
expect(world.state.health).toBe("healthy")
|
|
307
|
+
expect(result.generations.map((generation) => [generation.image, generation.status])).toEqual([[CURRENT_IMAGE, "rolled_back"]])
|
|
308
|
+
expect(result.generations[0].previousImage).toBe(TARGET_IMAGE)
|
|
309
|
+
} finally {
|
|
310
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
311
|
+
}
|
|
312
|
+
})
|
|
313
|
+
|
|
314
|
+
test("a failed rollback is reported as a failed roll instead of a success", async () => {
|
|
315
|
+
const world = await createDockerWorld({healthFor: () => "unhealthy", image: CURRENT_IMAGE})
|
|
316
|
+
const config = buildConfig(world.projectDirectory, {activation: {intervalMs: 20, timeoutMs: 100}})
|
|
317
|
+
const unit = config.compose[0]
|
|
318
|
+
|
|
319
|
+
try {
|
|
320
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
321
|
+
|
|
322
|
+
expect(result.status).toBe("failed")
|
|
323
|
+
expect(result.stage).toBe("rollback_failed")
|
|
324
|
+
expect(result.error).toMatch(/rollback to registry\.local\/gateway:1\.4\.0 did not activate/)
|
|
325
|
+
expect((await readComposeLedger({config, unit})).generations).toEqual([])
|
|
326
|
+
} finally {
|
|
327
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
328
|
+
}
|
|
329
|
+
})
|
|
330
|
+
|
|
331
|
+
test("compose-rollback selects the retained previous image", async () => {
|
|
332
|
+
const world = await createDockerWorld({image: OLD_IMAGE})
|
|
333
|
+
const config = buildConfig(world.projectDirectory)
|
|
334
|
+
const unit = config.compose[0]
|
|
335
|
+
|
|
336
|
+
try {
|
|
337
|
+
await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
338
|
+
|
|
339
|
+
const result = await rollbackComposeUnit({config, runCommand: world.runCommand, unit})
|
|
340
|
+
|
|
341
|
+
expect(result.status).toBe("rolled")
|
|
342
|
+
expect(result.image).toBe(OLD_IMAGE)
|
|
343
|
+
expect(result.previousImage).toBe(TARGET_IMAGE)
|
|
344
|
+
expect(world.state.recreateImages).toEqual([TARGET_IMAGE, OLD_IMAGE])
|
|
345
|
+
expect(world.state.image).toBe(OLD_IMAGE)
|
|
346
|
+
} finally {
|
|
347
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
348
|
+
}
|
|
349
|
+
})
|
|
350
|
+
|
|
351
|
+
test("compose-rollback refuses when no earlier image is retained", async () => {
|
|
352
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE})
|
|
353
|
+
const config = buildConfig(world.projectDirectory)
|
|
354
|
+
|
|
355
|
+
try {
|
|
356
|
+
await expect(rollbackComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})).rejects.toThrow(/No earlier image is retained for Compose unit "gateway"/)
|
|
357
|
+
} finally {
|
|
358
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
359
|
+
}
|
|
360
|
+
})
|
|
361
|
+
|
|
362
|
+
test("refuses to roll a unit whose container publishes the Rollbridge proxy port", async () => {
|
|
363
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE, ports: [{HostPort: "8182", containerPort: "8182"}]})
|
|
364
|
+
const config = buildConfig(world.projectDirectory)
|
|
365
|
+
|
|
366
|
+
try {
|
|
367
|
+
await expect(rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})).rejects.toThrow(/publishes port 8182, the Rollbridge proxy port/)
|
|
368
|
+
expect(world.state.recreateImages).toEqual([])
|
|
369
|
+
} finally {
|
|
370
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
371
|
+
}
|
|
372
|
+
})
|
|
373
|
+
|
|
374
|
+
test("refuses to roll while a release generation transition is unresolved", async () => {
|
|
375
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE})
|
|
376
|
+
const config = buildConfig(world.projectDirectory)
|
|
377
|
+
|
|
378
|
+
try {
|
|
379
|
+
await expect(rollComposeUnit({
|
|
380
|
+
config,
|
|
381
|
+
inspectSocket: async () => ({activeReleaseId: "v1", alive: true, application: "compose-test"}),
|
|
382
|
+
runCommand: world.runCommand,
|
|
383
|
+
sendCommand: async () => ({
|
|
384
|
+
application: "compose-test",
|
|
385
|
+
generationTransition: {candidateReleaseId: "v2", phase: "activating_candidate", previousReleaseId: "v1"},
|
|
386
|
+
status: "success"
|
|
387
|
+
}),
|
|
388
|
+
unit: config.compose[0]
|
|
389
|
+
})).rejects.toThrow(/while release generation transition v2 is unresolved at activating_candidate/)
|
|
390
|
+
|
|
391
|
+
expect(world.state.recreateImages).toEqual([])
|
|
392
|
+
} finally {
|
|
393
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
394
|
+
}
|
|
395
|
+
})
|
|
396
|
+
|
|
397
|
+
test("compose-status reports the live container and the retained generations", async () => {
|
|
398
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE})
|
|
399
|
+
const config = buildConfig(world.projectDirectory)
|
|
400
|
+
const unit = config.compose[0]
|
|
401
|
+
|
|
402
|
+
try {
|
|
403
|
+
await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
404
|
+
|
|
405
|
+
const status = await composeUnitStatus({config, runCommand: world.runCommand, unit})
|
|
406
|
+
|
|
407
|
+
expect(status.unitId).toBe("gateway")
|
|
408
|
+
expect(status.container).toMatchObject({containerId: "container-2", healthStatus: "healthy", image: TARGET_IMAGE, state: "running"})
|
|
409
|
+
expect(status.generations.map((generation) => generation.image)).toEqual([TARGET_IMAGE])
|
|
410
|
+
expect(status.ledgerPath).toBe(path.join(world.projectDirectory, "compose.state.json"))
|
|
411
|
+
} finally {
|
|
412
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
413
|
+
}
|
|
414
|
+
})
|
|
415
|
+
})
|
|
@@ -143,7 +143,9 @@ test("validateConfig defaults the restart policy, accepts overrides, and rejects
|
|
|
143
143
|
const defaulted = validateRestart(undefined)
|
|
144
144
|
|
|
145
145
|
expect(defaulted.issues).toEqual([])
|
|
146
|
-
|
|
146
|
+
// Unset values are absent from the normalized shape rather than present as undefined, so a config
|
|
147
|
+
// that crossed the guardian's JSON transfer is still equal to a freshly loaded one.
|
|
148
|
+
expect(defaulted.config.processes[0].restart).toEqual({backoffFactor: 1, maxDelayMs: 0, windowMs: 0})
|
|
147
149
|
|
|
148
150
|
const custom = validateRestart({backoffFactor: 2, maxDelayMs: 30000, maxRestarts: 5, windowMs: 60000})
|
|
149
151
|
|
|
@@ -587,6 +589,108 @@ test("validateConfig defaults health.startDelayMs to 0, accepts an override, and
|
|
|
587
589
|
expect(negative.issues.some((issue) => issue.message === "processes[0].health.startDelayMs must be a non-negative number")).toBeTruthy()
|
|
588
590
|
})
|
|
589
591
|
|
|
592
|
+
test("validateConfig gives an unconfigured proxied process the measured production boot budget", () => {
|
|
593
|
+
/**
|
|
594
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} [processConfig] - Extra keys for the proxied process.
|
|
595
|
+
* @returns {import("../src/config.js").RollbridgeConfig} Normalized config.
|
|
596
|
+
*/
|
|
597
|
+
const normalized = (processConfig = {}) => validateConfig({
|
|
598
|
+
application: "demo",
|
|
599
|
+
control: {path: "/tmp/demo.sock"},
|
|
600
|
+
processes: [
|
|
601
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}, ...processConfig}
|
|
602
|
+
],
|
|
603
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
604
|
+
}).config
|
|
605
|
+
|
|
606
|
+
const defaults = normalized()
|
|
607
|
+
|
|
608
|
+
expect(defaults.proxy.healthTimeoutMs).toBe(120000)
|
|
609
|
+
// The per-process gate inherits the release budget, so a project that configures no health
|
|
610
|
+
// timeout at all gets a deadline above the TensorBuzz boot measured on 2026-11-06 instead of
|
|
611
|
+
// the 30000ms that killed two production deploys.
|
|
612
|
+
expect(defaults.processes[0].health?.timeoutMs).toBe(120000)
|
|
613
|
+
expect(defaults.processes[0].health?.intervalMs).toBe(250)
|
|
614
|
+
|
|
615
|
+
const releaseBudget = normalized({health: {path: "/healthz"}})
|
|
616
|
+
|
|
617
|
+
expect(releaseBudget.proxy.healthTimeoutMs).toBe(120000)
|
|
618
|
+
expect(releaseBudget.processes[0].health?.path).toBe("/healthz")
|
|
619
|
+
expect(releaseBudget.processes[0].health?.timeoutMs).toBe(120000)
|
|
620
|
+
|
|
621
|
+
// Both configuration levels still win over the default: the release-level budget and the
|
|
622
|
+
// per-process budget, which is what a project with a known boot time sets.
|
|
623
|
+
const configuredProxy = validateConfig({
|
|
624
|
+
application: "demo",
|
|
625
|
+
control: {path: "/tmp/demo.sock"},
|
|
626
|
+
processes: [
|
|
627
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
|
|
628
|
+
],
|
|
629
|
+
proxy: {healthTimeoutMs: 45000, host: "127.0.0.1", port: 8182}
|
|
630
|
+
}).config
|
|
631
|
+
|
|
632
|
+
expect(configuredProxy.proxy.healthTimeoutMs).toBe(45000)
|
|
633
|
+
expect(configuredProxy.processes[0].health?.timeoutMs).toBe(45000)
|
|
634
|
+
|
|
635
|
+
const perProcess = normalized({health: {path: "/ping", timeoutMs: 900}})
|
|
636
|
+
|
|
637
|
+
expect(perProcess.proxy.healthTimeoutMs).toBe(120000)
|
|
638
|
+
expect(perProcess.processes[0].health?.timeoutMs).toBe(900)
|
|
639
|
+
})
|
|
640
|
+
|
|
641
|
+
test("validateConfig defaults dependsOn and its own wait budget, and rejects dependencies it cannot wait on", () => {
|
|
642
|
+
/**
|
|
643
|
+
* @param {import("../src/json.js").JsonValue} processes - Processes under test.
|
|
644
|
+
* @returns {{config: import("../src/config.js").RollbridgeConfig, issues: import("../src/config.js").ConfigIssue[]}} Validation result.
|
|
645
|
+
*/
|
|
646
|
+
const validateProcesses = (processes) => validateConfig({
|
|
647
|
+
application: "demo",
|
|
648
|
+
control: {path: "/tmp/demo.sock"},
|
|
649
|
+
processes,
|
|
650
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
651
|
+
})
|
|
652
|
+
|
|
653
|
+
const base = [
|
|
654
|
+
{command: "run beacon", id: "beacon", policy: "service", port: 7330},
|
|
655
|
+
{command: "run jobs", dependsOn: ["beacon"], id: "jobs", policy: "service", port: {from: 7331, to: 7399}},
|
|
656
|
+
{command: "run web", dependsOn: ["jobs"], id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
|
|
657
|
+
]
|
|
658
|
+
const valid = validateProcesses(base)
|
|
659
|
+
|
|
660
|
+
expect(valid.issues).toEqual([])
|
|
661
|
+
expect(valid.config.processes[0].dependsOn).toEqual([])
|
|
662
|
+
expect(valid.config.processes[1].dependsOn).toEqual(["beacon"])
|
|
663
|
+
expect(valid.config.processes[1].dependencyTimeoutMs).toBe(30000)
|
|
664
|
+
|
|
665
|
+
const custom = validateProcesses([...base.slice(0, 2), {...base[2], dependencyTimeoutMs: 60000}])
|
|
666
|
+
|
|
667
|
+
expect(custom.issues).toEqual([])
|
|
668
|
+
expect(custom.config.processes[2].dependencyTimeoutMs).toBe(60000)
|
|
669
|
+
|
|
670
|
+
const invalid = validateProcesses([
|
|
671
|
+
{command: "run web", dependsOn: ["missing", "web", "jobs", "worker"], dependencyTimeoutMs: 0, id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
672
|
+
{command: "run jobs", id: "jobs", policy: "singleton"},
|
|
673
|
+
{command: "run worker", id: "worker", policy: "companion"}
|
|
674
|
+
])
|
|
675
|
+
const messages = invalid.issues.map((issue) => issue.message)
|
|
676
|
+
|
|
677
|
+
expect(messages.some((message) => message === 'Process "web" depends on unknown process "missing"')).toBeTruthy()
|
|
678
|
+
expect(messages.some((message) => message === 'Process "web" cannot depend on itself')).toBeTruthy()
|
|
679
|
+
expect(messages.some((message) => message === 'Process "web" cannot depend on singleton "jobs"')).toBeTruthy()
|
|
680
|
+
expect(messages.some((message) => message === 'Process "worker" has no port, so "web" cannot wait for it')).toBeTruthy()
|
|
681
|
+
expect(messages.some((message) => message === "processes[0].dependencyTimeoutMs must be a positive number")).toBeTruthy()
|
|
682
|
+
|
|
683
|
+
const backwards = validateProcesses([
|
|
684
|
+
{command: "run web", dependsOn: "web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
685
|
+
{command: "run worker", dependsOn: ["web"], id: "worker", policy: "companion", port: 7001}
|
|
686
|
+
])
|
|
687
|
+
const backwardsMessages = backwards.issues.map((issue) => issue.message)
|
|
688
|
+
|
|
689
|
+
expect(backwardsMessages.some((message) => message === "processes[0].dependsOn must be an array")).toBeTruthy()
|
|
690
|
+
expect(backwardsMessages.some((message) => message === 'Process "worker" depends on "web", which Rollbridge starts after it')).toBeTruthy()
|
|
691
|
+
expect(backwards.issues.every((issue) => typeof issue.fix === "string" && issue.fix.length > 0)).toBe(true)
|
|
692
|
+
})
|
|
693
|
+
|
|
590
694
|
test("validateConfig defaults releaseRetention, accepts overrides, and rejects bad values", () => {
|
|
591
695
|
/**
|
|
592
696
|
* @param {import("../src/json.js").JsonValue} releaseRetention - Retention config under test, or undefined.
|
|
@@ -603,17 +707,25 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
|
|
|
603
707
|
const defaulted = validateRetention(undefined)
|
|
604
708
|
|
|
605
709
|
expect(defaulted.issues).toEqual([])
|
|
606
|
-
expect(defaulted.config.releaseRetention).toEqual({escalateWedgedGenerations: false, keep: 10, maxAgeMs: 0})
|
|
710
|
+
expect(defaulted.config.releaseRetention).toEqual({escalateWedgedGenerations: false, keep: 10, maxAgeMs: 0, maxConcurrentDrainingReleaseGroups: 12})
|
|
607
711
|
|
|
608
|
-
const custom = validateRetention({keep: 3, maxAgeMs: 60000})
|
|
712
|
+
const custom = validateRetention({escalateWedgedGenerations: true, keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
609
713
|
|
|
610
714
|
expect(custom.issues).toEqual([])
|
|
611
|
-
expect(custom.config.releaseRetention).toEqual({escalateWedgedGenerations:
|
|
715
|
+
expect(custom.config.releaseRetention).toEqual({escalateWedgedGenerations: true, keep: 3, maxAgeMs: 60000, maxConcurrentDrainingReleaseGroups: 4})
|
|
716
|
+
|
|
717
|
+
// 0 is a valid bound: it disables the pre-activation draining-generation check.
|
|
718
|
+
const unbounded = validateRetention({maxConcurrentDrainingReleaseGroups: 0})
|
|
612
719
|
|
|
613
|
-
|
|
720
|
+
expect(unbounded.issues).toEqual([])
|
|
721
|
+
expect(unbounded.config.releaseRetention.maxConcurrentDrainingReleaseGroups).toBe(0)
|
|
614
722
|
|
|
723
|
+
const invalid = validateRetention({escalateWedgedGenerations: "yes", keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
|
|
724
|
+
|
|
725
|
+
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.escalateWedgedGenerations must be a boolean")).toBeTruthy()
|
|
615
726
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.keep must be a non-negative integer")).toBeTruthy()
|
|
616
727
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxAgeMs must be a non-negative number")).toBeTruthy()
|
|
728
|
+
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxConcurrentDrainingReleaseGroups must be a non-negative integer")).toBeTruthy()
|
|
617
729
|
})
|
|
618
730
|
|
|
619
731
|
test("validateConfig leaves statePath unset by default, accepts a string, and rejects non-strings", () => {
|
|
@@ -661,6 +773,24 @@ test("ownerRecovery requires durable state and a non-negative integer reconnecti
|
|
|
661
773
|
expect(invalidGrace.issues.some((issue) => issue.message === "ownerRecovery.reconnectGraceMs must be a non-negative integer")).toBeTruthy()
|
|
662
774
|
})
|
|
663
775
|
|
|
776
|
+
test("a normalized config is identical to its own JSON form, so the guardian transfer cannot change it", () => {
|
|
777
|
+
const {config, issues} = validateConfig({
|
|
778
|
+
application: "demo",
|
|
779
|
+
control: {path: "/tmp/demo.sock"},
|
|
780
|
+
processes: [{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}],
|
|
781
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
782
|
+
}, "/srv/app/rollbridge.js")
|
|
783
|
+
|
|
784
|
+
expect(issues).toEqual([])
|
|
785
|
+
|
|
786
|
+
// Owner replacement publishes release configs to the candidate daemon over the guardian's JSON
|
|
787
|
+
// socket, which drops undefined-valued properties. A config that is not its own JSON form therefore
|
|
788
|
+
// arrives at the candidate unequal to the config file it was started from, and the live-reload fence
|
|
789
|
+
// aborts the next deploy with "Config changes to control cannot be applied live" even though the
|
|
790
|
+
// file never changed — measured in production on 2026-11-16 (tensorbuzz/production).
|
|
791
|
+
expect(config).toEqual(JSON.parse(JSON.stringify(config)))
|
|
792
|
+
})
|
|
793
|
+
|
|
664
794
|
test("normalizeConfig throws an aggregated error listing every issue", async () => {
|
|
665
795
|
const normalization = Promise.resolve().then(() => normalizeConfig({
|
|
666
796
|
application: "demo",
|
|
@@ -101,6 +101,28 @@ test("a known command missing a required field returns a field error", async ()
|
|
|
101
101
|
expect(response.status).toBe("error")
|
|
102
102
|
expect(response.error).toBe("releasePath is required")
|
|
103
103
|
})
|
|
104
|
+
|
|
105
|
+
test("a control connection closed during owner handoff rejects the command", async () => {
|
|
106
|
+
const handoffSocketPath = path.join(root, "handoff.sock")
|
|
107
|
+
const handoffServer = net.createServer((socket) => {
|
|
108
|
+
socket.on("data", () => socket.destroy())
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
await new Promise((resolve, reject) => {
|
|
112
|
+
handoffServer.once("error", reject)
|
|
113
|
+
handoffServer.listen(handoffSocketPath, () => resolve(undefined))
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
try {
|
|
117
|
+
await expect(sendControlCommand({command: {command: "deploy"}, path: handoffSocketPath, timeoutMs: 100})).rejects.toThrow(
|
|
118
|
+
`Rollbridge control connection ${handoffSocketPath} closed before its response`
|
|
119
|
+
)
|
|
120
|
+
} finally {
|
|
121
|
+
await new Promise((resolve) => handoffServer.close(resolve))
|
|
122
|
+
await fs.rm(handoffSocketPath, {force: true})
|
|
123
|
+
}
|
|
124
|
+
})
|
|
125
|
+
|
|
104
126
|
test("status can omit process logs without changing the default status payload", async () => {
|
|
105
127
|
if (!daemon) throw new Error("Missing required fixture: daemon")
|
|
106
128
|
await daemon.deploy({releaseId: "v1", releasePath: root, revision: "v1"})
|