rollbridge 0.1.56 → 0.1.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -0
- package/TODO.md +12 -0
- package/changelog.d/20261013-recover-dead-guardian-state.md +1 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/docs/cli.md +70 -1
- package/docs/compose-services.md +220 -0
- package/docs/config.md +55 -0
- package/docs/troubleshooting.md +24 -0
- package/package.json +1 -1
- package/src/cli.js +82 -4
- package/src/compose-unit.js +725 -0
- package/src/config.js +201 -2
- package/src/daemon.js +23 -8
- package/src/doctor.js +67 -0
- package/src/guardian-client.js +13 -2
- package/src/process-guardian.js +137 -17
- package/src/recover.js +128 -8
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +158 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +70 -0
- package/test/recover.test.js +119 -0
- package/test/rollbridge.test.js +106 -6
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises"
|
|
4
|
+
import os from "node:os"
|
|
5
|
+
import path from "node:path"
|
|
6
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
7
|
+
import {normalizeConfig} from "../src/config.js"
|
|
8
|
+
import {composeUnitStatus, readComposeLedger, rollbackComposeUnit, rollComposeUnit} from "../src/compose-unit.js"
|
|
9
|
+
|
|
10
|
+
describe("compose-unit", () => {
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @typedef {{code: number, signal: string | null, stderr: string, stdout: string}} CommandResult
|
|
14
|
+
* @typedef {{args?: string[], command?: string}} RecordedCall
|
|
15
|
+
* @typedef {{containerId: string, health: string, image: string | null, ports: {HostPort: string, containerPort: string}[], recreateImages: string[], state: string}} DockerState
|
|
16
|
+
* @typedef {{healthFor?: (image: string) => string, image: string, ports?: {HostPort: string, containerPort: string}[], recreateError?: string, shell?: (command: string) => CommandResult}} DockerWorldOptions
|
|
17
|
+
* @typedef {{calls: RecordedCall[], composeFile: string, projectDirectory: string, runCommand: import("../src/compose-unit.js").CommandRunner, state: DockerState}} DockerWorld
|
|
18
|
+
* @typedef {import("../src/config.js").RollbridgeConfig & {compose: import("../src/config.js").ComposeUnitConfig[]}} ComposeTestConfig
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
const CURRENT_IMAGE = "registry.local/gateway:1.4.0"
|
|
22
|
+
const TARGET_IMAGE = "registry.local/gateway:2.0.0"
|
|
23
|
+
const OLD_IMAGE = "registry.local/gateway:1.0.0"
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Stand-in for the docker/docker-compose CLI: it records every invocation and mutates a small
|
|
27
|
+
* container model, so a roll can be proven stage by stage without a Docker daemon.
|
|
28
|
+
* @param {DockerWorldOptions} options - Initial container state and per-command behaviour.
|
|
29
|
+
* @returns {Promise<DockerWorld>} World with its recorded calls and bound command runner.
|
|
30
|
+
*/
|
|
31
|
+
async function createDockerWorld(options) {
|
|
32
|
+
const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-unit-"))
|
|
33
|
+
const composeFile = path.join(projectDirectory, "gateway.compose.yml")
|
|
34
|
+
|
|
35
|
+
await fs.writeFile(composeFile, `services:\n gateway:\n image: ${CURRENT_IMAGE}\n`)
|
|
36
|
+
|
|
37
|
+
/** @type {DockerState} */
|
|
38
|
+
const state = {
|
|
39
|
+
containerId: "container-1",
|
|
40
|
+
health: "healthy",
|
|
41
|
+
image: options.image,
|
|
42
|
+
ports: options.ports ?? [],
|
|
43
|
+
recreateImages: [],
|
|
44
|
+
state: "running"
|
|
45
|
+
}
|
|
46
|
+
/** @type {RecordedCall[]} */
|
|
47
|
+
const calls = []
|
|
48
|
+
|
|
49
|
+
/** @type {import("../src/compose-unit.js").CommandRunner} */
|
|
50
|
+
const runCommand = async ({args, command}) => {
|
|
51
|
+
if (command !== undefined) {
|
|
52
|
+
calls.push({command})
|
|
53
|
+
|
|
54
|
+
return options.shell?.(command) ?? {code: 0, signal: null, stderr: "", stdout: ""}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
calls.push({args})
|
|
58
|
+
|
|
59
|
+
const [executable, ...rest] = args ?? []
|
|
60
|
+
|
|
61
|
+
if (executable === "docker" && rest[0] === "compose") {
|
|
62
|
+
if (rest.includes("ps")) return {code: 0, signal: null, stderr: "", stdout: `${state.containerId}\n`}
|
|
63
|
+
|
|
64
|
+
if (rest.includes("up")) {
|
|
65
|
+
const overridePath = rest[rest.findIndex((token) => token.endsWith(".override.yml"))]
|
|
66
|
+
const override = await fs.readFile(overridePath, "utf8")
|
|
67
|
+
const image = /** @type {string[]} */ (override.match(/image: "([^"]+)"/))[1]
|
|
68
|
+
|
|
69
|
+
// A compose recreate that the daemon refuses does not touch the running container, which is
|
|
70
|
+
// what the real-daemon proof measures; the stand-in keeps the same observable state.
|
|
71
|
+
if (options.recreateError !== undefined) {
|
|
72
|
+
return {code: 1, signal: null, stderr: `${options.recreateError} (image ${image})`, stdout: ""}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
state.recreateImages.push(image)
|
|
76
|
+
state.image = image
|
|
77
|
+
state.containerId = `container-${state.recreateImages.length + 1}`
|
|
78
|
+
state.health = options.healthFor?.(image) ?? "healthy"
|
|
79
|
+
|
|
80
|
+
return {code: 0, signal: null, stderr: "", stdout: ""}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (executable === "docker" && rest[0] === "inspect") {
|
|
85
|
+
const portJson = state.ports.length === 0
|
|
86
|
+
? "null"
|
|
87
|
+
: JSON.stringify(Object.fromEntries(state.ports.map((port) => [`${port.containerPort}/tcp`, [{HostIp: "0.0.0.0", HostPort: port.HostPort}]])))
|
|
88
|
+
|
|
89
|
+
return {code: 0, signal: null, stderr: "", stdout: `${state.image}|${state.state}|${state.health}|${portJson}|null\n`}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return {code: 127, signal: null, stderr: `unexpected command: ${(args ?? []).join(" ")}`, stdout: ""}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return {calls, composeFile, projectDirectory, runCommand, state}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* @param {string} projectDirectory - Directory holding the Compose file.
|
|
100
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} [composeOverrides] - Unit field overrides.
|
|
101
|
+
* @returns {ComposeTestConfig} Config with one Compose unit.
|
|
102
|
+
*/
|
|
103
|
+
function buildConfig(projectDirectory, composeOverrides = {}) {
|
|
104
|
+
return /** @type {ComposeTestConfig} */ (normalizeConfig({
|
|
105
|
+
application: "compose-test",
|
|
106
|
+
compose: [{
|
|
107
|
+
activation: {intervalMs: 20, timeoutMs: 200},
|
|
108
|
+
drain: {command: "quiesce {{service}}", timeoutMs: 500},
|
|
109
|
+
id: "gateway",
|
|
110
|
+
image: TARGET_IMAGE,
|
|
111
|
+
inFlight: {command: "await in-flight {{service}}", intervalMs: 20, timeoutMs: 200},
|
|
112
|
+
projectDirectory,
|
|
113
|
+
service: "gateway",
|
|
114
|
+
stopGracePeriodMs: 30000,
|
|
115
|
+
...composeOverrides
|
|
116
|
+
}],
|
|
117
|
+
composeStatePath: path.join(projectDirectory, "compose.state.json"),
|
|
118
|
+
control: {path: path.join(projectDirectory, "rollbridge.sock")},
|
|
119
|
+
processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
120
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
121
|
+
}))
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* @param {string} prefix - Command prefix that should fail.
|
|
126
|
+
* @param {string} stderr - Failure output.
|
|
127
|
+
* @returns {(command: string) => CommandResult} Shell handler failing every command with this prefix.
|
|
128
|
+
*/
|
|
129
|
+
function failingShell(prefix, stderr) {
|
|
130
|
+
return (command) => command.startsWith(prefix)
|
|
131
|
+
? {code: 1, signal: null, stderr, stdout: ""}
|
|
132
|
+
: {code: 0, signal: null, stderr: "", stdout: ""}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
test("rolls one Compose unit drain-first and retains the previous image", async () => {
|
|
136
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE})
|
|
137
|
+
const config = buildConfig(world.projectDirectory)
|
|
138
|
+
const unit = config.compose[0]
|
|
139
|
+
|
|
140
|
+
try {
|
|
141
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
142
|
+
|
|
143
|
+
expect(result.status).toBe("rolled")
|
|
144
|
+
expect(result.stage).toBe("activation")
|
|
145
|
+
expect(result.image).toBe(TARGET_IMAGE)
|
|
146
|
+
expect(result.previousImage).toBe(CURRENT_IMAGE)
|
|
147
|
+
expect(result.generations.map((generation) => [generation.image, generation.status])).toEqual([[TARGET_IMAGE, "activated"]])
|
|
148
|
+
expect(world.state.recreateImages).toEqual([TARGET_IMAGE])
|
|
149
|
+
|
|
150
|
+
const commands = world.calls.map((call) => call.command).filter(Boolean)
|
|
151
|
+
|
|
152
|
+
expect(commands).toEqual(["quiesce gateway", "await in-flight gateway"])
|
|
153
|
+
|
|
154
|
+
const recreateArgs = /** @type {string[]} */ (world.calls.find((call) => call.args?.includes("up"))?.args)
|
|
155
|
+
|
|
156
|
+
expect(recreateArgs.slice(-5)).toEqual(["up", "-d", "--no-deps", "--force-recreate", "gateway"])
|
|
157
|
+
expect(recreateArgs).toContain("--no-deps")
|
|
158
|
+
|
|
159
|
+
const overridePath = recreateArgs[recreateArgs.findIndex((token) => token.endsWith(".override.yml"))]
|
|
160
|
+
|
|
161
|
+
expect(overridePath.startsWith(world.projectDirectory)).toBeTruthy()
|
|
162
|
+
|
|
163
|
+
const ledger = await readComposeLedger({config, unit})
|
|
164
|
+
|
|
165
|
+
expect(ledger.generations[0].previousImage).toBe(CURRENT_IMAGE)
|
|
166
|
+
expect(ledger.generations[0].releaseId).toBe(result.releaseId)
|
|
167
|
+
await expect(fs.stat(overridePath)).rejects.toThrow()
|
|
168
|
+
} finally {
|
|
169
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
170
|
+
}
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
test("the generated override pins the target image and the stop grace period", async () => {
|
|
174
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE})
|
|
175
|
+
const config = buildConfig(world.projectDirectory)
|
|
176
|
+
/** @type {string | undefined} */
|
|
177
|
+
let overrideContents
|
|
178
|
+
|
|
179
|
+
try {
|
|
180
|
+
/** @type {import("../src/compose-unit.js").CommandRunner} */
|
|
181
|
+
const recordingRunner = async (request) => {
|
|
182
|
+
if (request.args?.includes("up")) {
|
|
183
|
+
overrideContents = await fs.readFile(request.args[request.args.findIndex((token) => token.endsWith(".override.yml"))], "utf8")
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
return await world.runCommand(request)
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const result = await rollComposeUnit({config, runCommand: recordingRunner, unit: config.compose[0]})
|
|
190
|
+
|
|
191
|
+
expect(result.status).toBe("rolled")
|
|
192
|
+
expect(overrideContents).toMatch(/image: "registry\.local\/gateway:2\.0\.0"/)
|
|
193
|
+
expect(overrideContents).toMatch(/stop_grace_period: 30s/)
|
|
194
|
+
} finally {
|
|
195
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
196
|
+
}
|
|
197
|
+
})
|
|
198
|
+
|
|
199
|
+
test("a unit that cannot be quiesced fails the roll and is never recreated", async () => {
|
|
200
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE, shell: failingShell("quiesce", "gateway refused to drain: drain socket timeout")})
|
|
201
|
+
const config = buildConfig(world.projectDirectory)
|
|
202
|
+
const unit = config.compose[0]
|
|
203
|
+
|
|
204
|
+
try {
|
|
205
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
206
|
+
|
|
207
|
+
expect(result.status).toBe("failed")
|
|
208
|
+
expect(result.stage).toBe("drain")
|
|
209
|
+
expect(result.error).toMatch(/could not be quiesced/)
|
|
210
|
+
expect(result.error).toMatch(/gateway refused to drain/)
|
|
211
|
+
expect(world.state.recreateImages).toEqual([])
|
|
212
|
+
expect(world.state.image).toBe(CURRENT_IMAGE)
|
|
213
|
+
expect(world.state.containerId).toBe("container-1")
|
|
214
|
+
expect(world.calls.some((call) => call.args?.includes("up"))).toBe(false)
|
|
215
|
+
expect((await readComposeLedger({config, unit})).generations).toEqual([])
|
|
216
|
+
} finally {
|
|
217
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
218
|
+
}
|
|
219
|
+
})
|
|
220
|
+
|
|
221
|
+
test("in-flight work that never settles fails the roll and keeps the current container", async () => {
|
|
222
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE, shell: failingShell("await", "3 handoffs still running")})
|
|
223
|
+
const config = buildConfig(world.projectDirectory, {inFlight: {command: "await in-flight {{service}}", intervalMs: 20, timeoutMs: 60}})
|
|
224
|
+
|
|
225
|
+
try {
|
|
226
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})
|
|
227
|
+
|
|
228
|
+
expect(result.status).toBe("failed")
|
|
229
|
+
expect(result.stage).toBe("in_flight")
|
|
230
|
+
expect(result.error).toMatch(/still had in-flight work after 60ms/)
|
|
231
|
+
expect(result.error).toMatch(/3 handoffs still running/)
|
|
232
|
+
expect(world.state.recreateImages).toEqual([])
|
|
233
|
+
expect(world.state.image).toBe(CURRENT_IMAGE)
|
|
234
|
+
} finally {
|
|
235
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
236
|
+
}
|
|
237
|
+
})
|
|
238
|
+
|
|
239
|
+
test("in-flight work that settles after a retry lets the roll continue", async () => {
|
|
240
|
+
let attempts = 0
|
|
241
|
+
const world = await createDockerWorld({
|
|
242
|
+
image: CURRENT_IMAGE,
|
|
243
|
+
shell: (command) => {
|
|
244
|
+
if (!command.startsWith("await")) return {code: 0, signal: null, stderr: "", stdout: ""}
|
|
245
|
+
|
|
246
|
+
attempts += 1
|
|
247
|
+
|
|
248
|
+
return attempts < 3
|
|
249
|
+
? {code: 1, signal: null, stderr: "1 handoff still running", stdout: ""}
|
|
250
|
+
: {code: 0, signal: null, stderr: "", stdout: ""}
|
|
251
|
+
}
|
|
252
|
+
})
|
|
253
|
+
const config = buildConfig(world.projectDirectory, {inFlight: {command: "await in-flight {{service}}", intervalMs: 20, timeoutMs: 500}})
|
|
254
|
+
|
|
255
|
+
try {
|
|
256
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})
|
|
257
|
+
|
|
258
|
+
expect(result.status).toBe("rolled")
|
|
259
|
+
expect(attempts).toBe(3)
|
|
260
|
+
expect(world.state.recreateImages).toEqual([TARGET_IMAGE])
|
|
261
|
+
} finally {
|
|
262
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
263
|
+
}
|
|
264
|
+
})
|
|
265
|
+
|
|
266
|
+
test("a refused recreate fails the roll and leaves the previous container serving", async () => {
|
|
267
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE, recreateError: "Error response from daemon: No such image: registry.local/gateway:2.0.0"})
|
|
268
|
+
const config = buildConfig(world.projectDirectory)
|
|
269
|
+
const unit = config.compose[0]
|
|
270
|
+
|
|
271
|
+
try {
|
|
272
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
273
|
+
|
|
274
|
+
expect(result.status).toBe("failed")
|
|
275
|
+
expect(result.stage).toBe("recreate")
|
|
276
|
+
expect(result.image).toBe(CURRENT_IMAGE)
|
|
277
|
+
expect(result.previousImage).toBe(CURRENT_IMAGE)
|
|
278
|
+
expect(result.rollback).toBeUndefined()
|
|
279
|
+
expect(result.error).toMatch(/Compose recreate failed for unit "gateway" \(image registry\.local\/gateway:2\.0\.0, previous registry\.local\/gateway:1\.4\.0\)/)
|
|
280
|
+
expect(result.error).toMatch(/No such image/)
|
|
281
|
+
expect(world.state.recreateImages).toEqual([])
|
|
282
|
+
expect(world.state.image).toBe(CURRENT_IMAGE)
|
|
283
|
+
expect(world.state.containerId).toBe("container-1")
|
|
284
|
+
expect(world.state.state).toBe("running")
|
|
285
|
+
expect(world.calls.filter((call) => call.args?.includes("up"))).toHaveLength(1)
|
|
286
|
+
expect((await readComposeLedger({config, unit})).generations).toEqual([])
|
|
287
|
+
expect(await fs.stat(path.join(world.projectDirectory, ".rollbridge-gateway.override.yml")).then(() => true, () => false)).toBe(false)
|
|
288
|
+
} finally {
|
|
289
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
290
|
+
}
|
|
291
|
+
})
|
|
292
|
+
|
|
293
|
+
test("a candidate image that never activates is rolled back to the previous image", async () => {
|
|
294
|
+
const world = await createDockerWorld({healthFor: (image) => image === TARGET_IMAGE ? "unhealthy" : "healthy", image: CURRENT_IMAGE})
|
|
295
|
+
const config = buildConfig(world.projectDirectory, {activation: {intervalMs: 20, timeoutMs: 120}})
|
|
296
|
+
|
|
297
|
+
try {
|
|
298
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})
|
|
299
|
+
|
|
300
|
+
expect(result.status).toBe("rolled_back")
|
|
301
|
+
expect(result.stage).toBe("rollback")
|
|
302
|
+
expect(result.error).toMatch(/did not activate within 120ms/)
|
|
303
|
+
expect(result.rollback).toMatchObject({image: CURRENT_IMAGE, status: "rolled_back"})
|
|
304
|
+
expect(world.state.recreateImages).toEqual([TARGET_IMAGE, CURRENT_IMAGE])
|
|
305
|
+
expect(world.state.image).toBe(CURRENT_IMAGE)
|
|
306
|
+
expect(world.state.health).toBe("healthy")
|
|
307
|
+
expect(result.generations.map((generation) => [generation.image, generation.status])).toEqual([[CURRENT_IMAGE, "rolled_back"]])
|
|
308
|
+
expect(result.generations[0].previousImage).toBe(TARGET_IMAGE)
|
|
309
|
+
} finally {
|
|
310
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
311
|
+
}
|
|
312
|
+
})
|
|
313
|
+
|
|
314
|
+
test("a failed rollback is reported as a failed roll instead of a success", async () => {
|
|
315
|
+
const world = await createDockerWorld({healthFor: () => "unhealthy", image: CURRENT_IMAGE})
|
|
316
|
+
const config = buildConfig(world.projectDirectory, {activation: {intervalMs: 20, timeoutMs: 100}})
|
|
317
|
+
const unit = config.compose[0]
|
|
318
|
+
|
|
319
|
+
try {
|
|
320
|
+
const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
321
|
+
|
|
322
|
+
expect(result.status).toBe("failed")
|
|
323
|
+
expect(result.stage).toBe("rollback_failed")
|
|
324
|
+
expect(result.error).toMatch(/rollback to registry\.local\/gateway:1\.4\.0 did not activate/)
|
|
325
|
+
expect((await readComposeLedger({config, unit})).generations).toEqual([])
|
|
326
|
+
} finally {
|
|
327
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
328
|
+
}
|
|
329
|
+
})
|
|
330
|
+
|
|
331
|
+
test("compose-rollback selects the retained previous image", async () => {
|
|
332
|
+
const world = await createDockerWorld({image: OLD_IMAGE})
|
|
333
|
+
const config = buildConfig(world.projectDirectory)
|
|
334
|
+
const unit = config.compose[0]
|
|
335
|
+
|
|
336
|
+
try {
|
|
337
|
+
await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
338
|
+
|
|
339
|
+
const result = await rollbackComposeUnit({config, runCommand: world.runCommand, unit})
|
|
340
|
+
|
|
341
|
+
expect(result.status).toBe("rolled")
|
|
342
|
+
expect(result.image).toBe(OLD_IMAGE)
|
|
343
|
+
expect(result.previousImage).toBe(TARGET_IMAGE)
|
|
344
|
+
expect(world.state.recreateImages).toEqual([TARGET_IMAGE, OLD_IMAGE])
|
|
345
|
+
expect(world.state.image).toBe(OLD_IMAGE)
|
|
346
|
+
} finally {
|
|
347
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
348
|
+
}
|
|
349
|
+
})
|
|
350
|
+
|
|
351
|
+
test("compose-rollback refuses when no earlier image is retained", async () => {
|
|
352
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE})
|
|
353
|
+
const config = buildConfig(world.projectDirectory)
|
|
354
|
+
|
|
355
|
+
try {
|
|
356
|
+
await expect(rollbackComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})).rejects.toThrow(/No earlier image is retained for Compose unit "gateway"/)
|
|
357
|
+
} finally {
|
|
358
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
359
|
+
}
|
|
360
|
+
})
|
|
361
|
+
|
|
362
|
+
test("refuses to roll a unit whose container publishes the Rollbridge proxy port", async () => {
|
|
363
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE, ports: [{HostPort: "8182", containerPort: "8182"}]})
|
|
364
|
+
const config = buildConfig(world.projectDirectory)
|
|
365
|
+
|
|
366
|
+
try {
|
|
367
|
+
await expect(rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})).rejects.toThrow(/publishes port 8182, the Rollbridge proxy port/)
|
|
368
|
+
expect(world.state.recreateImages).toEqual([])
|
|
369
|
+
} finally {
|
|
370
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
371
|
+
}
|
|
372
|
+
})
|
|
373
|
+
|
|
374
|
+
test("refuses to roll while a release generation transition is unresolved", async () => {
|
|
375
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE})
|
|
376
|
+
const config = buildConfig(world.projectDirectory)
|
|
377
|
+
|
|
378
|
+
try {
|
|
379
|
+
await expect(rollComposeUnit({
|
|
380
|
+
config,
|
|
381
|
+
inspectSocket: async () => ({activeReleaseId: "v1", alive: true, application: "compose-test"}),
|
|
382
|
+
runCommand: world.runCommand,
|
|
383
|
+
sendCommand: async () => ({
|
|
384
|
+
application: "compose-test",
|
|
385
|
+
generationTransition: {candidateReleaseId: "v2", phase: "activating_candidate", previousReleaseId: "v1"},
|
|
386
|
+
status: "success"
|
|
387
|
+
}),
|
|
388
|
+
unit: config.compose[0]
|
|
389
|
+
})).rejects.toThrow(/while release generation transition v2 is unresolved at activating_candidate/)
|
|
390
|
+
|
|
391
|
+
expect(world.state.recreateImages).toEqual([])
|
|
392
|
+
} finally {
|
|
393
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
394
|
+
}
|
|
395
|
+
})
|
|
396
|
+
|
|
397
|
+
test("compose-status reports the live container and the retained generations", async () => {
|
|
398
|
+
const world = await createDockerWorld({image: CURRENT_IMAGE})
|
|
399
|
+
const config = buildConfig(world.projectDirectory)
|
|
400
|
+
const unit = config.compose[0]
|
|
401
|
+
|
|
402
|
+
try {
|
|
403
|
+
await rollComposeUnit({config, runCommand: world.runCommand, unit})
|
|
404
|
+
|
|
405
|
+
const status = await composeUnitStatus({config, runCommand: world.runCommand, unit})
|
|
406
|
+
|
|
407
|
+
expect(status.unitId).toBe("gateway")
|
|
408
|
+
expect(status.container).toMatchObject({containerId: "container-2", healthStatus: "healthy", image: TARGET_IMAGE, state: "running"})
|
|
409
|
+
expect(status.generations.map((generation) => generation.image)).toEqual([TARGET_IMAGE])
|
|
410
|
+
expect(status.ledgerPath).toBe(path.join(world.projectDirectory, "compose.state.json"))
|
|
411
|
+
} finally {
|
|
412
|
+
await fs.rm(world.projectDirectory, {force: true, recursive: true})
|
|
413
|
+
}
|
|
414
|
+
})
|
|
415
|
+
})
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import http from "node:http"
|
|
4
|
+
import fs from "node:fs"
|
|
5
|
+
|
|
6
|
+
const port = Number(requiredEnv("ROLLBRIDGE_PORT"))
|
|
7
|
+
const gateArg = process.argv.find((arg) => arg.startsWith("--gate="))
|
|
8
|
+
|
|
9
|
+
if (!gateArg) throw new Error("--gate=<path> is required")
|
|
10
|
+
|
|
11
|
+
const gatePath = gateArg.slice("--gate=".length)
|
|
12
|
+
const releaseId = process.env.ROLLBRIDGE_RELEASE_ID || "unknown"
|
|
13
|
+
// v3 stays unhealthy until the gate file appears, so its deploy fails at health.
|
|
14
|
+
const unhealthy = releaseId === "v3" && !fs.existsSync(gatePath)
|
|
15
|
+
|
|
16
|
+
const server = http.createServer((request, response) => {
|
|
17
|
+
if (unhealthy) {
|
|
18
|
+
response.writeHead(500)
|
|
19
|
+
response.end("bad")
|
|
20
|
+
return
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (request.url === "/ping") {
|
|
24
|
+
response.writeHead(200, {"Content-Type": "application/json"})
|
|
25
|
+
response.end(JSON.stringify({message: "Pong", releaseId}))
|
|
26
|
+
return
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
response.writeHead(200, {"Content-Type": "text/plain; charset=utf-8"})
|
|
30
|
+
response.end(`${releaseId}\n`)
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
process.on("SIGTERM", () => {
|
|
34
|
+
server.close(() => process.exit(0))
|
|
35
|
+
setTimeout(() => process.exit(0), 10).unref()
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
server.listen(port, "127.0.0.1")
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* @param {string} key - Environment variable name.
|
|
42
|
+
* @returns {string} Environment variable value.
|
|
43
|
+
*/
|
|
44
|
+
function requiredEnv(key) {
|
|
45
|
+
const value = process.env[key]
|
|
46
|
+
|
|
47
|
+
if (!value) throw new Error(`${key} is required`)
|
|
48
|
+
|
|
49
|
+
return value
|
|
50
|
+
}
|
|
@@ -58,6 +58,76 @@ test("guardian inventory removes only an exact owned provenance", async () => {
|
|
|
58
58
|
}
|
|
59
59
|
})
|
|
60
60
|
|
|
61
|
+
test("a refused owner mutation names the held mutation, its process, its age and its blocker", async () => {
|
|
62
|
+
const fixture = await createGuardian()
|
|
63
|
+
|
|
64
|
+
try {
|
|
65
|
+
const mutationId = await fixture.client.beginOwnerMutation("restart", {ownerPid: process.pid, waitingFor: "the managed process to stop and start again"})
|
|
66
|
+
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/^Another owner mutation is already in progress: owner mutation "restart"; mutation [0-9a-f-]{36}; held by process \d+; since 20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ; \d+s elapsed; waiting for the managed process to stop and start again; refused deploy$/)
|
|
67
|
+
await fixture.client.reportOwnerMutationWait(mutationId, "an idle forked runner that never exits")
|
|
68
|
+
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/waiting for an idle forked runner that never exits/)
|
|
69
|
+
await fixture.client.endOwnerMutation(mutationId)
|
|
70
|
+
expect(typeof await fixture.client.beginOwnerMutation("deploy")).toBe("string")
|
|
71
|
+
} finally {
|
|
72
|
+
await cleanupGuardian(fixture)
|
|
73
|
+
}
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
test("an owner mutation that outruns its lease is released and the holder reports the expiry", async () => {
|
|
77
|
+
const fixture = await createGuardian()
|
|
78
|
+
|
|
79
|
+
try {
|
|
80
|
+
const wedged = await fixture.client.beginOwnerMutation("restart", {mutationTimeoutMs: 200, ownerPid: process.pid, waitingFor: "a drain that never returns"})
|
|
81
|
+
|
|
82
|
+
await new Promise((resolve) => setTimeout(resolve, 600))
|
|
83
|
+
const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
|
|
84
|
+
|
|
85
|
+
expect(typeof successor).toBe("string")
|
|
86
|
+
await expect(fixture.client.endOwnerMutation(wedged)).rejects.toThrow(/^Owner mutation restart exceeded its 200 ms lease and was released after \d+ ms; the mutation failed and its lease is gone$/)
|
|
87
|
+
await fixture.client.endOwnerMutation(successor)
|
|
88
|
+
} finally {
|
|
89
|
+
await cleanupGuardian(fixture)
|
|
90
|
+
}
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
|
|
94
|
+
const fixture = await createGuardian()
|
|
95
|
+
const liveInstance = fixture.client.process("live-generation", definition("live-generation"))
|
|
96
|
+
const deadInstance = fixture.client.process("dead-generation", definition("dead-generation"))
|
|
97
|
+
|
|
98
|
+
try {
|
|
99
|
+
await liveInstance.start()
|
|
100
|
+
await deadInstance.start()
|
|
101
|
+
const deadPid = (await deadInstance.status()).pid
|
|
102
|
+
|
|
103
|
+
if (!deadPid) throw new Error("Missing required fixture: dead-generation pid")
|
|
104
|
+
await deadInstance.stop()
|
|
105
|
+
await waitForProcessExit(deadPid)
|
|
106
|
+
const deadStatus = await deadInstance.status()
|
|
107
|
+
|
|
108
|
+
expect(deadStatus.state).toBe("stopped")
|
|
109
|
+
expect(deadStatus.pid).toBeUndefined()
|
|
110
|
+
|
|
111
|
+
// The wedge this guards against: a later deploy reconstructs the same key from a
|
|
112
|
+
// different release definition. The old child is conclusively gone, so the old
|
|
113
|
+
// provenance must not fence the new registration.
|
|
114
|
+
const nextInstance = fixture.client.process("dead-generation", {...definition("dead-generation"), env: {REPLACED_BY_LATER_DEPLOY: "1"}})
|
|
115
|
+
const reRegistered = nextInstance.status()
|
|
116
|
+
|
|
117
|
+
expect(reRegistered.state).toBe("stopped")
|
|
118
|
+
await nextInstance.start()
|
|
119
|
+
expect((await nextInstance.status()).state).toBe("running")
|
|
120
|
+
expect((await fixture.client.inventory()).some((entry) => entry.key === "dead-generation" && entry.provenance === nextInstance.provenance)).toBe(true)
|
|
121
|
+
|
|
122
|
+
// A live process with a different provenance stays fail-closed.
|
|
123
|
+
const hostileInstance = fixture.client.process("live-generation", {...definition("live-generation"), env: {HOSTILE: "1"}})
|
|
124
|
+
|
|
125
|
+
await expect(hostileInstance.start()).rejects.toThrow(/provenance mismatch/)
|
|
126
|
+
} finally {
|
|
127
|
+
await cleanupGuardian(fixture)
|
|
128
|
+
}
|
|
129
|
+
})
|
|
130
|
+
|
|
61
131
|
test("guardian runs a strict activation lifecycle command for the exact registered process", async () => {
|
|
62
132
|
const fixture = await createGuardian()
|
|
63
133
|
const activationPath = path.join(fixture.root, "activated")
|
package/test/recover.test.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import {spawn} from "node:child_process"
|
|
4
|
+
import net from "node:net"
|
|
4
5
|
import {once} from "node:events"
|
|
5
6
|
import fs from "node:fs/promises"
|
|
6
7
|
import os from "node:os"
|
|
@@ -166,4 +167,122 @@ test("recover refuses while a daemon is running", async () => {
|
|
|
166
167
|
await fs.rm(dir, {force: true, recursive: true})
|
|
167
168
|
}
|
|
168
169
|
})
|
|
170
|
+
|
|
171
|
+
test("recover archives a state naming a dead guardian once every orphan is stopped", async () => {
|
|
172
|
+
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-recover-"))
|
|
173
|
+
|
|
174
|
+
try {
|
|
175
|
+
const statePath = path.join(dir, "rollbridge-state.json")
|
|
176
|
+
const guardianSocketPath = `${statePath}.guardian.sock`
|
|
177
|
+
const orphan = await spawnOrphan()
|
|
178
|
+
const orphanPid = orphan.pid ?? 0
|
|
179
|
+
const deadPid = await spawnDeadPid()
|
|
180
|
+
const listener = net.createServer()
|
|
181
|
+
|
|
182
|
+
await once(listener.listen(guardianSocketPath), "listening")
|
|
183
|
+
await new Promise((resolve) => listener.close(resolve))
|
|
184
|
+
|
|
185
|
+
await writeState(statePath, {
|
|
186
|
+
activeReleaseId: "v1",
|
|
187
|
+
recovery: {
|
|
188
|
+
configDigest: "digest",
|
|
189
|
+
format: 1,
|
|
190
|
+
guardian: {pid: deadPid, socketPath: guardianSocketPath, token: "token"},
|
|
191
|
+
reconnectGraceMs: 30000
|
|
192
|
+
},
|
|
193
|
+
releases: [{processes: [{id: "worker", pid: orphanPid}], releaseId: "v1"}],
|
|
194
|
+
services: [],
|
|
195
|
+
singletons: []
|
|
196
|
+
})
|
|
197
|
+
|
|
198
|
+
const result = await recoverOrphans({config: buildConfig(dir, {statePath}), force: true})
|
|
199
|
+
|
|
200
|
+
expect("error" in result).toBeFalse()
|
|
201
|
+
|
|
202
|
+
if ("error" in result) return
|
|
203
|
+
|
|
204
|
+
expect(result.forced).toEqual(true)
|
|
205
|
+
expect(result.archived).toEqual(true)
|
|
206
|
+
expect(result.deadGuardianState).toEqual(true)
|
|
207
|
+
expect(result.archivePath).toBeTruthy()
|
|
208
|
+
expect(result.remaining).toEqual([])
|
|
209
|
+
|
|
210
|
+
const archiveBytes = await fs.readFile(/** @type {string} */ (result.archivePath), "utf8")
|
|
211
|
+
|
|
212
|
+
expect(JSON.parse(archiveBytes).activeReleaseId).toEqual("v1")
|
|
213
|
+
await expectMissing(statePath)
|
|
214
|
+
await expectMissing(guardianSocketPath)
|
|
215
|
+
} finally {
|
|
216
|
+
await fs.rm(dir, {force: true, recursive: true})
|
|
217
|
+
}
|
|
218
|
+
})
|
|
219
|
+
|
|
220
|
+
test("recover leaves the state alone when a guardian listener is still live", async () => {
|
|
221
|
+
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-recover-"))
|
|
222
|
+
|
|
223
|
+
try {
|
|
224
|
+
const statePath = path.join(dir, "rollbridge-state.json")
|
|
225
|
+
const guardianSocketPath = `${statePath}.guardian.sock`
|
|
226
|
+
const listener = net.createServer()
|
|
227
|
+
|
|
228
|
+
await once(listener.listen(guardianSocketPath), "listening")
|
|
229
|
+
|
|
230
|
+
try {
|
|
231
|
+
await writeState(statePath, {
|
|
232
|
+
activeReleaseId: "v1",
|
|
233
|
+
recovery: {
|
|
234
|
+
configDigest: "digest",
|
|
235
|
+
format: 1,
|
|
236
|
+
guardian: {socketPath: guardianSocketPath, token: "token"},
|
|
237
|
+
reconnectGraceMs: 30000
|
|
238
|
+
},
|
|
239
|
+
releases: [],
|
|
240
|
+
services: [],
|
|
241
|
+
singletons: []
|
|
242
|
+
})
|
|
243
|
+
|
|
244
|
+
const result = await recoverOrphans({config: buildConfig(dir, {statePath}), force: true})
|
|
245
|
+
|
|
246
|
+
expect("error" in result).toBeFalse()
|
|
247
|
+
|
|
248
|
+
if ("error" in result) return
|
|
249
|
+
|
|
250
|
+
expect(result.archived).toEqual(false)
|
|
251
|
+
expect(result.deadGuardianState).toEqual(false)
|
|
252
|
+
expect(result.cleared).toEqual(true)
|
|
253
|
+
} finally {
|
|
254
|
+
await new Promise((resolve) => listener.close(resolve))
|
|
255
|
+
}
|
|
256
|
+
} finally {
|
|
257
|
+
await fs.rm(dir, {force: true, recursive: true})
|
|
258
|
+
}
|
|
259
|
+
})
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* @returns {Promise<number>} The pid of a process that has exited.
|
|
263
|
+
*/
|
|
264
|
+
async function spawnDeadPid() {
|
|
265
|
+
const child = spawn(process.execPath, ["-e", "true"], {detached: true, stdio: "ignore"})
|
|
266
|
+
const pid = child.pid ?? 0
|
|
267
|
+
|
|
268
|
+
await once(child, "exit")
|
|
269
|
+
await waitFor(() => !isProcessAlive(pid))
|
|
270
|
+
|
|
271
|
+
return pid
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* @param {string} targetPath - Path that must no longer exist.
|
|
276
|
+
* @returns {Promise<void>} Resolves when the path is gone.
|
|
277
|
+
*/
|
|
278
|
+
async function expectMissing(targetPath) {
|
|
279
|
+
try {
|
|
280
|
+
await fs.access(targetPath)
|
|
281
|
+
} catch {
|
|
282
|
+
return
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
throw new Error(`Expected ${targetPath} to be gone`)
|
|
286
|
+
}
|
|
287
|
+
|
|
169
288
|
})
|