rollbridge 0.1.56 → 0.1.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -0
- package/TODO.md +12 -0
- package/changelog.d/20261013-recover-dead-guardian-state.md +1 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/docs/cli.md +70 -1
- package/docs/compose-services.md +220 -0
- package/docs/config.md +55 -0
- package/docs/troubleshooting.md +24 -0
- package/package.json +1 -1
- package/src/cli.js +82 -4
- package/src/compose-unit.js +725 -0
- package/src/config.js +201 -2
- package/src/daemon.js +23 -8
- package/src/doctor.js +67 -0
- package/src/guardian-client.js +13 -2
- package/src/process-guardian.js +137 -17
- package/src/recover.js +128 -8
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +158 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +70 -0
- package/test/recover.test.js +119 -0
- package/test/rollbridge.test.js +106 -6
package/src/recover.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
+
import fs from "node:fs/promises"
|
|
4
|
+
import net from "node:net"
|
|
3
5
|
import {inspectControlSocket} from "./daemon.js"
|
|
4
6
|
import {clearState, isProcessAlive, liveProcesses, readState} from "./state-store.js"
|
|
5
7
|
|
|
@@ -7,9 +9,10 @@ import {clearState, isProcessAlive, liveProcesses, readState} from "./state-stor
|
|
|
7
9
|
const KILL_CONFIRM_TIMEOUT_MS = 500
|
|
8
10
|
|
|
9
11
|
/**
|
|
12
|
+
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
10
13
|
* @typedef {{id: string, pid: number, releaseId: string | null}} OrphanProcess
|
|
11
14
|
* @typedef {{error: string}} RecoverError
|
|
12
|
-
* @typedef {{cleared: boolean, forced: boolean, orphans: OrphanProcess[], remaining: OrphanProcess[]}} RecoverReport
|
|
15
|
+
* @typedef {{archived: boolean, archivePath?: string, cleared: boolean, deadGuardianState: boolean, forced: boolean, orphans: OrphanProcess[], remaining: OrphanProcess[]}} RecoverReport
|
|
13
16
|
* @typedef {RecoverError | RecoverReport} RecoverResult
|
|
14
17
|
*/
|
|
15
18
|
|
|
@@ -29,10 +32,11 @@ const KILL_CONFIRM_TIMEOUT_MS = 500
|
|
|
29
32
|
* @param {object} args - Options.
|
|
30
33
|
* @param {import("./config.js").RollbridgeConfig} args.config - Normalized config.
|
|
31
34
|
* @param {boolean} args.force - Whether to actually stop the orphans (otherwise list them).
|
|
35
|
+
* @param {(socketPath: string) => Promise<boolean | undefined>} [args.probeUnixListener] - Probes whether a Unix socket has a live listener (defaults to the real probe; injectable for tests).
|
|
32
36
|
* @param {(pid: number, timeoutMs: number) => Promise<boolean>} [args.stopGroup] - Stops a process group and resolves to whether it is gone afterward (defaults to the real implementation; injectable for tests).
|
|
33
37
|
* @returns {Promise<RecoverResult>} The orphans found and whether they were stopped, or an error.
|
|
34
38
|
*/
|
|
35
|
-
export async function recoverOrphans({config, force, stopGroup = stopProcessGroup}) {
|
|
39
|
+
export async function recoverOrphans({config, force, probeUnixListener = probeUnixListenerAt, stopGroup = stopProcessGroup}) {
|
|
36
40
|
if (config.statePath === undefined) {
|
|
37
41
|
return {error: "No statePath is configured; set statePath in the config to enable recovery."}
|
|
38
42
|
}
|
|
@@ -41,9 +45,11 @@ export async function recoverOrphans({config, force, stopGroup = stopProcessGrou
|
|
|
41
45
|
return {error: `A daemon (or another process) is using ${config.control.path}; stop it before recovering — recover is for cleaning up after a crash.`}
|
|
42
46
|
}
|
|
43
47
|
|
|
44
|
-
const
|
|
48
|
+
const state = await readState(config.statePath)
|
|
49
|
+
const deadGuardianState = await stateNamesDeadGuardian(state, probeUnixListener)
|
|
50
|
+
const orphans = liveProcesses(state)
|
|
45
51
|
|
|
46
|
-
if (!force) return {cleared: false, forced: false, orphans, remaining: []}
|
|
52
|
+
if (!force) return {archived: false, cleared: false, deadGuardianState, forced: false, orphans, remaining: []}
|
|
47
53
|
|
|
48
54
|
/** @type {OrphanProcess[]} */
|
|
49
55
|
const remaining = []
|
|
@@ -54,11 +60,35 @@ export async function recoverOrphans({config, force, stopGroup = stopProcessGrou
|
|
|
54
60
|
if (!stopped) remaining.push(orphan)
|
|
55
61
|
}
|
|
56
62
|
|
|
57
|
-
//
|
|
58
|
-
|
|
59
|
-
|
|
63
|
+
// Keep the state file while any orphan survives so the operator can still find and retry it.
|
|
64
|
+
if (remaining.length > 0) {
|
|
65
|
+
return {archived: false, cleared: false, deadGuardianState, forced: true, orphans, remaining}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// With every orphan gone, a conclusively dead guardian identity is the documented dead-end
|
|
69
|
+
// case: archive the state under a timestamped evidence name (bytes preserved verbatim) and
|
|
70
|
+
// move the dead socket metadata aside, so the next deploy can boot a fresh generation.
|
|
71
|
+
// See the TensorBuzz production runbook docs/production-rollbridge-recovery.md.
|
|
72
|
+
if (deadGuardianState) {
|
|
73
|
+
const archivePath = deadGuardianArchivePath(config.statePath)
|
|
74
|
+
const guardianSocketPath = stateGuardianIdentity(state)?.socketPath
|
|
75
|
+
|
|
76
|
+
if (guardianSocketPath !== undefined) {
|
|
77
|
+
try {
|
|
78
|
+
await fs.rename(guardianSocketPath, `${guardianSocketPath}.dead-guardian-${archiveStamp()}`)
|
|
79
|
+
} catch (error) {
|
|
80
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
await fs.rename(config.statePath, archivePath)
|
|
85
|
+
|
|
86
|
+
return {archived: true, archivePath, cleared: false, deadGuardianState, forced: true, orphans, remaining: []}
|
|
87
|
+
}
|
|
60
88
|
|
|
61
|
-
|
|
89
|
+
await clearState(config.statePath)
|
|
90
|
+
|
|
91
|
+
return {archived: false, cleared: true, deadGuardianState, forced: true, orphans, remaining: []}
|
|
62
92
|
}
|
|
63
93
|
|
|
64
94
|
/**
|
|
@@ -132,3 +162,93 @@ function sendSignal(pid, signal) {
|
|
|
132
162
|
return "denied"
|
|
133
163
|
}
|
|
134
164
|
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* @param {JsonValue} state - Persisted state.
|
|
168
|
+
* @returns {{pid?: number, socketPath: string} | undefined} The persisted guardian identity when present and shaped as expected.
|
|
169
|
+
*/
|
|
170
|
+
function stateGuardianIdentity(state) {
|
|
171
|
+
if (!state || typeof state !== "object" || Array.isArray(state)) return undefined
|
|
172
|
+
|
|
173
|
+
const recovery = /** @type {Record<string, JsonValue>} */ (state).recovery
|
|
174
|
+
|
|
175
|
+
if (!recovery || typeof recovery !== "object" || Array.isArray(recovery)) return undefined
|
|
176
|
+
|
|
177
|
+
const guardian = /** @type {Record<string, JsonValue>} */ (recovery).guardian
|
|
178
|
+
|
|
179
|
+
if (!guardian || typeof guardian !== "object" || Array.isArray(guardian)) return undefined
|
|
180
|
+
|
|
181
|
+
const socketPath = /** @type {Record<string, JsonValue>} */ (guardian).socketPath
|
|
182
|
+
|
|
183
|
+
if (typeof socketPath !== "string" || socketPath === "") return undefined
|
|
184
|
+
|
|
185
|
+
const pid = /** @type {Record<string, JsonValue>} */ (guardian).pid
|
|
186
|
+
|
|
187
|
+
return {pid: typeof pid === "number" ? pid : undefined, socketPath}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Whether the state conclusively names a dead guardian: its recorded pid is gone (when present)
|
|
192
|
+
* and nothing is listening on its Unix socket. Any ambiguity returns false so only a conclusive
|
|
193
|
+
* proof may archive the state.
|
|
194
|
+
* @param {JsonValue} state - Persisted state.
|
|
195
|
+
* @param {(socketPath: string) => Promise<boolean | undefined>} probeUnixListener - Listener probe.
|
|
196
|
+
* @returns {Promise<boolean>} True only on a conclusive dead-guardian proof.
|
|
197
|
+
*/
|
|
198
|
+
async function stateNamesDeadGuardian(state, probeUnixListener) {
|
|
199
|
+
const guardian = stateGuardianIdentity(state)
|
|
200
|
+
|
|
201
|
+
if (!guardian) return false
|
|
202
|
+
if (guardian.pid !== undefined && isProcessAlive(guardian.pid)) return false
|
|
203
|
+
|
|
204
|
+
return (await probeUnixListener(guardian.socketPath)) === false
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Probes whether anything is listening on a Unix socket path.
|
|
209
|
+
* @param {string} socketPath - Socket path.
|
|
210
|
+
* @returns {Promise<boolean | undefined>} True when a listener accepted the connection, false when conclusively nothing listens (refused or missing), undefined when the probe was inconclusive.
|
|
211
|
+
*/
|
|
212
|
+
function probeUnixListenerAt(socketPath) {
|
|
213
|
+
return new Promise((resolve) => {
|
|
214
|
+
const socket = net.createConnection(socketPath)
|
|
215
|
+
let settled = false
|
|
216
|
+
|
|
217
|
+
/** @param {boolean | undefined} result - Probe result. */
|
|
218
|
+
const finish = (result) => {
|
|
219
|
+
if (settled) return
|
|
220
|
+
|
|
221
|
+
settled = true
|
|
222
|
+
clearTimeout(timer)
|
|
223
|
+
socket.destroy()
|
|
224
|
+
resolve(result)
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const timer = setTimeout(() => finish(undefined), 1000)
|
|
228
|
+
|
|
229
|
+
socket.once("connect", () => finish(true))
|
|
230
|
+
socket.once("error", (error) => {
|
|
231
|
+
const code = error && typeof error === "object" && "code" in error ? error.code : undefined
|
|
232
|
+
|
|
233
|
+
finish(code === "ECONNREFUSED" || code === "ENOENT" ? false : undefined)
|
|
234
|
+
})
|
|
235
|
+
})
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* @returns {string} A UTC archive stamp in the established evidence-name shape (for example 20261013T1102Z).
|
|
240
|
+
*/
|
|
241
|
+
function archiveStamp() {
|
|
242
|
+
return `${new Date().toISOString().slice(0, 16).replace(/[-:]/g, "")}Z`
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* @param {string} statePath - State file path.
|
|
247
|
+
* @returns {string} Timestamped archive path preserving the state bytes as evidence.
|
|
248
|
+
*/
|
|
249
|
+
function deadGuardianArchivePath(statePath) {
|
|
250
|
+
const stamp = archiveStamp()
|
|
251
|
+
const suffix = statePath.endsWith(".json") ? ".json" : ""
|
|
252
|
+
|
|
253
|
+
return `${statePath.slice(0, statePath.length - suffix.length)}.dead-guardian-${stamp}${suffix}`
|
|
254
|
+
}
|
package/test/completion.test.js
CHANGED
|
@@ -42,10 +42,12 @@ test("completion bash prints a sourceable script with commands and option flags"
|
|
|
42
42
|
|
|
43
43
|
expect(code).not.toBe(1)
|
|
44
44
|
expect(output).toMatch(/complete -F _rollbridge rollbridge/)
|
|
45
|
-
expect(output).toMatch(/compgen -W "daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion"/)
|
|
45
|
+
expect(output).toMatch(/compgen -W "daemon deploy rollback recover-generation-transition compose-roll compose-rollback compose-status ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion"/)
|
|
46
46
|
// A command's own options are completed after the command.
|
|
47
47
|
expect(output).toMatch(/deploy\)\n\s+opts="[^"]*--release-path[^"]*"/)
|
|
48
48
|
expect(output).toMatch(/recover-generation-transition\)\n\s+opts="--config --release-path --release-id --revision --previous-release-id --accept-retired-incumbent"/)
|
|
49
|
+
expect(output).toMatch(/compose-roll\)\n\s+opts="--config --unit --image"/)
|
|
50
|
+
expect(output).toMatch(/compose-status\)\n\s+opts="--config --unit"/)
|
|
49
51
|
expect(output).toMatch(/ensure-daemon\)\n\s+opts="[^"]*--daemon-runtime-path[^"]*"/)
|
|
50
52
|
expect(output).toMatch(/restart\)\n\s+opts="[^"]*--policy[^"]*"/)
|
|
51
53
|
})
|
|
@@ -55,8 +57,9 @@ test("completion zsh prints a #compdef script with per-command options", async (
|
|
|
55
57
|
|
|
56
58
|
expect(output).toMatch(/^#compdef rollbridge/)
|
|
57
59
|
expect(output).toMatch(/compdef _rollbridge rollbridge/)
|
|
58
|
-
expect(output).toMatch(/commands=\(daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion\)/)
|
|
60
|
+
expect(output).toMatch(/commands=\(daemon deploy rollback recover-generation-transition compose-roll compose-rollback compose-status ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion\)/)
|
|
59
61
|
expect(output).toMatch(/recover-generation-transition\) compadd -- --config --release-path --release-id --revision --previous-release-id --accept-retired-incumbent/)
|
|
62
|
+
expect(output).toMatch(/compose-rollback\) compadd -- --config --unit --image/)
|
|
60
63
|
expect(output).toMatch(/events\) compadd -- [^\n]*--limit/)
|
|
61
64
|
})
|
|
62
65
|
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import path from "node:path"
|
|
4
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
5
|
+
import {validateConfig} from "../src/config.js"
|
|
6
|
+
|
|
7
|
+
describe("compose-config", () => {
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} [unitOverrides] - Compose unit fields.
|
|
11
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} [configOverrides] - Top-level overrides.
|
|
12
|
+
* @returns {Record<string, import("../src/json.js").JsonValue>} Raw config with one Compose unit.
|
|
13
|
+
*/
|
|
14
|
+
function buildRawConfig(unitOverrides = {}, configOverrides = {}) {
|
|
15
|
+
return {
|
|
16
|
+
application: "compose-config-test",
|
|
17
|
+
compose: [{
|
|
18
|
+
id: "gateway",
|
|
19
|
+
image: "registry.local/gateway:1.4.0",
|
|
20
|
+
projectDirectory: "/srv/gateway",
|
|
21
|
+
service: "gateway",
|
|
22
|
+
...unitOverrides
|
|
23
|
+
}],
|
|
24
|
+
control: {path: "/tmp/rollbridge-compose-config.sock"},
|
|
25
|
+
processes: [{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}],
|
|
26
|
+
proxy: {host: "127.0.0.1", port: 8182},
|
|
27
|
+
...configOverrides
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {{fix: string, message: string}[]} issues - Collected issues.
|
|
33
|
+
* @returns {string[]} Issue messages.
|
|
34
|
+
*/
|
|
35
|
+
function messages(issues) {
|
|
36
|
+
return issues.map((issue) => issue.message)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* @param {import("../src/config.js").RollbridgeConfig} config - Validated config.
|
|
41
|
+
* @returns {import("../src/config.js").ComposeUnitConfig} The one Compose unit the test declared.
|
|
42
|
+
*/
|
|
43
|
+
function onlyUnit(config) {
|
|
44
|
+
const unit = config.compose?.[0]
|
|
45
|
+
|
|
46
|
+
if (unit === undefined) throw new Error("the test config declares no compose unit")
|
|
47
|
+
|
|
48
|
+
return unit
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
test("a Compose unit is normalized with drain-first defaults", () => {
|
|
52
|
+
const {config, issues} = validateConfig(buildRawConfig(), "/srv/app/rollbridge.js")
|
|
53
|
+
|
|
54
|
+
expect(issues).toEqual([])
|
|
55
|
+
|
|
56
|
+
const unit = onlyUnit(config)
|
|
57
|
+
|
|
58
|
+
expect(unit.id).toBe("gateway")
|
|
59
|
+
expect(unit.composeFile).toBe("compose.yml")
|
|
60
|
+
expect(unit.stopGracePeriodMs).toBe(10000)
|
|
61
|
+
expect(unit.recreateTimeoutMs).toBe(120000)
|
|
62
|
+
expect(unit.drain).toEqual({command: undefined, timeoutMs: 60000})
|
|
63
|
+
expect(unit.inFlight).toEqual({command: undefined, intervalMs: 1000, timeoutMs: 0})
|
|
64
|
+
expect(unit.activation).toMatchObject({healthcheck: true, intervalMs: 1000, startDelayMs: 0, timeoutMs: 60000})
|
|
65
|
+
expect(unit.rollback).toEqual({enabled: true})
|
|
66
|
+
expect(config.composeStatePath).toBe(path.resolve("/srv/app", "rollbridge-compose-config-test.compose.state.json"))
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
test("a Compose unit keeps its own compose file, env file, project name, and hooks", () => {
|
|
70
|
+
const {config, issues} = validateConfig(buildRawConfig({
|
|
71
|
+
activation: {intervalMs: 500, readyCommand: "probe {{service}}", readyUrl: "http://127.0.0.1:7421/ready", startDelayMs: 2000, timeoutMs: 90000},
|
|
72
|
+
composeFile: "gateway.compose.yml",
|
|
73
|
+
drain: {command: "docker compose {{composeArgs}} exec {{service}} appctl drain", timeoutMs: 30000},
|
|
74
|
+
envFile: "gateway.env",
|
|
75
|
+
inFlight: {command: "docker compose {{composeArgs}} exec {{service}} appctl in-flight", intervalMs: 2000, timeoutMs: 120000},
|
|
76
|
+
projectName: "socketduct-gateway",
|
|
77
|
+
rollback: {enabled: false},
|
|
78
|
+
stopGracePeriodMs: 45000
|
|
79
|
+
}))
|
|
80
|
+
|
|
81
|
+
expect(issues).toEqual([])
|
|
82
|
+
|
|
83
|
+
const unit = onlyUnit(config)
|
|
84
|
+
|
|
85
|
+
expect(unit.composeFile).toBe("gateway.compose.yml")
|
|
86
|
+
expect(unit.envFile).toBe("gateway.env")
|
|
87
|
+
expect(unit.projectName).toBe("socketduct-gateway")
|
|
88
|
+
expect(unit.stopGracePeriodMs).toBe(45000)
|
|
89
|
+
expect(unit.rollback.enabled).toBe(false)
|
|
90
|
+
expect(unit.drain.timeoutMs).toBe(30000)
|
|
91
|
+
expect(unit.inFlight.timeoutMs).toBe(120000)
|
|
92
|
+
expect(unit.activation.readyUrl).toBe("http://127.0.0.1:7421/ready")
|
|
93
|
+
expect(unit.activation.readyCommand).toBe("probe {{service}}")
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
test("an unpinned or malformed image reference is rejected", () => {
|
|
97
|
+
const whitespace = validateConfig(buildRawConfig({image: "registry.local/gateway: 1.4.0"}))
|
|
98
|
+
const digest = validateConfig(buildRawConfig({image: "registry.local/gateway@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}))
|
|
99
|
+
const badDigest = validateConfig(buildRawConfig({image: "registry.local/gateway@sha256:nope"}))
|
|
100
|
+
|
|
101
|
+
expect(whitespace.issues.map((issue) => issue.message)).toEqual([
|
|
102
|
+
'compose[0].image "registry.local/gateway: 1.4.0" is not a valid image reference'
|
|
103
|
+
])
|
|
104
|
+
expect(digest.issues).toEqual([])
|
|
105
|
+
expect(messages(badDigest.issues)).toContain('compose[0].image "registry.local/gateway@sha256:nope" is not a valid image reference')
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
test("compose unit ids must be unique and must not collide with process ids", () => {
|
|
109
|
+
const duplicate = validateConfig({
|
|
110
|
+
...buildRawConfig(),
|
|
111
|
+
compose: [
|
|
112
|
+
{id: "gateway", image: "registry.local/gateway:1.4.0", projectDirectory: "/srv/gateway", service: "gateway"},
|
|
113
|
+
{id: "gateway", image: "registry.local/hub:1.4.0", projectDirectory: "/srv/hub", service: "hub"}
|
|
114
|
+
]
|
|
115
|
+
})
|
|
116
|
+
const collision = validateConfig(buildRawConfig({id: "web"}))
|
|
117
|
+
|
|
118
|
+
expect(messages(duplicate.issues)).toContain("Duplicate compose unit id: gateway")
|
|
119
|
+
expect(messages(collision.issues)).toContain('Compose unit id "web" collides with a process id')
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
test("a drain or in-flight command without a positive timeout is rejected", () => {
|
|
123
|
+
const drain = validateConfig(buildRawConfig({drain: {command: "appctl drain", timeoutMs: 0}}))
|
|
124
|
+
const inFlight = validateConfig(buildRawConfig({inFlight: {command: "appctl in-flight"}}))
|
|
125
|
+
|
|
126
|
+
expect(messages(drain.issues)).toContain("compose[0].drain.command requires a positive compose[0].drain.timeoutMs")
|
|
127
|
+
expect(messages(inFlight.issues)).toContain("compose[0].inFlight.command requires a positive compose[0].inFlight.timeoutMs")
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
test("a compose unit must name its service, project directory, and image", () => {
|
|
131
|
+
const {issues} = validateConfig(buildRawConfig({image: "", projectDirectory: "", service: ""}))
|
|
132
|
+
|
|
133
|
+
expect(messages(issues)).toEqual([
|
|
134
|
+
"compose[0].service is required",
|
|
135
|
+
"compose[0].projectDirectory is required",
|
|
136
|
+
"compose[0].image is required"
|
|
137
|
+
])
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
test("an empty compose list is rejected", () => {
|
|
141
|
+
const {issues} = validateConfig(buildRawConfig({}, {compose: []}))
|
|
142
|
+
|
|
143
|
+
expect(messages(issues)).toEqual(["compose must define at least one unit"])
|
|
144
|
+
})
|
|
145
|
+
|
|
146
|
+
test("compose must be a list of unit mappings", () => {
|
|
147
|
+
const notArray = validateConfig(buildRawConfig({}, {compose: {id: "gateway"}}))
|
|
148
|
+
const notObject = validateConfig(buildRawConfig({}, {compose: ["gateway"]}))
|
|
149
|
+
|
|
150
|
+
expect(messages(notArray.issues)).toEqual(["compose must be an array"])
|
|
151
|
+
expect(messages(notObject.issues)).toEqual([
|
|
152
|
+
"compose[0] must be an object",
|
|
153
|
+
"compose[0].service is required",
|
|
154
|
+
"compose[0].projectDirectory is required",
|
|
155
|
+
"compose[0].image is required"
|
|
156
|
+
])
|
|
157
|
+
})
|
|
158
|
+
})
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises"
|
|
4
|
+
import os from "node:os"
|
|
5
|
+
import path from "node:path"
|
|
6
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
7
|
+
import {normalizeConfig} from "../src/config.js"
|
|
8
|
+
import {readComposeLedger, rollComposeUnit, runComposeCommand} from "../src/compose-unit.js"
|
|
9
|
+
|
|
10
|
+
// Real Docker Compose proof. It is opt-in because CI lanes are not guaranteed a reachable
|
|
11
|
+
// daemon: ROLLBRIDGE_COMPOSE_DOCKER_TEST=1 plus DOCKER_HOST (or the local socket) enables it.
|
|
12
|
+
// A published port is bound by the machine running the daemon, so when DOCKER_HOST points
|
|
13
|
+
// elsewhere the readiness URL has to name that machine: ROLLBRIDGE_COMPOSE_DOCKER_READY_HOST=dind.
|
|
14
|
+
const enabled = process.env.ROLLBRIDGE_COMPOSE_DOCKER_TEST === "1"
|
|
15
|
+
const readyHost = process.env.ROLLBRIDGE_COMPOSE_DOCKER_READY_HOST ?? "127.0.0.1"
|
|
16
|
+
const portBase = 20000 + (process.pid % 4000)
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* @typedef {import("../src/config.js").RollbridgeConfig & {compose: import("../src/config.js").ComposeUnitConfig[]}} ComposeTestConfig
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
describe("compose-docker", () => {
|
|
23
|
+
|
|
24
|
+
const PROJECT = `rollbridge-compose-proof-${process.pid}`
|
|
25
|
+
const V1 = "rollbridge-proof-gateway:v1"
|
|
26
|
+
const V2 = "rollbridge-proof-gateway:v2"
|
|
27
|
+
const V3 = "rollbridge-proof-gateway:v3-unhealthy"
|
|
28
|
+
|
|
29
|
+
// The gateway records what it does into /work/timeline, a shared volume, so the ordering survives
|
|
30
|
+
// container replacement. Drain and settle markers are container-local, so every container has to
|
|
31
|
+
// drain and settle for itself before a roll is allowed to recreate it.
|
|
32
|
+
const DOCKERFILE = [
|
|
33
|
+
"FROM alpine:3.20",
|
|
34
|
+
"ARG VERSION=v1",
|
|
35
|
+
"ARG FAIL_ACTIVATION=0",
|
|
36
|
+
"ENV VERSION=$VERSION FAIL_ACTIVATION=$FAIL_ACTIVATION SETTLE_SECONDS=2",
|
|
37
|
+
"RUN apk add --no-cache busybox-extras && mkdir -p /srv /work && printf '%s\\n' \\",
|
|
38
|
+
" '#!/bin/sh' \\",
|
|
39
|
+
" 'set -eu' \\",
|
|
40
|
+
" 'log() { printf \"%s %s\\n\" \"$VERSION\" \"$1\" >> /work/timeline; }' \\",
|
|
41
|
+
" 'if [ \"$FAIL_ACTIVATION\" = 1 ]; then log start; exec sleep 3600; fi' \\",
|
|
42
|
+
" 'echo ready > /srv/ready' \\",
|
|
43
|
+
" '/usr/sbin/httpd -p 7421 -h /srv &' \\",
|
|
44
|
+
" 'log start' \\",
|
|
45
|
+
" 'while :; do' \\",
|
|
46
|
+
" ' if [ -f /tmp/draining ]; then log drain; break; fi' \\",
|
|
47
|
+
" ' log admit' \\",
|
|
48
|
+
" ' sleep 0.3' \\",
|
|
49
|
+
" 'done' \\",
|
|
50
|
+
" 'sleep \"$SETTLE_SECONDS\"' \\",
|
|
51
|
+
" 'touch /tmp/settled' \\",
|
|
52
|
+
" 'log settled' \\",
|
|
53
|
+
" 'while :; do sleep 1; done' \\",
|
|
54
|
+
" > /usr/local/bin/app && chmod +x /usr/local/bin/app",
|
|
55
|
+
"ENTRYPOINT [\"/bin/sh\", \"/usr/local/bin/app\"]",
|
|
56
|
+
"HEALTHCHECK --interval=1s --timeout=2s --retries=40 CMD wget -q -O /dev/null http://127.0.0.1:7421/ready || exit 1",
|
|
57
|
+
""
|
|
58
|
+
].join("\n")
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* @param {string} projectDirectory - Directory to write the Compose project into.
|
|
62
|
+
* @param {number} hostPort - Host port the gateway publishes.
|
|
63
|
+
* @returns {Promise<string[]>} Compose global arguments for the project.
|
|
64
|
+
*/
|
|
65
|
+
async function writeComposeProject(projectDirectory, hostPort) {
|
|
66
|
+
await fs.writeFile(path.join(projectDirectory, "gateway.compose.yml"), [
|
|
67
|
+
`name: ${PROJECT}`,
|
|
68
|
+
"services:",
|
|
69
|
+
" gateway:",
|
|
70
|
+
` image: ${V1}`,
|
|
71
|
+
" init: true",
|
|
72
|
+
" ports:",
|
|
73
|
+
` - "${hostPort}:7421"`,
|
|
74
|
+
" volumes:",
|
|
75
|
+
" - gateway-work:/work",
|
|
76
|
+
"volumes:",
|
|
77
|
+
" gateway-work:",
|
|
78
|
+
""
|
|
79
|
+
].join("\n"))
|
|
80
|
+
|
|
81
|
+
return ["docker", "compose", "--project-name", PROJECT, "-f", path.join(projectDirectory, "gateway.compose.yml")]
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* @param {string[]} composeArgs - Compose global arguments.
|
|
86
|
+
* @returns {Promise<string[]>} Timeline lines recorded by the gateway containers.
|
|
87
|
+
*/
|
|
88
|
+
async function readTimeline(composeArgs) {
|
|
89
|
+
const result = await runComposeCommand({args: [...composeArgs, "exec", "-T", "gateway", "cat", "/work/timeline"], timeoutMs: 30000})
|
|
90
|
+
|
|
91
|
+
if (result.code !== 0) throw new Error(`could not read the gateway timeline: ${result.stderr.trim()}`)
|
|
92
|
+
|
|
93
|
+
return result.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* @param {string[]} lines - Timeline lines.
|
|
98
|
+
* @param {string} line - Line to locate.
|
|
99
|
+
* @returns {number} Index of the last occurrence, or -1 when the line is absent.
|
|
100
|
+
*/
|
|
101
|
+
function lastLine(lines, line) {
|
|
102
|
+
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
103
|
+
if (lines[index] === line) return index
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return -1
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* @param {string[]} composeArgs - Compose global arguments.
|
|
111
|
+
* @returns {Promise<string>} Image of the container Compose currently lists for the gateway.
|
|
112
|
+
*/
|
|
113
|
+
async function liveImage(composeArgs) {
|
|
114
|
+
const listed = await runComposeCommand({args: [...composeArgs, "ps", "-q", "gateway"], timeoutMs: 15000})
|
|
115
|
+
const inspected = await runComposeCommand({args: ["docker", "inspect", "--format", "{{.Config.Image}}", listed.stdout.trim()], timeoutMs: 15000})
|
|
116
|
+
|
|
117
|
+
return inspected.stdout.trim()
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* @param {string[]} composeArgs - Compose global arguments.
|
|
122
|
+
* @returns {Promise<void>} Resolves once the gateway container reports healthy.
|
|
123
|
+
*/
|
|
124
|
+
async function waitHealthy(composeArgs) {
|
|
125
|
+
const deadline = Date.now() + 60000
|
|
126
|
+
|
|
127
|
+
while (Date.now() < deadline) {
|
|
128
|
+
const listed = await runComposeCommand({args: [...composeArgs, "ps", "-q", "gateway"], timeoutMs: 15000})
|
|
129
|
+
const containerId = listed.stdout.trim().split("\n").filter(Boolean).at(-1)
|
|
130
|
+
|
|
131
|
+
if (containerId !== undefined) {
|
|
132
|
+
const inspected = await runComposeCommand({args: ["docker", "inspect", "--format", "{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}", containerId], timeoutMs: 15000})
|
|
133
|
+
const [state, health] = inspected.stdout.trim().split("|")
|
|
134
|
+
|
|
135
|
+
if (state === "running" && (health === "" || health === "healthy")) return
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
await new Promise((resolve) => setTimeout(resolve, 500))
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
throw new Error("the proof gateway never became healthy")
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* @param {string} projectDirectory - Build context.
|
|
146
|
+
* @param {string} tag - Image tag to build.
|
|
147
|
+
* @param {string} version - Version the container logs into the timeline.
|
|
148
|
+
* @param {boolean} [unhealthy] - Build a container that never passes its healthcheck.
|
|
149
|
+
* @returns {Promise<void>} Resolves once the image is built.
|
|
150
|
+
*/
|
|
151
|
+
async function buildImage(projectDirectory, tag, version, unhealthy = false) {
|
|
152
|
+
const result = await runComposeCommand({
|
|
153
|
+
args: ["docker", "build", "-t", tag, "--build-arg", `VERSION=${version}`, "--build-arg", `FAIL_ACTIVATION=${unhealthy ? "1" : "0"}`, "-f", path.join(projectDirectory, "Dockerfile"), projectDirectory],
|
|
154
|
+
timeoutMs: 300000
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
if (result.code !== 0) throw new Error(`docker build failed for ${tag}: ${result.stderr.trim()}`)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
test("rolls a real Compose service drain-first, retains the previous tag, and rolls back when activation fails", {timeoutMs: 300000}, async () => {
|
|
161
|
+
if (!enabled) return
|
|
162
|
+
|
|
163
|
+
const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-docker-"))
|
|
164
|
+
const hostPort = portBase
|
|
165
|
+
const composeArgs = await writeComposeProject(projectDirectory, hostPort)
|
|
166
|
+
|
|
167
|
+
await fs.writeFile(path.join(projectDirectory, "Dockerfile"), DOCKERFILE)
|
|
168
|
+
await buildImage(projectDirectory, V1, "v1")
|
|
169
|
+
await buildImage(projectDirectory, V2, "v2")
|
|
170
|
+
await buildImage(projectDirectory, V3, "v3", true)
|
|
171
|
+
|
|
172
|
+
const config = /** @type {ComposeTestConfig} */ (normalizeConfig({
|
|
173
|
+
application: "compose-docker-proof",
|
|
174
|
+
compose: [{
|
|
175
|
+
activation: {intervalMs: 500, readyUrl: `http://${readyHost}:${hostPort}/ready`, timeoutMs: 60000},
|
|
176
|
+
composeFile: "gateway.compose.yml",
|
|
177
|
+
drain: {command: `${composeArgs.join(" ")} exec -T gateway touch /tmp/draining`, timeoutMs: 30000},
|
|
178
|
+
id: "gateway",
|
|
179
|
+
image: V2,
|
|
180
|
+
inFlight: {command: `${composeArgs.join(" ")} exec -T gateway test -f /tmp/settled`, intervalMs: 500, timeoutMs: 30000},
|
|
181
|
+
projectDirectory,
|
|
182
|
+
recreateTimeoutMs: 120000,
|
|
183
|
+
service: "gateway",
|
|
184
|
+
stopGracePeriodMs: 10000
|
|
185
|
+
}],
|
|
186
|
+
composeStatePath: path.join(projectDirectory, "compose.state.json"),
|
|
187
|
+
control: {path: path.join(projectDirectory, "rollbridge.sock")},
|
|
188
|
+
processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
189
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
190
|
+
}))
|
|
191
|
+
const unit = config.compose[0]
|
|
192
|
+
|
|
193
|
+
try {
|
|
194
|
+
const started = await runComposeCommand({args: [...composeArgs, "up", "-d", "--force-recreate", "gateway"], timeoutMs: 120000})
|
|
195
|
+
|
|
196
|
+
if (started.code !== 0) throw new Error(`compose up failed: ${started.stderr.trim()}`)
|
|
197
|
+
|
|
198
|
+
await waitHealthy(composeArgs)
|
|
199
|
+
|
|
200
|
+
const rolled = await rollComposeUnit({config, runCommand: runComposeCommand, unit})
|
|
201
|
+
const afterRoll = await readTimeline(composeArgs)
|
|
202
|
+
|
|
203
|
+
expect(rolled.status).toBe("rolled")
|
|
204
|
+
expect(rolled.stage).toBe("activation")
|
|
205
|
+
expect(rolled.image).toBe(V2)
|
|
206
|
+
expect(rolled.previousImage).toBe(V1)
|
|
207
|
+
expect(await liveImage(composeArgs)).toBe(V2)
|
|
208
|
+
|
|
209
|
+
// Quiesce, then the in-flight work settling, then the new image starting — in that order.
|
|
210
|
+
const v1Drain = afterRoll.indexOf("v1 drain")
|
|
211
|
+
const v1Settled = afterRoll.indexOf("v1 settled")
|
|
212
|
+
const v2Start = afterRoll.indexOf("v2 start")
|
|
213
|
+
|
|
214
|
+
expect(v1Drain >= 0).toBe(true)
|
|
215
|
+
expect(v1Settled > v1Drain).toBe(true)
|
|
216
|
+
expect(v2Start > v1Settled).toBe(true)
|
|
217
|
+
expect(afterRoll.slice(v1Drain + 1, v2Start).filter((line) => line === "v1 admit")).toEqual([])
|
|
218
|
+
expect((await readComposeLedger({config, unit})).generations.map((generation) => [generation.image, generation.status])).toEqual([[V2, "activated"]])
|
|
219
|
+
|
|
220
|
+
// Roll to an image whose healthcheck never passes: the previous tag must be serving again.
|
|
221
|
+
const rolledBack = await rollComposeUnit({config, image: V3, runCommand: runComposeCommand, unit})
|
|
222
|
+
const afterRollback = await readTimeline(composeArgs)
|
|
223
|
+
|
|
224
|
+
expect(rolledBack.status).toBe("rolled_back")
|
|
225
|
+
expect(rolledBack.stage).toBe("rollback")
|
|
226
|
+
expect(rolledBack.rollback).toMatchObject({image: V2, status: "rolled_back"})
|
|
227
|
+
expect(rolledBack.previousImage).toBe(V3)
|
|
228
|
+
|
|
229
|
+
const v2Drain = afterRollback.indexOf("v2 drain")
|
|
230
|
+
const v2Settled = afterRollback.indexOf("v2 settled")
|
|
231
|
+
const v3Start = afterRollback.indexOf("v3 start")
|
|
232
|
+
const v2Serving = lastLine(afterRollback, "v2 start")
|
|
233
|
+
|
|
234
|
+
expect(v2Drain > v2Start).toBe(true)
|
|
235
|
+
expect(v2Settled > v2Drain).toBe(true)
|
|
236
|
+
expect(v3Start > v2Settled).toBe(true)
|
|
237
|
+
expect(v2Serving > v3Start).toBe(true)
|
|
238
|
+
expect(afterRollback.slice(v2Drain + 1, v3Start).filter((line) => line === "v2 admit")).toEqual([])
|
|
239
|
+
|
|
240
|
+
expect(await liveImage(composeArgs)).toBe(V2)
|
|
241
|
+
await waitHealthy(composeArgs)
|
|
242
|
+
|
|
243
|
+
const ready = await runComposeCommand({args: [...composeArgs, "exec", "-T", "gateway", "wget", "-q", "-O", "-", "http://127.0.0.1:7421/ready"], timeoutMs: 15000})
|
|
244
|
+
|
|
245
|
+
expect(ready.stdout.trim()).toBe("ready")
|
|
246
|
+
expect((await readComposeLedger({config, unit})).generations.map((generation) => [generation.image, generation.status])).toEqual([[V2, "activated"], [V2, "rolled_back"]])
|
|
247
|
+
} finally {
|
|
248
|
+
await runComposeCommand({args: [...composeArgs, "down", "-v"], timeoutMs: 120000})
|
|
249
|
+
await runComposeCommand({args: ["docker", "rmi", "-f", V1, V2, V3], timeoutMs: 60000})
|
|
250
|
+
await fs.rm(projectDirectory, {force: true, recursive: true})
|
|
251
|
+
}
|
|
252
|
+
})
|
|
253
|
+
|
|
254
|
+
test("refuses to roll a real Compose service that publishes the Rollbridge proxy port", {timeoutMs: 180000}, async () => {
|
|
255
|
+
if (!enabled) return
|
|
256
|
+
|
|
257
|
+
const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-proxy-guard-"))
|
|
258
|
+
const proxyPort = portBase + 1
|
|
259
|
+
const composeArgs = await writeComposeProject(projectDirectory, proxyPort)
|
|
260
|
+
|
|
261
|
+
await fs.writeFile(path.join(projectDirectory, "Dockerfile"), DOCKERFILE)
|
|
262
|
+
await buildImage(projectDirectory, V1, "v1")
|
|
263
|
+
|
|
264
|
+
const config = /** @type {ComposeTestConfig} */ (normalizeConfig({
|
|
265
|
+
application: "compose-docker-proof",
|
|
266
|
+
compose: [{
|
|
267
|
+
activation: {intervalMs: 500, timeoutMs: 30000},
|
|
268
|
+
composeFile: "gateway.compose.yml",
|
|
269
|
+
id: "gateway",
|
|
270
|
+
image: V1,
|
|
271
|
+
projectDirectory,
|
|
272
|
+
service: "gateway",
|
|
273
|
+
stopGracePeriodMs: 10000
|
|
274
|
+
}],
|
|
275
|
+
composeStatePath: path.join(projectDirectory, "compose.state.json"),
|
|
276
|
+
control: {path: path.join(projectDirectory, "rollbridge.sock")},
|
|
277
|
+
processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
278
|
+
proxy: {host: "127.0.0.1", port: proxyPort}
|
|
279
|
+
}))
|
|
280
|
+
|
|
281
|
+
try {
|
|
282
|
+
const started = await runComposeCommand({args: [...composeArgs, "up", "-d", "--force-recreate", "gateway"], timeoutMs: 120000})
|
|
283
|
+
|
|
284
|
+
if (started.code !== 0) throw new Error(`compose up failed: ${started.stderr.trim()}`)
|
|
285
|
+
|
|
286
|
+
await waitHealthy(composeArgs)
|
|
287
|
+
|
|
288
|
+
await expect(rollComposeUnit({config, runCommand: runComposeCommand, unit: config.compose[0]})).rejects.toThrow(new RegExp(`publishes port ${proxyPort}, the Rollbridge proxy port`))
|
|
289
|
+
|
|
290
|
+
expect(await liveImage(composeArgs)).toBe(V1)
|
|
291
|
+
} finally {
|
|
292
|
+
await runComposeCommand({args: [...composeArgs, "down", "-v"], timeoutMs: 120000})
|
|
293
|
+
await runComposeCommand({args: ["docker", "rmi", "-f", V1], timeoutMs: 60000})
|
|
294
|
+
await fs.rm(projectDirectory, {force: true, recursive: true})
|
|
295
|
+
}
|
|
296
|
+
})
|
|
297
|
+
})
|