rollbridge 0.1.56 → 0.1.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/recover.js CHANGED
@@ -1,5 +1,7 @@
1
1
  // @ts-check
2
2
 
3
+ import fs from "node:fs/promises"
4
+ import net from "node:net"
3
5
  import {inspectControlSocket} from "./daemon.js"
4
6
  import {clearState, isProcessAlive, liveProcesses, readState} from "./state-store.js"
5
7
 
@@ -7,9 +9,10 @@ import {clearState, isProcessAlive, liveProcesses, readState} from "./state-stor
7
9
  const KILL_CONFIRM_TIMEOUT_MS = 500
8
10
 
9
11
  /**
12
+ * @typedef {import("./json.js").JsonValue} JsonValue
10
13
  * @typedef {{id: string, pid: number, releaseId: string | null}} OrphanProcess
11
14
  * @typedef {{error: string}} RecoverError
12
- * @typedef {{cleared: boolean, forced: boolean, orphans: OrphanProcess[], remaining: OrphanProcess[]}} RecoverReport
15
+ * @typedef {{archived: boolean, archivePath?: string, cleared: boolean, deadGuardianState: boolean, forced: boolean, orphans: OrphanProcess[], remaining: OrphanProcess[]}} RecoverReport
13
16
  * @typedef {RecoverError | RecoverReport} RecoverResult
14
17
  */
15
18
 
@@ -29,10 +32,11 @@ const KILL_CONFIRM_TIMEOUT_MS = 500
29
32
  * @param {object} args - Options.
30
33
  * @param {import("./config.js").RollbridgeConfig} args.config - Normalized config.
31
34
  * @param {boolean} args.force - Whether to actually stop the orphans (otherwise list them).
35
+ * @param {(socketPath: string) => Promise<boolean | undefined>} [args.probeUnixListener] - Probes whether a Unix socket has a live listener (defaults to the real probe; injectable for tests).
32
36
  * @param {(pid: number, timeoutMs: number) => Promise<boolean>} [args.stopGroup] - Stops a process group and resolves to whether it is gone afterward (defaults to the real implementation; injectable for tests).
33
37
  * @returns {Promise<RecoverResult>} The orphans found and whether they were stopped, or an error.
34
38
  */
35
- export async function recoverOrphans({config, force, stopGroup = stopProcessGroup}) {
39
+ export async function recoverOrphans({config, force, probeUnixListener = probeUnixListenerAt, stopGroup = stopProcessGroup}) {
36
40
  if (config.statePath === undefined) {
37
41
  return {error: "No statePath is configured; set statePath in the config to enable recovery."}
38
42
  }
@@ -41,9 +45,11 @@ export async function recoverOrphans({config, force, stopGroup = stopProcessGrou
41
45
  return {error: `A daemon (or another process) is using ${config.control.path}; stop it before recovering — recover is for cleaning up after a crash.`}
42
46
  }
43
47
 
44
- const orphans = liveProcesses(await readState(config.statePath))
48
+ const state = await readState(config.statePath)
49
+ const deadGuardianState = await stateNamesDeadGuardian(state, probeUnixListener)
50
+ const orphans = liveProcesses(state)
45
51
 
46
- if (!force) return {cleared: false, forced: false, orphans, remaining: []}
52
+ if (!force) return {archived: false, cleared: false, deadGuardianState, forced: false, orphans, remaining: []}
47
53
 
48
54
  /** @type {OrphanProcess[]} */
49
55
  const remaining = []
@@ -54,11 +60,35 @@ export async function recoverOrphans({config, force, stopGroup = stopProcessGrou
54
60
  if (!stopped) remaining.push(orphan)
55
61
  }
56
62
 
57
- // Only clear the state file once every orphan is confirmed gone; otherwise keep it so the
58
- // operator can still find and retry the survivors on the next run.
59
- if (remaining.length === 0) await clearState(config.statePath)
63
+ // Keep the state file while any orphan survives so the operator can still find and retry it.
64
+ if (remaining.length > 0) {
65
+ return {archived: false, cleared: false, deadGuardianState, forced: true, orphans, remaining}
66
+ }
67
+
68
+ // With every orphan gone, a conclusively dead guardian identity is the documented dead-end
69
+ // case: archive the state under a timestamped evidence name (bytes preserved verbatim) and
70
+ // move the dead socket metadata aside, so the next deploy can boot a fresh generation.
71
+ // See the TensorBuzz production runbook docs/production-rollbridge-recovery.md.
72
+ if (deadGuardianState) {
73
+ const archivePath = deadGuardianArchivePath(config.statePath)
74
+ const guardianSocketPath = stateGuardianIdentity(state)?.socketPath
75
+
76
+ if (guardianSocketPath !== undefined) {
77
+ try {
78
+ await fs.rename(guardianSocketPath, `${guardianSocketPath}.dead-guardian-${archiveStamp()}`)
79
+ } catch (error) {
80
+ if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") throw error
81
+ }
82
+ }
83
+
84
+ await fs.rename(config.statePath, archivePath)
85
+
86
+ return {archived: true, archivePath, cleared: false, deadGuardianState, forced: true, orphans, remaining: []}
87
+ }
60
88
 
61
- return {cleared: remaining.length === 0, forced: true, orphans, remaining}
89
+ await clearState(config.statePath)
90
+
91
+ return {archived: false, cleared: true, deadGuardianState, forced: true, orphans, remaining: []}
62
92
  }
63
93
 
64
94
  /**
@@ -132,3 +162,93 @@ function sendSignal(pid, signal) {
132
162
  return "denied"
133
163
  }
134
164
  }
165
+
166
+ /**
167
+ * @param {JsonValue} state - Persisted state.
168
+ * @returns {{pid?: number, socketPath: string} | undefined} The persisted guardian identity when present and shaped as expected.
169
+ */
170
+ function stateGuardianIdentity(state) {
171
+ if (!state || typeof state !== "object" || Array.isArray(state)) return undefined
172
+
173
+ const recovery = /** @type {Record<string, JsonValue>} */ (state).recovery
174
+
175
+ if (!recovery || typeof recovery !== "object" || Array.isArray(recovery)) return undefined
176
+
177
+ const guardian = /** @type {Record<string, JsonValue>} */ (recovery).guardian
178
+
179
+ if (!guardian || typeof guardian !== "object" || Array.isArray(guardian)) return undefined
180
+
181
+ const socketPath = /** @type {Record<string, JsonValue>} */ (guardian).socketPath
182
+
183
+ if (typeof socketPath !== "string" || socketPath === "") return undefined
184
+
185
+ const pid = /** @type {Record<string, JsonValue>} */ (guardian).pid
186
+
187
+ return {pid: typeof pid === "number" ? pid : undefined, socketPath}
188
+ }
189
+
190
+ /**
191
+ * Whether the state conclusively names a dead guardian: its recorded pid is gone (when present)
192
+ * and nothing is listening on its Unix socket. Any ambiguity returns false so only a conclusive
193
+ * proof may archive the state.
194
+ * @param {JsonValue} state - Persisted state.
195
+ * @param {(socketPath: string) => Promise<boolean | undefined>} probeUnixListener - Listener probe.
196
+ * @returns {Promise<boolean>} True only on a conclusive dead-guardian proof.
197
+ */
198
+ async function stateNamesDeadGuardian(state, probeUnixListener) {
199
+ const guardian = stateGuardianIdentity(state)
200
+
201
+ if (!guardian) return false
202
+ if (guardian.pid !== undefined && isProcessAlive(guardian.pid)) return false
203
+
204
+ return (await probeUnixListener(guardian.socketPath)) === false
205
+ }
206
+
207
+ /**
208
+ * Probes whether anything is listening on a Unix socket path.
209
+ * @param {string} socketPath - Socket path.
210
+ * @returns {Promise<boolean | undefined>} True when a listener accepted the connection, false when conclusively nothing listens (refused or missing), undefined when the probe was inconclusive.
211
+ */
212
+ function probeUnixListenerAt(socketPath) {
213
+ return new Promise((resolve) => {
214
+ const socket = net.createConnection(socketPath)
215
+ let settled = false
216
+
217
+ /** @param {boolean | undefined} result - Probe result. */
218
+ const finish = (result) => {
219
+ if (settled) return
220
+
221
+ settled = true
222
+ clearTimeout(timer)
223
+ socket.destroy()
224
+ resolve(result)
225
+ }
226
+
227
+ const timer = setTimeout(() => finish(undefined), 1000)
228
+
229
+ socket.once("connect", () => finish(true))
230
+ socket.once("error", (error) => {
231
+ const code = error && typeof error === "object" && "code" in error ? error.code : undefined
232
+
233
+ finish(code === "ECONNREFUSED" || code === "ENOENT" ? false : undefined)
234
+ })
235
+ })
236
+ }
237
+
238
+ /**
239
+ * @returns {string} A UTC archive stamp in the established evidence-name shape (for example 20261013T1102Z).
240
+ */
241
+ function archiveStamp() {
242
+ return `${new Date().toISOString().slice(0, 16).replace(/[-:]/g, "")}Z`
243
+ }
244
+
245
+ /**
246
+ * @param {string} statePath - State file path.
247
+ * @returns {string} Timestamped archive path preserving the state bytes as evidence.
248
+ */
249
+ function deadGuardianArchivePath(statePath) {
250
+ const stamp = archiveStamp()
251
+ const suffix = statePath.endsWith(".json") ? ".json" : ""
252
+
253
+ return `${statePath.slice(0, statePath.length - suffix.length)}.dead-guardian-${stamp}${suffix}`
254
+ }
@@ -42,10 +42,12 @@ test("completion bash prints a sourceable script with commands and option flags"
42
42
 
43
43
  expect(code).not.toBe(1)
44
44
  expect(output).toMatch(/complete -F _rollbridge rollbridge/)
45
- expect(output).toMatch(/compgen -W "daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion"/)
45
+ expect(output).toMatch(/compgen -W "daemon deploy rollback recover-generation-transition compose-roll compose-rollback compose-status ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion"/)
46
46
  // A command's own options are completed after the command.
47
47
  expect(output).toMatch(/deploy\)\n\s+opts="[^"]*--release-path[^"]*"/)
48
48
  expect(output).toMatch(/recover-generation-transition\)\n\s+opts="--config --release-path --release-id --revision --previous-release-id --accept-retired-incumbent"/)
49
+ expect(output).toMatch(/compose-roll\)\n\s+opts="--config --unit --image"/)
50
+ expect(output).toMatch(/compose-status\)\n\s+opts="--config --unit"/)
49
51
  expect(output).toMatch(/ensure-daemon\)\n\s+opts="[^"]*--daemon-runtime-path[^"]*"/)
50
52
  expect(output).toMatch(/restart\)\n\s+opts="[^"]*--policy[^"]*"/)
51
53
  })
@@ -55,8 +57,9 @@ test("completion zsh prints a #compdef script with per-command options", async (
55
57
 
56
58
  expect(output).toMatch(/^#compdef rollbridge/)
57
59
  expect(output).toMatch(/compdef _rollbridge rollbridge/)
58
- expect(output).toMatch(/commands=\(daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion\)/)
60
+ expect(output).toMatch(/commands=\(daemon deploy rollback recover-generation-transition compose-roll compose-rollback compose-status ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion\)/)
59
61
  expect(output).toMatch(/recover-generation-transition\) compadd -- --config --release-path --release-id --revision --previous-release-id --accept-retired-incumbent/)
62
+ expect(output).toMatch(/compose-rollback\) compadd -- --config --unit --image/)
60
63
  expect(output).toMatch(/events\) compadd -- [^\n]*--limit/)
61
64
  })
62
65
 
@@ -0,0 +1,158 @@
1
+ // @ts-check
2
+
3
+ import path from "node:path"
4
+ import {describe, expect, test} from "@velocious/testing"
5
+ import {validateConfig} from "../src/config.js"
6
+
7
+ describe("compose-config", () => {
8
+
9
+ /**
10
+ * @param {Record<string, import("../src/json.js").JsonValue>} [unitOverrides] - Compose unit fields.
11
+ * @param {Record<string, import("../src/json.js").JsonValue>} [configOverrides] - Top-level overrides.
12
+ * @returns {Record<string, import("../src/json.js").JsonValue>} Raw config with one Compose unit.
13
+ */
14
+ function buildRawConfig(unitOverrides = {}, configOverrides = {}) {
15
+ return {
16
+ application: "compose-config-test",
17
+ compose: [{
18
+ id: "gateway",
19
+ image: "registry.local/gateway:1.4.0",
20
+ projectDirectory: "/srv/gateway",
21
+ service: "gateway",
22
+ ...unitOverrides
23
+ }],
24
+ control: {path: "/tmp/rollbridge-compose-config.sock"},
25
+ processes: [{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}],
26
+ proxy: {host: "127.0.0.1", port: 8182},
27
+ ...configOverrides
28
+ }
29
+ }
30
+
31
+ /**
32
+ * @param {{fix: string, message: string}[]} issues - Collected issues.
33
+ * @returns {string[]} Issue messages.
34
+ */
35
+ function messages(issues) {
36
+ return issues.map((issue) => issue.message)
37
+ }
38
+
39
+ /**
40
+ * @param {import("../src/config.js").RollbridgeConfig} config - Validated config.
41
+ * @returns {import("../src/config.js").ComposeUnitConfig} The one Compose unit the test declared.
42
+ */
43
+ function onlyUnit(config) {
44
+ const unit = config.compose?.[0]
45
+
46
+ if (unit === undefined) throw new Error("the test config declares no compose unit")
47
+
48
+ return unit
49
+ }
50
+
51
+ test("a Compose unit is normalized with drain-first defaults", () => {
52
+ const {config, issues} = validateConfig(buildRawConfig(), "/srv/app/rollbridge.js")
53
+
54
+ expect(issues).toEqual([])
55
+
56
+ const unit = onlyUnit(config)
57
+
58
+ expect(unit.id).toBe("gateway")
59
+ expect(unit.composeFile).toBe("compose.yml")
60
+ expect(unit.stopGracePeriodMs).toBe(10000)
61
+ expect(unit.recreateTimeoutMs).toBe(120000)
62
+ expect(unit.drain).toEqual({command: undefined, timeoutMs: 60000})
63
+ expect(unit.inFlight).toEqual({command: undefined, intervalMs: 1000, timeoutMs: 0})
64
+ expect(unit.activation).toMatchObject({healthcheck: true, intervalMs: 1000, startDelayMs: 0, timeoutMs: 60000})
65
+ expect(unit.rollback).toEqual({enabled: true})
66
+ expect(config.composeStatePath).toBe(path.resolve("/srv/app", "rollbridge-compose-config-test.compose.state.json"))
67
+ })
68
+
69
+ test("a Compose unit keeps its own compose file, env file, project name, and hooks", () => {
70
+ const {config, issues} = validateConfig(buildRawConfig({
71
+ activation: {intervalMs: 500, readyCommand: "probe {{service}}", readyUrl: "http://127.0.0.1:7421/ready", startDelayMs: 2000, timeoutMs: 90000},
72
+ composeFile: "gateway.compose.yml",
73
+ drain: {command: "docker compose {{composeArgs}} exec {{service}} appctl drain", timeoutMs: 30000},
74
+ envFile: "gateway.env",
75
+ inFlight: {command: "docker compose {{composeArgs}} exec {{service}} appctl in-flight", intervalMs: 2000, timeoutMs: 120000},
76
+ projectName: "socketduct-gateway",
77
+ rollback: {enabled: false},
78
+ stopGracePeriodMs: 45000
79
+ }))
80
+
81
+ expect(issues).toEqual([])
82
+
83
+ const unit = onlyUnit(config)
84
+
85
+ expect(unit.composeFile).toBe("gateway.compose.yml")
86
+ expect(unit.envFile).toBe("gateway.env")
87
+ expect(unit.projectName).toBe("socketduct-gateway")
88
+ expect(unit.stopGracePeriodMs).toBe(45000)
89
+ expect(unit.rollback.enabled).toBe(false)
90
+ expect(unit.drain.timeoutMs).toBe(30000)
91
+ expect(unit.inFlight.timeoutMs).toBe(120000)
92
+ expect(unit.activation.readyUrl).toBe("http://127.0.0.1:7421/ready")
93
+ expect(unit.activation.readyCommand).toBe("probe {{service}}")
94
+ })
95
+
96
+ test("an unpinned or malformed image reference is rejected", () => {
97
+ const whitespace = validateConfig(buildRawConfig({image: "registry.local/gateway: 1.4.0"}))
98
+ const digest = validateConfig(buildRawConfig({image: "registry.local/gateway@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}))
99
+ const badDigest = validateConfig(buildRawConfig({image: "registry.local/gateway@sha256:nope"}))
100
+
101
+ expect(whitespace.issues.map((issue) => issue.message)).toEqual([
102
+ 'compose[0].image "registry.local/gateway: 1.4.0" is not a valid image reference'
103
+ ])
104
+ expect(digest.issues).toEqual([])
105
+ expect(messages(badDigest.issues)).toContain('compose[0].image "registry.local/gateway@sha256:nope" is not a valid image reference')
106
+ })
107
+
108
+ test("compose unit ids must be unique and must not collide with process ids", () => {
109
+ const duplicate = validateConfig({
110
+ ...buildRawConfig(),
111
+ compose: [
112
+ {id: "gateway", image: "registry.local/gateway:1.4.0", projectDirectory: "/srv/gateway", service: "gateway"},
113
+ {id: "gateway", image: "registry.local/hub:1.4.0", projectDirectory: "/srv/hub", service: "hub"}
114
+ ]
115
+ })
116
+ const collision = validateConfig(buildRawConfig({id: "web"}))
117
+
118
+ expect(messages(duplicate.issues)).toContain("Duplicate compose unit id: gateway")
119
+ expect(messages(collision.issues)).toContain('Compose unit id "web" collides with a process id')
120
+ })
121
+
122
+ test("a drain or in-flight command without a positive timeout is rejected", () => {
123
+ const drain = validateConfig(buildRawConfig({drain: {command: "appctl drain", timeoutMs: 0}}))
124
+ const inFlight = validateConfig(buildRawConfig({inFlight: {command: "appctl in-flight"}}))
125
+
126
+ expect(messages(drain.issues)).toContain("compose[0].drain.command requires a positive compose[0].drain.timeoutMs")
127
+ expect(messages(inFlight.issues)).toContain("compose[0].inFlight.command requires a positive compose[0].inFlight.timeoutMs")
128
+ })
129
+
130
+ test("a compose unit must name its service, project directory, and image", () => {
131
+ const {issues} = validateConfig(buildRawConfig({image: "", projectDirectory: "", service: ""}))
132
+
133
+ expect(messages(issues)).toEqual([
134
+ "compose[0].service is required",
135
+ "compose[0].projectDirectory is required",
136
+ "compose[0].image is required"
137
+ ])
138
+ })
139
+
140
+ test("an empty compose list is rejected", () => {
141
+ const {issues} = validateConfig(buildRawConfig({}, {compose: []}))
142
+
143
+ expect(messages(issues)).toEqual(["compose must define at least one unit"])
144
+ })
145
+
146
+ test("compose must be a list of unit mappings", () => {
147
+ const notArray = validateConfig(buildRawConfig({}, {compose: {id: "gateway"}}))
148
+ const notObject = validateConfig(buildRawConfig({}, {compose: ["gateway"]}))
149
+
150
+ expect(messages(notArray.issues)).toEqual(["compose must be an array"])
151
+ expect(messages(notObject.issues)).toEqual([
152
+ "compose[0] must be an object",
153
+ "compose[0].service is required",
154
+ "compose[0].projectDirectory is required",
155
+ "compose[0].image is required"
156
+ ])
157
+ })
158
+ })
@@ -0,0 +1,297 @@
1
+ // @ts-check
2
+
3
+ import fs from "node:fs/promises"
4
+ import os from "node:os"
5
+ import path from "node:path"
6
+ import {describe, expect, test} from "@velocious/testing"
7
+ import {normalizeConfig} from "../src/config.js"
8
+ import {readComposeLedger, rollComposeUnit, runComposeCommand} from "../src/compose-unit.js"
9
+
10
+ // Real Docker Compose proof. It is opt-in because CI lanes are not guaranteed a reachable
11
+ // daemon: ROLLBRIDGE_COMPOSE_DOCKER_TEST=1 plus DOCKER_HOST (or the local socket) enables it.
12
+ // A published port is bound by the machine running the daemon, so when DOCKER_HOST points
13
+ // elsewhere the readiness URL has to name that machine: ROLLBRIDGE_COMPOSE_DOCKER_READY_HOST=dind.
14
+ const enabled = process.env.ROLLBRIDGE_COMPOSE_DOCKER_TEST === "1"
15
+ const readyHost = process.env.ROLLBRIDGE_COMPOSE_DOCKER_READY_HOST ?? "127.0.0.1"
16
+ const portBase = 20000 + (process.pid % 4000)
17
+
18
+ /**
19
+ * @typedef {import("../src/config.js").RollbridgeConfig & {compose: import("../src/config.js").ComposeUnitConfig[]}} ComposeTestConfig
20
+ */
21
+
22
+ describe("compose-docker", () => {
23
+
24
+ const PROJECT = `rollbridge-compose-proof-${process.pid}`
25
+ const V1 = "rollbridge-proof-gateway:v1"
26
+ const V2 = "rollbridge-proof-gateway:v2"
27
+ const V3 = "rollbridge-proof-gateway:v3-unhealthy"
28
+
29
+ // The gateway records what it does into /work/timeline, a shared volume, so the ordering survives
30
+ // container replacement. Drain and settle markers are container-local, so every container has to
31
+ // drain and settle for itself before a roll is allowed to recreate it.
32
+ const DOCKERFILE = [
33
+ "FROM alpine:3.20",
34
+ "ARG VERSION=v1",
35
+ "ARG FAIL_ACTIVATION=0",
36
+ "ENV VERSION=$VERSION FAIL_ACTIVATION=$FAIL_ACTIVATION SETTLE_SECONDS=2",
37
+ "RUN apk add --no-cache busybox-extras && mkdir -p /srv /work && printf '%s\\n' \\",
38
+ " '#!/bin/sh' \\",
39
+ " 'set -eu' \\",
40
+ " 'log() { printf \"%s %s\\n\" \"$VERSION\" \"$1\" >> /work/timeline; }' \\",
41
+ " 'if [ \"$FAIL_ACTIVATION\" = 1 ]; then log start; exec sleep 3600; fi' \\",
42
+ " 'echo ready > /srv/ready' \\",
43
+ " '/usr/sbin/httpd -p 7421 -h /srv &' \\",
44
+ " 'log start' \\",
45
+ " 'while :; do' \\",
46
+ " ' if [ -f /tmp/draining ]; then log drain; break; fi' \\",
47
+ " ' log admit' \\",
48
+ " ' sleep 0.3' \\",
49
+ " 'done' \\",
50
+ " 'sleep \"$SETTLE_SECONDS\"' \\",
51
+ " 'touch /tmp/settled' \\",
52
+ " 'log settled' \\",
53
+ " 'while :; do sleep 1; done' \\",
54
+ " > /usr/local/bin/app && chmod +x /usr/local/bin/app",
55
+ "ENTRYPOINT [\"/bin/sh\", \"/usr/local/bin/app\"]",
56
+ "HEALTHCHECK --interval=1s --timeout=2s --retries=40 CMD wget -q -O /dev/null http://127.0.0.1:7421/ready || exit 1",
57
+ ""
58
+ ].join("\n")
59
+
60
+ /**
61
+ * @param {string} projectDirectory - Directory to write the Compose project into.
62
+ * @param {number} hostPort - Host port the gateway publishes.
63
+ * @returns {Promise<string[]>} Compose global arguments for the project.
64
+ */
65
+ async function writeComposeProject(projectDirectory, hostPort) {
66
+ await fs.writeFile(path.join(projectDirectory, "gateway.compose.yml"), [
67
+ `name: ${PROJECT}`,
68
+ "services:",
69
+ " gateway:",
70
+ ` image: ${V1}`,
71
+ " init: true",
72
+ " ports:",
73
+ ` - "${hostPort}:7421"`,
74
+ " volumes:",
75
+ " - gateway-work:/work",
76
+ "volumes:",
77
+ " gateway-work:",
78
+ ""
79
+ ].join("\n"))
80
+
81
+ return ["docker", "compose", "--project-name", PROJECT, "-f", path.join(projectDirectory, "gateway.compose.yml")]
82
+ }
83
+
84
+ /**
85
+ * @param {string[]} composeArgs - Compose global arguments.
86
+ * @returns {Promise<string[]>} Timeline lines recorded by the gateway containers.
87
+ */
88
+ async function readTimeline(composeArgs) {
89
+ const result = await runComposeCommand({args: [...composeArgs, "exec", "-T", "gateway", "cat", "/work/timeline"], timeoutMs: 30000})
90
+
91
+ if (result.code !== 0) throw new Error(`could not read the gateway timeline: ${result.stderr.trim()}`)
92
+
93
+ return result.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
94
+ }
95
+
96
+ /**
97
+ * @param {string[]} lines - Timeline lines.
98
+ * @param {string} line - Line to locate.
99
+ * @returns {number} Index of the last occurrence, or -1 when the line is absent.
100
+ */
101
+ function lastLine(lines, line) {
102
+ for (let index = lines.length - 1; index >= 0; index -= 1) {
103
+ if (lines[index] === line) return index
104
+ }
105
+
106
+ return -1
107
+ }
108
+
109
+ /**
110
+ * @param {string[]} composeArgs - Compose global arguments.
111
+ * @returns {Promise<string>} Image of the container Compose currently lists for the gateway.
112
+ */
113
+ async function liveImage(composeArgs) {
114
+ const listed = await runComposeCommand({args: [...composeArgs, "ps", "-q", "gateway"], timeoutMs: 15000})
115
+ const inspected = await runComposeCommand({args: ["docker", "inspect", "--format", "{{.Config.Image}}", listed.stdout.trim()], timeoutMs: 15000})
116
+
117
+ return inspected.stdout.trim()
118
+ }
119
+
120
+ /**
121
+ * @param {string[]} composeArgs - Compose global arguments.
122
+ * @returns {Promise<void>} Resolves once the gateway container reports healthy.
123
+ */
124
+ async function waitHealthy(composeArgs) {
125
+ const deadline = Date.now() + 60000
126
+
127
+ while (Date.now() < deadline) {
128
+ const listed = await runComposeCommand({args: [...composeArgs, "ps", "-q", "gateway"], timeoutMs: 15000})
129
+ const containerId = listed.stdout.trim().split("\n").filter(Boolean).at(-1)
130
+
131
+ if (containerId !== undefined) {
132
+ const inspected = await runComposeCommand({args: ["docker", "inspect", "--format", "{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}", containerId], timeoutMs: 15000})
133
+ const [state, health] = inspected.stdout.trim().split("|")
134
+
135
+ if (state === "running" && (health === "" || health === "healthy")) return
136
+ }
137
+
138
+ await new Promise((resolve) => setTimeout(resolve, 500))
139
+ }
140
+
141
+ throw new Error("the proof gateway never became healthy")
142
+ }
143
+
144
+ /**
145
+ * @param {string} projectDirectory - Build context.
146
+ * @param {string} tag - Image tag to build.
147
+ * @param {string} version - Version the container logs into the timeline.
148
+ * @param {boolean} [unhealthy] - Build a container that never passes its healthcheck.
149
+ * @returns {Promise<void>} Resolves once the image is built.
150
+ */
151
+ async function buildImage(projectDirectory, tag, version, unhealthy = false) {
152
+ const result = await runComposeCommand({
153
+ args: ["docker", "build", "-t", tag, "--build-arg", `VERSION=${version}`, "--build-arg", `FAIL_ACTIVATION=${unhealthy ? "1" : "0"}`, "-f", path.join(projectDirectory, "Dockerfile"), projectDirectory],
154
+ timeoutMs: 300000
155
+ })
156
+
157
+ if (result.code !== 0) throw new Error(`docker build failed for ${tag}: ${result.stderr.trim()}`)
158
+ }
159
+
160
+ test("rolls a real Compose service drain-first, retains the previous tag, and rolls back when activation fails", {timeoutMs: 300000}, async () => {
161
+ if (!enabled) return
162
+
163
+ const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-docker-"))
164
+ const hostPort = portBase
165
+ const composeArgs = await writeComposeProject(projectDirectory, hostPort)
166
+
167
+ await fs.writeFile(path.join(projectDirectory, "Dockerfile"), DOCKERFILE)
168
+ await buildImage(projectDirectory, V1, "v1")
169
+ await buildImage(projectDirectory, V2, "v2")
170
+ await buildImage(projectDirectory, V3, "v3", true)
171
+
172
+ const config = /** @type {ComposeTestConfig} */ (normalizeConfig({
173
+ application: "compose-docker-proof",
174
+ compose: [{
175
+ activation: {intervalMs: 500, readyUrl: `http://${readyHost}:${hostPort}/ready`, timeoutMs: 60000},
176
+ composeFile: "gateway.compose.yml",
177
+ drain: {command: `${composeArgs.join(" ")} exec -T gateway touch /tmp/draining`, timeoutMs: 30000},
178
+ id: "gateway",
179
+ image: V2,
180
+ inFlight: {command: `${composeArgs.join(" ")} exec -T gateway test -f /tmp/settled`, intervalMs: 500, timeoutMs: 30000},
181
+ projectDirectory,
182
+ recreateTimeoutMs: 120000,
183
+ service: "gateway",
184
+ stopGracePeriodMs: 10000
185
+ }],
186
+ composeStatePath: path.join(projectDirectory, "compose.state.json"),
187
+ control: {path: path.join(projectDirectory, "rollbridge.sock")},
188
+ processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
189
+ proxy: {host: "127.0.0.1", port: 8182}
190
+ }))
191
+ const unit = config.compose[0]
192
+
193
+ try {
194
+ const started = await runComposeCommand({args: [...composeArgs, "up", "-d", "--force-recreate", "gateway"], timeoutMs: 120000})
195
+
196
+ if (started.code !== 0) throw new Error(`compose up failed: ${started.stderr.trim()}`)
197
+
198
+ await waitHealthy(composeArgs)
199
+
200
+ const rolled = await rollComposeUnit({config, runCommand: runComposeCommand, unit})
201
+ const afterRoll = await readTimeline(composeArgs)
202
+
203
+ expect(rolled.status).toBe("rolled")
204
+ expect(rolled.stage).toBe("activation")
205
+ expect(rolled.image).toBe(V2)
206
+ expect(rolled.previousImage).toBe(V1)
207
+ expect(await liveImage(composeArgs)).toBe(V2)
208
+
209
+ // Quiesce, then the in-flight work settling, then the new image starting — in that order.
210
+ const v1Drain = afterRoll.indexOf("v1 drain")
211
+ const v1Settled = afterRoll.indexOf("v1 settled")
212
+ const v2Start = afterRoll.indexOf("v2 start")
213
+
214
+ expect(v1Drain >= 0).toBe(true)
215
+ expect(v1Settled > v1Drain).toBe(true)
216
+ expect(v2Start > v1Settled).toBe(true)
217
+ expect(afterRoll.slice(v1Drain + 1, v2Start).filter((line) => line === "v1 admit")).toEqual([])
218
+ expect((await readComposeLedger({config, unit})).generations.map((generation) => [generation.image, generation.status])).toEqual([[V2, "activated"]])
219
+
220
+ // Roll to an image whose healthcheck never passes: the previous tag must be serving again.
221
+ const rolledBack = await rollComposeUnit({config, image: V3, runCommand: runComposeCommand, unit})
222
+ const afterRollback = await readTimeline(composeArgs)
223
+
224
+ expect(rolledBack.status).toBe("rolled_back")
225
+ expect(rolledBack.stage).toBe("rollback")
226
+ expect(rolledBack.rollback).toMatchObject({image: V2, status: "rolled_back"})
227
+ expect(rolledBack.previousImage).toBe(V3)
228
+
229
+ const v2Drain = afterRollback.indexOf("v2 drain")
230
+ const v2Settled = afterRollback.indexOf("v2 settled")
231
+ const v3Start = afterRollback.indexOf("v3 start")
232
+ const v2Serving = lastLine(afterRollback, "v2 start")
233
+
234
+ expect(v2Drain > v2Start).toBe(true)
235
+ expect(v2Settled > v2Drain).toBe(true)
236
+ expect(v3Start > v2Settled).toBe(true)
237
+ expect(v2Serving > v3Start).toBe(true)
238
+ expect(afterRollback.slice(v2Drain + 1, v3Start).filter((line) => line === "v2 admit")).toEqual([])
239
+
240
+ expect(await liveImage(composeArgs)).toBe(V2)
241
+ await waitHealthy(composeArgs)
242
+
243
+ const ready = await runComposeCommand({args: [...composeArgs, "exec", "-T", "gateway", "wget", "-q", "-O", "-", "http://127.0.0.1:7421/ready"], timeoutMs: 15000})
244
+
245
+ expect(ready.stdout.trim()).toBe("ready")
246
+ expect((await readComposeLedger({config, unit})).generations.map((generation) => [generation.image, generation.status])).toEqual([[V2, "activated"], [V2, "rolled_back"]])
247
+ } finally {
248
+ await runComposeCommand({args: [...composeArgs, "down", "-v"], timeoutMs: 120000})
249
+ await runComposeCommand({args: ["docker", "rmi", "-f", V1, V2, V3], timeoutMs: 60000})
250
+ await fs.rm(projectDirectory, {force: true, recursive: true})
251
+ }
252
+ })
253
+
254
+ test("refuses to roll a real Compose service that publishes the Rollbridge proxy port", {timeoutMs: 180000}, async () => {
255
+ if (!enabled) return
256
+
257
+ const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-proxy-guard-"))
258
+ const proxyPort = portBase + 1
259
+ const composeArgs = await writeComposeProject(projectDirectory, proxyPort)
260
+
261
+ await fs.writeFile(path.join(projectDirectory, "Dockerfile"), DOCKERFILE)
262
+ await buildImage(projectDirectory, V1, "v1")
263
+
264
+ const config = /** @type {ComposeTestConfig} */ (normalizeConfig({
265
+ application: "compose-docker-proof",
266
+ compose: [{
267
+ activation: {intervalMs: 500, timeoutMs: 30000},
268
+ composeFile: "gateway.compose.yml",
269
+ id: "gateway",
270
+ image: V1,
271
+ projectDirectory,
272
+ service: "gateway",
273
+ stopGracePeriodMs: 10000
274
+ }],
275
+ composeStatePath: path.join(projectDirectory, "compose.state.json"),
276
+ control: {path: path.join(projectDirectory, "rollbridge.sock")},
277
+ processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
278
+ proxy: {host: "127.0.0.1", port: proxyPort}
279
+ }))
280
+
281
+ try {
282
+ const started = await runComposeCommand({args: [...composeArgs, "up", "-d", "--force-recreate", "gateway"], timeoutMs: 120000})
283
+
284
+ if (started.code !== 0) throw new Error(`compose up failed: ${started.stderr.trim()}`)
285
+
286
+ await waitHealthy(composeArgs)
287
+
288
+ await expect(rollComposeUnit({config, runCommand: runComposeCommand, unit: config.compose[0]})).rejects.toThrow(new RegExp(`publishes port ${proxyPort}, the Rollbridge proxy port`))
289
+
290
+ expect(await liveImage(composeArgs)).toBe(V1)
291
+ } finally {
292
+ await runComposeCommand({args: [...composeArgs, "down", "-v"], timeoutMs: 120000})
293
+ await runComposeCommand({args: ["docker", "rmi", "-f", V1], timeoutMs: 60000})
294
+ await fs.rm(projectDirectory, {force: true, recursive: true})
295
+ }
296
+ })
297
+ })