rollbridge 0.1.73 → 0.1.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-bounded-control-client-close.md +4 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +69 -3
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/control-client.js +32 -5
- package/src/daemon.js +448 -76
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +92 -9
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +452 -39
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/control-protocol.test.js +22 -0
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +149 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +151 -20
- package/test/owner-replacement.test.js +176 -15
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +447 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/tensorbuzz.yml
CHANGED
|
@@ -1,4 +1,23 @@
|
|
|
1
|
+
# TensorBuzz build config for Rollbridge.
|
|
2
|
+
#
|
|
3
|
+
# `paths` (STOE-1179): this gate installs the dependency tree and runs typecheck, ESLint and the
|
|
4
|
+
# whole Velocious suite, and every one of those reads code, `examples/`, and the package/tooling
|
|
5
|
+
# manifests — no gate input is Markdown. A delivery that changes nothing but Markdown therefore
|
|
6
|
+
# has nothing for this lane to check, and TensorBuzz records it as a terminal `skipped` build
|
|
7
|
+
# instead of spending a container and ~3 minutes on it. `test/tensorbuzz-config.test.js` asserts
|
|
8
|
+
# the split against every tracked path so a later edit cannot carve a gate input out of the lane.
|
|
9
|
+
#
|
|
10
|
+
# Ordered rules, last match wins, so the negation has to come after `"**"`. Quote every negation:
|
|
11
|
+
# an unquoted `!x` is a YAML tag, and the config validator fails the whole file rather than
|
|
12
|
+
# filtering anything. The lane still runs whenever `tensorbuzz.yml` itself changes, whenever the
|
|
13
|
+
# changed-file list is missing, empty, or capped by the host, and on every manual rebuild.
|
|
14
|
+
paths:
|
|
15
|
+
- "**"
|
|
16
|
+
- "!**/*.md"
|
|
17
|
+
|
|
1
18
|
before_script:
|
|
2
19
|
- npm install
|
|
3
20
|
script:
|
|
4
|
-
|
|
21
|
+
# Rollbridge's Unix control/guardian sockets must stay below the platform path limit;
|
|
22
|
+
# the TensorBuzz runner's workspace TMPDIR is deliberately long.
|
|
23
|
+
- TMPDIR=/tmp npm run all-checks
|
package/test/completion.test.js
CHANGED
|
@@ -42,10 +42,12 @@ test("completion bash prints a sourceable script with commands and option flags"
|
|
|
42
42
|
|
|
43
43
|
expect(code).not.toBe(1)
|
|
44
44
|
expect(output).toMatch(/complete -F _rollbridge rollbridge/)
|
|
45
|
-
expect(output).toMatch(/compgen -W "daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion"/)
|
|
45
|
+
expect(output).toMatch(/compgen -W "daemon deploy rollback recover-generation-transition compose-roll compose-rollback compose-status ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion"/)
|
|
46
46
|
// A command's own options are completed after the command.
|
|
47
47
|
expect(output).toMatch(/deploy\)\n\s+opts="[^"]*--release-path[^"]*"/)
|
|
48
48
|
expect(output).toMatch(/recover-generation-transition\)\n\s+opts="--config --release-path --release-id --revision --previous-release-id --accept-retired-incumbent"/)
|
|
49
|
+
expect(output).toMatch(/compose-roll\)\n\s+opts="--config --unit --image"/)
|
|
50
|
+
expect(output).toMatch(/compose-status\)\n\s+opts="--config --unit"/)
|
|
49
51
|
expect(output).toMatch(/ensure-daemon\)\n\s+opts="[^"]*--daemon-runtime-path[^"]*"/)
|
|
50
52
|
expect(output).toMatch(/restart\)\n\s+opts="[^"]*--policy[^"]*"/)
|
|
51
53
|
})
|
|
@@ -55,8 +57,9 @@ test("completion zsh prints a #compdef script with per-command options", async (
|
|
|
55
57
|
|
|
56
58
|
expect(output).toMatch(/^#compdef rollbridge/)
|
|
57
59
|
expect(output).toMatch(/compdef _rollbridge rollbridge/)
|
|
58
|
-
expect(output).toMatch(/commands=\(daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion\)/)
|
|
60
|
+
expect(output).toMatch(/commands=\(daemon deploy rollback recover-generation-transition compose-roll compose-rollback compose-status ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion\)/)
|
|
59
61
|
expect(output).toMatch(/recover-generation-transition\) compadd -- --config --release-path --release-id --revision --previous-release-id --accept-retired-incumbent/)
|
|
62
|
+
expect(output).toMatch(/compose-rollback\) compadd -- --config --unit --image/)
|
|
60
63
|
expect(output).toMatch(/events\) compadd -- [^\n]*--limit/)
|
|
61
64
|
})
|
|
62
65
|
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import path from "node:path"
|
|
4
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
5
|
+
import {validateConfig} from "../src/config.js"
|
|
6
|
+
|
|
7
|
+
describe("compose-config", () => {
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} [unitOverrides] - Compose unit fields.
|
|
11
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} [configOverrides] - Top-level overrides.
|
|
12
|
+
* @returns {Record<string, import("../src/json.js").JsonValue>} Raw config with one Compose unit.
|
|
13
|
+
*/
|
|
14
|
+
function buildRawConfig(unitOverrides = {}, configOverrides = {}) {
|
|
15
|
+
return {
|
|
16
|
+
application: "compose-config-test",
|
|
17
|
+
compose: [{
|
|
18
|
+
id: "gateway",
|
|
19
|
+
image: "registry.local/gateway:1.4.0",
|
|
20
|
+
projectDirectory: "/srv/gateway",
|
|
21
|
+
service: "gateway",
|
|
22
|
+
...unitOverrides
|
|
23
|
+
}],
|
|
24
|
+
control: {path: "/tmp/rollbridge-compose-config.sock"},
|
|
25
|
+
processes: [{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}],
|
|
26
|
+
proxy: {host: "127.0.0.1", port: 8182},
|
|
27
|
+
...configOverrides
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {{fix: string, message: string}[]} issues - Collected issues.
|
|
33
|
+
* @returns {string[]} Issue messages.
|
|
34
|
+
*/
|
|
35
|
+
function messages(issues) {
|
|
36
|
+
return issues.map((issue) => issue.message)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* @param {import("../src/config.js").RollbridgeConfig} config - Validated config.
|
|
41
|
+
* @returns {import("../src/config.js").ComposeUnitConfig} The one Compose unit the test declared.
|
|
42
|
+
*/
|
|
43
|
+
function onlyUnit(config) {
|
|
44
|
+
const unit = config.compose?.[0]
|
|
45
|
+
|
|
46
|
+
if (unit === undefined) throw new Error("the test config declares no compose unit")
|
|
47
|
+
|
|
48
|
+
return unit
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
test("a Compose unit is normalized with drain-first defaults", () => {
|
|
52
|
+
const {config, issues} = validateConfig(buildRawConfig(), "/srv/app/rollbridge.js")
|
|
53
|
+
|
|
54
|
+
expect(issues).toEqual([])
|
|
55
|
+
|
|
56
|
+
const unit = onlyUnit(config)
|
|
57
|
+
|
|
58
|
+
expect(unit.id).toBe("gateway")
|
|
59
|
+
expect(unit.composeFile).toBe("compose.yml")
|
|
60
|
+
expect(unit.stopGracePeriodMs).toBe(10000)
|
|
61
|
+
expect(unit.recreateTimeoutMs).toBe(120000)
|
|
62
|
+
// Unset commands are absent from the normalized shape rather than present as undefined, so a unit
|
|
63
|
+
// definition that crossed the guardian's JSON transfer is still equal to a freshly loaded one.
|
|
64
|
+
expect(unit.drain).toEqual({timeoutMs: 60000})
|
|
65
|
+
expect(unit.inFlight).toEqual({intervalMs: 1000, timeoutMs: 0})
|
|
66
|
+
expect(unit.activation).toMatchObject({healthcheck: true, intervalMs: 1000, startDelayMs: 0, timeoutMs: 60000})
|
|
67
|
+
expect(unit.rollback).toEqual({enabled: true})
|
|
68
|
+
expect(config.composeStatePath).toBe(path.resolve("/srv/app", "rollbridge-compose-config-test.compose.state.json"))
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
test("a Compose unit keeps its own compose file, env file, project name, and hooks", () => {
|
|
72
|
+
const {config, issues} = validateConfig(buildRawConfig({
|
|
73
|
+
activation: {intervalMs: 500, readyCommand: "probe {{service}}", readyUrl: "http://127.0.0.1:7421/ready", startDelayMs: 2000, timeoutMs: 90000},
|
|
74
|
+
composeFile: "gateway.compose.yml",
|
|
75
|
+
drain: {command: "docker compose {{composeArgs}} exec {{service}} appctl drain", timeoutMs: 30000},
|
|
76
|
+
envFile: "gateway.env",
|
|
77
|
+
inFlight: {command: "docker compose {{composeArgs}} exec {{service}} appctl in-flight", intervalMs: 2000, timeoutMs: 120000},
|
|
78
|
+
projectName: "socketduct-gateway",
|
|
79
|
+
rollback: {enabled: false},
|
|
80
|
+
stopGracePeriodMs: 45000
|
|
81
|
+
}))
|
|
82
|
+
|
|
83
|
+
expect(issues).toEqual([])
|
|
84
|
+
|
|
85
|
+
const unit = onlyUnit(config)
|
|
86
|
+
|
|
87
|
+
expect(unit.composeFile).toBe("gateway.compose.yml")
|
|
88
|
+
expect(unit.envFile).toBe("gateway.env")
|
|
89
|
+
expect(unit.projectName).toBe("socketduct-gateway")
|
|
90
|
+
expect(unit.stopGracePeriodMs).toBe(45000)
|
|
91
|
+
expect(unit.rollback.enabled).toBe(false)
|
|
92
|
+
expect(unit.drain.timeoutMs).toBe(30000)
|
|
93
|
+
expect(unit.inFlight.timeoutMs).toBe(120000)
|
|
94
|
+
expect(unit.activation.readyUrl).toBe("http://127.0.0.1:7421/ready")
|
|
95
|
+
expect(unit.activation.readyCommand).toBe("probe {{service}}")
|
|
96
|
+
})
|
|
97
|
+
|
|
98
|
+
test("an unpinned or malformed image reference is rejected", () => {
|
|
99
|
+
const whitespace = validateConfig(buildRawConfig({image: "registry.local/gateway: 1.4.0"}))
|
|
100
|
+
const digest = validateConfig(buildRawConfig({image: "registry.local/gateway@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}))
|
|
101
|
+
const badDigest = validateConfig(buildRawConfig({image: "registry.local/gateway@sha256:nope"}))
|
|
102
|
+
|
|
103
|
+
expect(whitespace.issues.map((issue) => issue.message)).toEqual([
|
|
104
|
+
'compose[0].image "registry.local/gateway: 1.4.0" is not a valid image reference'
|
|
105
|
+
])
|
|
106
|
+
expect(digest.issues).toEqual([])
|
|
107
|
+
expect(messages(badDigest.issues)).toContain('compose[0].image "registry.local/gateway@sha256:nope" is not a valid image reference')
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
test("compose unit ids must be unique and must not collide with process ids", () => {
|
|
111
|
+
const duplicate = validateConfig({
|
|
112
|
+
...buildRawConfig(),
|
|
113
|
+
compose: [
|
|
114
|
+
{id: "gateway", image: "registry.local/gateway:1.4.0", projectDirectory: "/srv/gateway", service: "gateway"},
|
|
115
|
+
{id: "gateway", image: "registry.local/hub:1.4.0", projectDirectory: "/srv/hub", service: "hub"}
|
|
116
|
+
]
|
|
117
|
+
})
|
|
118
|
+
const collision = validateConfig(buildRawConfig({id: "web"}))
|
|
119
|
+
|
|
120
|
+
expect(messages(duplicate.issues)).toContain("Duplicate compose unit id: gateway")
|
|
121
|
+
expect(messages(collision.issues)).toContain('Compose unit id "web" collides with a process id')
|
|
122
|
+
})
|
|
123
|
+
|
|
124
|
+
test("a drain or in-flight command without a positive timeout is rejected", () => {
|
|
125
|
+
const drain = validateConfig(buildRawConfig({drain: {command: "appctl drain", timeoutMs: 0}}))
|
|
126
|
+
const inFlight = validateConfig(buildRawConfig({inFlight: {command: "appctl in-flight"}}))
|
|
127
|
+
|
|
128
|
+
expect(messages(drain.issues)).toContain("compose[0].drain.command requires a positive compose[0].drain.timeoutMs")
|
|
129
|
+
expect(messages(inFlight.issues)).toContain("compose[0].inFlight.command requires a positive compose[0].inFlight.timeoutMs")
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
test("a compose unit must name its service, project directory, and image", () => {
|
|
133
|
+
const {issues} = validateConfig(buildRawConfig({image: "", projectDirectory: "", service: ""}))
|
|
134
|
+
|
|
135
|
+
expect(messages(issues)).toEqual([
|
|
136
|
+
"compose[0].service is required",
|
|
137
|
+
"compose[0].projectDirectory is required",
|
|
138
|
+
"compose[0].image is required"
|
|
139
|
+
])
|
|
140
|
+
})
|
|
141
|
+
|
|
142
|
+
test("an empty compose list is rejected", () => {
|
|
143
|
+
const {issues} = validateConfig(buildRawConfig({}, {compose: []}))
|
|
144
|
+
|
|
145
|
+
expect(messages(issues)).toEqual(["compose must define at least one unit"])
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
test("compose must be a list of unit mappings", () => {
|
|
149
|
+
const notArray = validateConfig(buildRawConfig({}, {compose: {id: "gateway"}}))
|
|
150
|
+
const notObject = validateConfig(buildRawConfig({}, {compose: ["gateway"]}))
|
|
151
|
+
|
|
152
|
+
expect(messages(notArray.issues)).toEqual(["compose must be an array"])
|
|
153
|
+
expect(messages(notObject.issues)).toEqual([
|
|
154
|
+
"compose[0] must be an object",
|
|
155
|
+
"compose[0].service is required",
|
|
156
|
+
"compose[0].projectDirectory is required",
|
|
157
|
+
"compose[0].image is required"
|
|
158
|
+
])
|
|
159
|
+
})
|
|
160
|
+
})
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises"
|
|
4
|
+
import os from "node:os"
|
|
5
|
+
import path from "node:path"
|
|
6
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
7
|
+
import {normalizeConfig} from "../src/config.js"
|
|
8
|
+
import {readComposeLedger, rollComposeUnit, runComposeCommand} from "../src/compose-unit.js"
|
|
9
|
+
|
|
10
|
+
// Real Docker Compose proof. It is opt-in because CI lanes are not guaranteed a reachable
|
|
11
|
+
// daemon: ROLLBRIDGE_COMPOSE_DOCKER_TEST=1 plus DOCKER_HOST (or the local socket) enables it.
|
|
12
|
+
// A published port is bound by the machine running the daemon, so when DOCKER_HOST points
|
|
13
|
+
// elsewhere the readiness URL has to name that machine: ROLLBRIDGE_COMPOSE_DOCKER_READY_HOST=dind.
|
|
14
|
+
const enabled = process.env.ROLLBRIDGE_COMPOSE_DOCKER_TEST === "1"
|
|
15
|
+
const readyHost = process.env.ROLLBRIDGE_COMPOSE_DOCKER_READY_HOST ?? "127.0.0.1"
|
|
16
|
+
const portBase = 20000 + (process.pid % 4000)
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* @typedef {import("../src/config.js").RollbridgeConfig & {compose: import("../src/config.js").ComposeUnitConfig[]}} ComposeTestConfig
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
describe("compose-docker", () => {
|
|
23
|
+
|
|
24
|
+
const PROJECT = `rollbridge-compose-proof-${process.pid}`
|
|
25
|
+
const V1 = "rollbridge-proof-gateway:v1"
|
|
26
|
+
const V2 = "rollbridge-proof-gateway:v2"
|
|
27
|
+
const V3 = "rollbridge-proof-gateway:v3-unhealthy"
|
|
28
|
+
|
|
29
|
+
// The gateway records what it does into /work/timeline, a shared volume, so the ordering survives
|
|
30
|
+
// container replacement. Drain and settle markers are container-local, so every container has to
|
|
31
|
+
// drain and settle for itself before a roll is allowed to recreate it.
|
|
32
|
+
const DOCKERFILE = [
|
|
33
|
+
"FROM alpine:3.20",
|
|
34
|
+
"ARG VERSION=v1",
|
|
35
|
+
"ARG FAIL_ACTIVATION=0",
|
|
36
|
+
"ENV VERSION=$VERSION FAIL_ACTIVATION=$FAIL_ACTIVATION SETTLE_SECONDS=2",
|
|
37
|
+
"RUN apk add --no-cache busybox-extras && mkdir -p /srv /work && printf '%s\\n' \\",
|
|
38
|
+
" '#!/bin/sh' \\",
|
|
39
|
+
" 'set -eu' \\",
|
|
40
|
+
" 'log() { printf \"%s %s\\n\" \"$VERSION\" \"$1\" >> /work/timeline; }' \\",
|
|
41
|
+
" 'if [ \"$FAIL_ACTIVATION\" = 1 ]; then log start; exec sleep 3600; fi' \\",
|
|
42
|
+
" 'echo ready > /srv/ready' \\",
|
|
43
|
+
" '/usr/sbin/httpd -p 7421 -h /srv &' \\",
|
|
44
|
+
" 'log start' \\",
|
|
45
|
+
" 'while :; do' \\",
|
|
46
|
+
" ' if [ -f /tmp/draining ]; then log drain; break; fi' \\",
|
|
47
|
+
" ' log admit' \\",
|
|
48
|
+
" ' sleep 0.3' \\",
|
|
49
|
+
" 'done' \\",
|
|
50
|
+
" 'sleep \"$SETTLE_SECONDS\"' \\",
|
|
51
|
+
" 'touch /tmp/settled' \\",
|
|
52
|
+
" 'log settled' \\",
|
|
53
|
+
" 'while :; do sleep 1; done' \\",
|
|
54
|
+
" > /usr/local/bin/app && chmod +x /usr/local/bin/app",
|
|
55
|
+
"ENTRYPOINT [\"/bin/sh\", \"/usr/local/bin/app\"]",
|
|
56
|
+
"HEALTHCHECK --interval=1s --timeout=2s --retries=40 CMD wget -q -O /dev/null http://127.0.0.1:7421/ready || exit 1",
|
|
57
|
+
""
|
|
58
|
+
].join("\n")
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* @param {string} projectDirectory - Directory to write the Compose project into.
|
|
62
|
+
* @param {number} hostPort - Host port the gateway publishes.
|
|
63
|
+
* @returns {Promise<string[]>} Compose global arguments for the project.
|
|
64
|
+
*/
|
|
65
|
+
async function writeComposeProject(projectDirectory, hostPort) {
|
|
66
|
+
await fs.writeFile(path.join(projectDirectory, "gateway.compose.yml"), [
|
|
67
|
+
`name: ${PROJECT}`,
|
|
68
|
+
"services:",
|
|
69
|
+
" gateway:",
|
|
70
|
+
` image: ${V1}`,
|
|
71
|
+
" init: true",
|
|
72
|
+
" ports:",
|
|
73
|
+
` - "${hostPort}:7421"`,
|
|
74
|
+
" volumes:",
|
|
75
|
+
" - gateway-work:/work",
|
|
76
|
+
"volumes:",
|
|
77
|
+
" gateway-work:",
|
|
78
|
+
""
|
|
79
|
+
].join("\n"))
|
|
80
|
+
|
|
81
|
+
return ["docker", "compose", "--project-name", PROJECT, "-f", path.join(projectDirectory, "gateway.compose.yml")]
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* @param {string[]} composeArgs - Compose global arguments.
|
|
86
|
+
* @returns {Promise<string[]>} Timeline lines recorded by the gateway containers.
|
|
87
|
+
*/
|
|
88
|
+
async function readTimeline(composeArgs) {
|
|
89
|
+
const result = await runComposeCommand({args: [...composeArgs, "exec", "-T", "gateway", "cat", "/work/timeline"], timeoutMs: 30000})
|
|
90
|
+
|
|
91
|
+
if (result.code !== 0) throw new Error(`could not read the gateway timeline: ${result.stderr.trim()}`)
|
|
92
|
+
|
|
93
|
+
return result.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* @param {string[]} lines - Timeline lines.
|
|
98
|
+
* @param {string} line - Line to locate.
|
|
99
|
+
* @returns {number} Index of the last occurrence, or -1 when the line is absent.
|
|
100
|
+
*/
|
|
101
|
+
function lastLine(lines, line) {
|
|
102
|
+
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
103
|
+
if (lines[index] === line) return index
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return -1
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* @param {string[]} composeArgs - Compose global arguments.
|
|
111
|
+
* @returns {Promise<string>} Image of the container Compose currently lists for the gateway.
|
|
112
|
+
*/
|
|
113
|
+
async function liveImage(composeArgs) {
|
|
114
|
+
const listed = await runComposeCommand({args: [...composeArgs, "ps", "-q", "gateway"], timeoutMs: 15000})
|
|
115
|
+
const inspected = await runComposeCommand({args: ["docker", "inspect", "--format", "{{.Config.Image}}", listed.stdout.trim()], timeoutMs: 15000})
|
|
116
|
+
|
|
117
|
+
return inspected.stdout.trim()
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* @param {string[]} composeArgs - Compose global arguments.
|
|
122
|
+
* @returns {Promise<void>} Resolves once the gateway container reports healthy.
|
|
123
|
+
*/
|
|
124
|
+
async function waitHealthy(composeArgs) {
|
|
125
|
+
const deadline = Date.now() + 60000
|
|
126
|
+
|
|
127
|
+
while (Date.now() < deadline) {
|
|
128
|
+
const listed = await runComposeCommand({args: [...composeArgs, "ps", "-q", "gateway"], timeoutMs: 15000})
|
|
129
|
+
const containerId = listed.stdout.trim().split("\n").filter(Boolean).at(-1)
|
|
130
|
+
|
|
131
|
+
if (containerId !== undefined) {
|
|
132
|
+
const inspected = await runComposeCommand({args: ["docker", "inspect", "--format", "{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}", containerId], timeoutMs: 15000})
|
|
133
|
+
const [state, health] = inspected.stdout.trim().split("|")
|
|
134
|
+
|
|
135
|
+
if (state === "running" && (health === "" || health === "healthy")) return
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
await new Promise((resolve) => setTimeout(resolve, 500))
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
throw new Error("the proof gateway never became healthy")
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* @param {string} projectDirectory - Build context.
|
|
146
|
+
* @param {string} tag - Image tag to build.
|
|
147
|
+
* @param {string} version - Version the container logs into the timeline.
|
|
148
|
+
* @param {boolean} [unhealthy] - Build a container that never passes its healthcheck.
|
|
149
|
+
* @returns {Promise<void>} Resolves once the image is built.
|
|
150
|
+
*/
|
|
151
|
+
async function buildImage(projectDirectory, tag, version, unhealthy = false) {
|
|
152
|
+
const result = await runComposeCommand({
|
|
153
|
+
args: ["docker", "build", "-t", tag, "--build-arg", `VERSION=${version}`, "--build-arg", `FAIL_ACTIVATION=${unhealthy ? "1" : "0"}`, "-f", path.join(projectDirectory, "Dockerfile"), projectDirectory],
|
|
154
|
+
timeoutMs: 300000
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
if (result.code !== 0) throw new Error(`docker build failed for ${tag}: ${result.stderr.trim()}`)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
test("rolls a real Compose service drain-first, retains the previous tag, and rolls back when activation fails", {timeoutMs: 300000}, async () => {
|
|
161
|
+
if (!enabled) return
|
|
162
|
+
|
|
163
|
+
const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-docker-"))
|
|
164
|
+
const hostPort = portBase
|
|
165
|
+
const composeArgs = await writeComposeProject(projectDirectory, hostPort)
|
|
166
|
+
|
|
167
|
+
await fs.writeFile(path.join(projectDirectory, "Dockerfile"), DOCKERFILE)
|
|
168
|
+
await buildImage(projectDirectory, V1, "v1")
|
|
169
|
+
await buildImage(projectDirectory, V2, "v2")
|
|
170
|
+
await buildImage(projectDirectory, V3, "v3", true)
|
|
171
|
+
|
|
172
|
+
const config = /** @type {ComposeTestConfig} */ (normalizeConfig({
|
|
173
|
+
application: "compose-docker-proof",
|
|
174
|
+
compose: [{
|
|
175
|
+
activation: {intervalMs: 500, readyUrl: `http://${readyHost}:${hostPort}/ready`, timeoutMs: 60000},
|
|
176
|
+
composeFile: "gateway.compose.yml",
|
|
177
|
+
drain: {command: `${composeArgs.join(" ")} exec -T gateway touch /tmp/draining`, timeoutMs: 30000},
|
|
178
|
+
id: "gateway",
|
|
179
|
+
image: V2,
|
|
180
|
+
inFlight: {command: `${composeArgs.join(" ")} exec -T gateway test -f /tmp/settled`, intervalMs: 500, timeoutMs: 30000},
|
|
181
|
+
projectDirectory,
|
|
182
|
+
recreateTimeoutMs: 120000,
|
|
183
|
+
service: "gateway",
|
|
184
|
+
stopGracePeriodMs: 10000
|
|
185
|
+
}],
|
|
186
|
+
composeStatePath: path.join(projectDirectory, "compose.state.json"),
|
|
187
|
+
control: {path: path.join(projectDirectory, "rollbridge.sock")},
|
|
188
|
+
processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
189
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
190
|
+
}))
|
|
191
|
+
const unit = config.compose[0]
|
|
192
|
+
|
|
193
|
+
try {
|
|
194
|
+
const started = await runComposeCommand({args: [...composeArgs, "up", "-d", "--force-recreate", "gateway"], timeoutMs: 120000})
|
|
195
|
+
|
|
196
|
+
if (started.code !== 0) throw new Error(`compose up failed: ${started.stderr.trim()}`)
|
|
197
|
+
|
|
198
|
+
await waitHealthy(composeArgs)
|
|
199
|
+
|
|
200
|
+
const rolled = await rollComposeUnit({config, runCommand: runComposeCommand, unit})
|
|
201
|
+
const afterRoll = await readTimeline(composeArgs)
|
|
202
|
+
|
|
203
|
+
expect(rolled.status).toBe("rolled")
|
|
204
|
+
expect(rolled.stage).toBe("activation")
|
|
205
|
+
expect(rolled.image).toBe(V2)
|
|
206
|
+
expect(rolled.previousImage).toBe(V1)
|
|
207
|
+
expect(await liveImage(composeArgs)).toBe(V2)
|
|
208
|
+
|
|
209
|
+
// Quiesce, then the in-flight work settling, then the new image starting — in that order.
|
|
210
|
+
const v1Drain = afterRoll.indexOf("v1 drain")
|
|
211
|
+
const v1Settled = afterRoll.indexOf("v1 settled")
|
|
212
|
+
const v2Start = afterRoll.indexOf("v2 start")
|
|
213
|
+
|
|
214
|
+
expect(v1Drain >= 0).toBe(true)
|
|
215
|
+
expect(v1Settled > v1Drain).toBe(true)
|
|
216
|
+
expect(v2Start > v1Settled).toBe(true)
|
|
217
|
+
expect(afterRoll.slice(v1Drain + 1, v2Start).filter((line) => line === "v1 admit")).toEqual([])
|
|
218
|
+
expect((await readComposeLedger({config, unit})).generations.map((generation) => [generation.image, generation.status])).toEqual([[V2, "activated"]])
|
|
219
|
+
|
|
220
|
+
// Roll to an image whose healthcheck never passes: the previous tag must be serving again.
|
|
221
|
+
const rolledBack = await rollComposeUnit({config, image: V3, runCommand: runComposeCommand, unit})
|
|
222
|
+
const afterRollback = await readTimeline(composeArgs)
|
|
223
|
+
|
|
224
|
+
expect(rolledBack.status).toBe("rolled_back")
|
|
225
|
+
expect(rolledBack.stage).toBe("rollback")
|
|
226
|
+
expect(rolledBack.rollback).toMatchObject({image: V2, status: "rolled_back"})
|
|
227
|
+
expect(rolledBack.previousImage).toBe(V3)
|
|
228
|
+
|
|
229
|
+
const v2Drain = afterRollback.indexOf("v2 drain")
|
|
230
|
+
const v2Settled = afterRollback.indexOf("v2 settled")
|
|
231
|
+
const v3Start = afterRollback.indexOf("v3 start")
|
|
232
|
+
const v2Serving = lastLine(afterRollback, "v2 start")
|
|
233
|
+
|
|
234
|
+
expect(v2Drain > v2Start).toBe(true)
|
|
235
|
+
expect(v2Settled > v2Drain).toBe(true)
|
|
236
|
+
expect(v3Start > v2Settled).toBe(true)
|
|
237
|
+
expect(v2Serving > v3Start).toBe(true)
|
|
238
|
+
expect(afterRollback.slice(v2Drain + 1, v3Start).filter((line) => line === "v2 admit")).toEqual([])
|
|
239
|
+
|
|
240
|
+
expect(await liveImage(composeArgs)).toBe(V2)
|
|
241
|
+
await waitHealthy(composeArgs)
|
|
242
|
+
|
|
243
|
+
const ready = await runComposeCommand({args: [...composeArgs, "exec", "-T", "gateway", "wget", "-q", "-O", "-", "http://127.0.0.1:7421/ready"], timeoutMs: 15000})
|
|
244
|
+
|
|
245
|
+
expect(ready.stdout.trim()).toBe("ready")
|
|
246
|
+
expect((await readComposeLedger({config, unit})).generations.map((generation) => [generation.image, generation.status])).toEqual([[V2, "activated"], [V2, "rolled_back"]])
|
|
247
|
+
} finally {
|
|
248
|
+
await runComposeCommand({args: [...composeArgs, "down", "-v"], timeoutMs: 120000})
|
|
249
|
+
await runComposeCommand({args: ["docker", "rmi", "-f", V1, V2, V3], timeoutMs: 60000})
|
|
250
|
+
await fs.rm(projectDirectory, {force: true, recursive: true})
|
|
251
|
+
}
|
|
252
|
+
})
|
|
253
|
+
|
|
254
|
+
test("refuses to roll a real Compose service that publishes the Rollbridge proxy port", {timeoutMs: 180000}, async () => {
|
|
255
|
+
if (!enabled) return
|
|
256
|
+
|
|
257
|
+
const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-proxy-guard-"))
|
|
258
|
+
const proxyPort = portBase + 1
|
|
259
|
+
const composeArgs = await writeComposeProject(projectDirectory, proxyPort)
|
|
260
|
+
|
|
261
|
+
await fs.writeFile(path.join(projectDirectory, "Dockerfile"), DOCKERFILE)
|
|
262
|
+
await buildImage(projectDirectory, V1, "v1")
|
|
263
|
+
|
|
264
|
+
const config = /** @type {ComposeTestConfig} */ (normalizeConfig({
|
|
265
|
+
application: "compose-docker-proof",
|
|
266
|
+
compose: [{
|
|
267
|
+
activation: {intervalMs: 500, timeoutMs: 30000},
|
|
268
|
+
composeFile: "gateway.compose.yml",
|
|
269
|
+
id: "gateway",
|
|
270
|
+
image: V1,
|
|
271
|
+
projectDirectory,
|
|
272
|
+
service: "gateway",
|
|
273
|
+
stopGracePeriodMs: 10000
|
|
274
|
+
}],
|
|
275
|
+
composeStatePath: path.join(projectDirectory, "compose.state.json"),
|
|
276
|
+
control: {path: path.join(projectDirectory, "rollbridge.sock")},
|
|
277
|
+
processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
|
|
278
|
+
proxy: {host: "127.0.0.1", port: proxyPort}
|
|
279
|
+
}))
|
|
280
|
+
|
|
281
|
+
try {
|
|
282
|
+
const started = await runComposeCommand({args: [...composeArgs, "up", "-d", "--force-recreate", "gateway"], timeoutMs: 120000})
|
|
283
|
+
|
|
284
|
+
if (started.code !== 0) throw new Error(`compose up failed: ${started.stderr.trim()}`)
|
|
285
|
+
|
|
286
|
+
await waitHealthy(composeArgs)
|
|
287
|
+
|
|
288
|
+
await expect(rollComposeUnit({config, runCommand: runComposeCommand, unit: config.compose[0]})).rejects.toThrow(new RegExp(`publishes port ${proxyPort}, the Rollbridge proxy port`))
|
|
289
|
+
|
|
290
|
+
expect(await liveImage(composeArgs)).toBe(V1)
|
|
291
|
+
} finally {
|
|
292
|
+
await runComposeCommand({args: [...composeArgs, "down", "-v"], timeoutMs: 120000})
|
|
293
|
+
await runComposeCommand({args: ["docker", "rmi", "-f", V1], timeoutMs: 60000})
|
|
294
|
+
await fs.rm(projectDirectory, {force: true, recursive: true})
|
|
295
|
+
}
|
|
296
|
+
})
|
|
297
|
+
})
|