rollbridge 0.1.70 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -63
- package/TODO.md +0 -13
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +65 -387
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +22 -212
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -435
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +5 -119
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +21 -125
- package/test/owner-replacement.test.js +14 -175
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +72 -737
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
package/src/compose-unit.js
DELETED
|
@@ -1,725 +0,0 @@
|
|
|
1
|
-
// @ts-check
|
|
2
|
-
|
|
3
|
-
import fs from "node:fs/promises"
|
|
4
|
-
import path from "node:path"
|
|
5
|
-
import {spawn} from "node:child_process"
|
|
6
|
-
import {setTimeout as sleep} from "node:timers/promises"
|
|
7
|
-
import {inspectControlSocket} from "./daemon.js"
|
|
8
|
-
import {sendControlCommand} from "./control-client.js"
|
|
9
|
-
import {readState, writeState} from "./state-store.js"
|
|
10
|
-
import {renderTemplate} from "./template.js"
|
|
11
|
-
|
|
12
|
-
/**
|
|
13
|
-
* @typedef {import("./config.js").ComposeUnitConfig} ComposeUnitConfig
|
|
14
|
-
* @typedef {import("./config.js").RollbridgeConfig} RollbridgeConfig
|
|
15
|
-
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
16
|
-
* @typedef {{args?: string[], command?: string, cwd?: string, env?: Record<string, string>, timeoutMs?: number}} CommandRequest
|
|
17
|
-
* @typedef {{code: number, signal: string | null, stderr: string, stdout: string}} CommandResult
|
|
18
|
-
* @typedef {(request: CommandRequest) => Promise<CommandResult>} CommandRunner
|
|
19
|
-
* @typedef {{containerId: string | null, healthStatus: string, image: string | null, ports: number[], state: string}} ContainerState
|
|
20
|
-
* @typedef {{activatedAt: string, containerId: string | null, image: string, previousImage: string | null, releaseId: string, status: "activated" | "rolled_back", unitId: string}} ComposeGeneration
|
|
21
|
-
* @typedef {{generations: ComposeGeneration[], unitId: string}} ComposeLedger
|
|
22
|
-
* @typedef {{containerId: string | null, error: string | undefined, generations: ComposeGeneration[], image: string, previousImage: string | null, releaseId: string, rollback: ComposeRollbackReport | undefined, stage: string, status: "rolled" | "rolled_back" | "failed", unitId: string}} ComposeRollResult
|
|
23
|
-
* @typedef {{error: string | undefined, image: string, status: "rolled_back" | "failed"}} ComposeRollbackReport
|
|
24
|
-
*/
|
|
25
|
-
|
|
26
|
-
const GENERATION_HISTORY_LIMIT = 20
|
|
27
|
-
const DOCKER_INSPECT_TIMEOUT_MS = 30000
|
|
28
|
-
const INSPECT_FORMAT = "{{.Config.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}|{{json .NetworkSettings.Ports}}|{{json .HostConfig.PortBindings}}"
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Runs one command as an argv list or through a shell, bounded by a timeout so no hook can
|
|
32
|
-
* wedge a roll.
|
|
33
|
-
* @param {CommandRequest} request - Command to run.
|
|
34
|
-
* @returns {Promise<CommandResult>} Exit status and captured output.
|
|
35
|
-
*/
|
|
36
|
-
export async function runComposeCommand({args, command, cwd, env, timeoutMs}) {
|
|
37
|
-
return await new Promise((resolve) => {
|
|
38
|
-
/** @type {import("node:child_process").SpawnOptions} */
|
|
39
|
-
const options = {cwd, env: {...process.env, ...env}, stdio: "pipe"}
|
|
40
|
-
/** @type {import("node:child_process").ChildProcess} */
|
|
41
|
-
const child = command !== undefined
|
|
42
|
-
? spawn(command, {...options, shell: true})
|
|
43
|
-
: spawn(/** @type {string} */ ((args ?? [""])[0]), (args ?? []).slice(1), options)
|
|
44
|
-
let stdout = ""
|
|
45
|
-
let stderr = ""
|
|
46
|
-
let settled = false
|
|
47
|
-
|
|
48
|
-
const timer = timeoutMs !== undefined && timeoutMs > 0
|
|
49
|
-
? setTimeout(() => {
|
|
50
|
-
if (settled) return
|
|
51
|
-
settled = true
|
|
52
|
-
child.kill("SIGKILL")
|
|
53
|
-
resolve({code: 124, signal: "SIGKILL", stderr: `${command ?? (args ?? []).join(" ")} timed out after ${timeoutMs}ms and was killed`, stdout})
|
|
54
|
-
}, timeoutMs)
|
|
55
|
-
: undefined
|
|
56
|
-
|
|
57
|
-
if (timer) timer.unref()
|
|
58
|
-
|
|
59
|
-
child.stdout?.setEncoding("utf8")
|
|
60
|
-
child.stderr?.setEncoding("utf8")
|
|
61
|
-
child.stdout?.on("data", (chunk) => {
|
|
62
|
-
stdout += chunk
|
|
63
|
-
})
|
|
64
|
-
child.stderr?.on("data", (chunk) => {
|
|
65
|
-
stderr += chunk
|
|
66
|
-
})
|
|
67
|
-
child.on("error", (error) => {
|
|
68
|
-
if (settled) return
|
|
69
|
-
settled = true
|
|
70
|
-
if (timer) clearTimeout(timer)
|
|
71
|
-
resolve({code: 127, signal: null, stderr: error.message, stdout})
|
|
72
|
-
})
|
|
73
|
-
child.on("close", (code, signal) => {
|
|
74
|
-
if (settled) return
|
|
75
|
-
settled = true
|
|
76
|
-
if (timer) clearTimeout(timer)
|
|
77
|
-
resolve({code: code ?? 1, signal, stderr, stdout})
|
|
78
|
-
})
|
|
79
|
-
})
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
/**
|
|
83
|
-
* Builds the shared `docker compose` global arguments for one unit.
|
|
84
|
-
* @param {ComposeUnitConfig} unit - Compose unit config.
|
|
85
|
-
* @returns {string[]} Argv prefix up to (but excluding) the compose subcommand.
|
|
86
|
-
*/
|
|
87
|
-
export function composeGlobalArgs(unit) {
|
|
88
|
-
return [
|
|
89
|
-
"docker",
|
|
90
|
-
"compose",
|
|
91
|
-
...(unit.projectName ? ["--project-name", unit.projectName] : []),
|
|
92
|
-
"-f", path.resolve(unit.projectDirectory, unit.composeFile),
|
|
93
|
-
...(unit.envFile ? ["--env-file", path.resolve(unit.projectDirectory, unit.envFile)] : [])
|
|
94
|
-
]
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
/**
|
|
98
|
-
* Reads the retained generation ledger for one Compose unit.
|
|
99
|
-
* @param {object} args - Options.
|
|
100
|
-
* @param {RollbridgeConfig} args.config - Rollbridge config.
|
|
101
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
102
|
-
* @returns {Promise<ComposeLedger>} Ledger snapshot (empty when none is stored yet).
|
|
103
|
-
*/
|
|
104
|
-
export async function readComposeLedger({config, unit}) {
|
|
105
|
-
const snapshot = await readState(composeLedgerPath(config))
|
|
106
|
-
|
|
107
|
-
if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)) return {generations: [], unitId: unit.id}
|
|
108
|
-
|
|
109
|
-
const rawGenerations = snapshot.generations
|
|
110
|
-
const generations = Array.isArray(rawGenerations)
|
|
111
|
-
? rawGenerations.filter((entry) => isGeneration(entry, unit.id))
|
|
112
|
-
: []
|
|
113
|
-
|
|
114
|
-
return {generations, unitId: unit.id}
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
* The image a rollback would select for a unit: the image that was serving before the newest
|
|
119
|
-
* activated one.
|
|
120
|
-
* @param {ComposeLedger} ledger - Ledger snapshot.
|
|
121
|
-
* @returns {string | undefined} Earlier image reference, or undefined when none is retained.
|
|
122
|
-
*/
|
|
123
|
-
export function rollbackImageForUnit(ledger) {
|
|
124
|
-
const newest = ledger.generations.at(-1)
|
|
125
|
-
|
|
126
|
-
if (ledger.generations.length >= 2) return ledger.generations[ledger.generations.length - 2].image
|
|
127
|
-
if (newest?.previousImage !== undefined && newest.previousImage !== null) return newest.previousImage
|
|
128
|
-
|
|
129
|
-
return undefined
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
/**
|
|
133
|
-
* Rolls one Compose-managed service drain-first: quiesce the unit, wait for its in-flight work
|
|
134
|
-
* to settle, recreate the container with the target image, gate on activation, and roll back to
|
|
135
|
-
* the previous image when activation fails. A drain or in-flight failure fails the roll before
|
|
136
|
-
* anything is recreated, so the current container keeps serving.
|
|
137
|
-
* @param {object} args - Options.
|
|
138
|
-
* @param {RollbridgeConfig} args.config - Rollbridge config.
|
|
139
|
-
* @param {string} [args.image] - Target image reference; defaults to the unit's configured image.
|
|
140
|
-
* @param {(socketPath: string) => Promise<import("./daemon.js").ControlSocketInspection>} [args.inspectSocket] - Control socket probe.
|
|
141
|
-
* @param {() => Date} [args.now] - Clock used for generation ids.
|
|
142
|
-
* @param {CommandRunner} [args.runCommand] - Command runner.
|
|
143
|
-
* @param {(args: {command: Record<string, JsonValue>, path: string}) => Promise<Record<string, JsonValue>>} [args.sendCommand] - Control command sender.
|
|
144
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
145
|
-
* @returns {Promise<ComposeRollResult>} Roll outcome.
|
|
146
|
-
*/
|
|
147
|
-
export async function rollComposeUnit({
|
|
148
|
-
config,
|
|
149
|
-
image,
|
|
150
|
-
inspectSocket = inspectControlSocket,
|
|
151
|
-
now = () => new Date(),
|
|
152
|
-
runCommand = runComposeCommand,
|
|
153
|
-
sendCommand = sendControlCommand,
|
|
154
|
-
unit
|
|
155
|
-
}) {
|
|
156
|
-
const releaseId = now().toISOString().replace(/[^0-9]/g, "")
|
|
157
|
-
const targetImage = image ?? unit.image
|
|
158
|
-
const ledger = await readComposeLedger({config, unit})
|
|
159
|
-
const composeArgs = composeGlobalArgs(unit)
|
|
160
|
-
const context = templateContext({application: config.application, composeArgs, releaseId, unit})
|
|
161
|
-
|
|
162
|
-
assertImageReference(targetImage, unit)
|
|
163
|
-
await assertNoUnresolvedReleaseTransition({config, inspectSocket, sendCommand, unit})
|
|
164
|
-
|
|
165
|
-
const before = await inspectUnitContainer({composeArgs, runCommand, unit})
|
|
166
|
-
const frontingError = proxyFrontingError({container: before, proxyPort: config.proxy.port, unit})
|
|
167
|
-
|
|
168
|
-
if (frontingError) throw new Error(frontingError)
|
|
169
|
-
|
|
170
|
-
const previousImage = before.image ?? ledger.generations.at(-1)?.image ?? null
|
|
171
|
-
|
|
172
|
-
const drainFailure = await runDrain({context, runCommand, unit})
|
|
173
|
-
|
|
174
|
-
if (drainFailure) {
|
|
175
|
-
return failedRoll({containerId: before.containerId, error: drainFailure, ledger, previousImage, releaseId, stage: "drain", targetImage, unit})
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
const inFlightFailure = await waitForInFlightWork({context, runCommand, unit})
|
|
179
|
-
|
|
180
|
-
if (inFlightFailure) {
|
|
181
|
-
return failedRoll({containerId: before.containerId, error: inFlightFailure, ledger, previousImage, releaseId, stage: "in_flight", targetImage, unit})
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
const recreateFailure = await recreateUnitContainer({composeArgs, previousImage, runCommand, targetImage, unit})
|
|
185
|
-
|
|
186
|
-
if (recreateFailure) {
|
|
187
|
-
return failedRoll({containerId: before.containerId, error: recreateFailure, ledger, previousImage, releaseId, stage: "recreate", targetImage, unit})
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
const activationFailure = await waitForActivation({composeArgs, context, proxyPort: config.proxy.port, runCommand, unit})
|
|
191
|
-
|
|
192
|
-
if (!activationFailure) {
|
|
193
|
-
const after = await inspectUnitContainer({composeArgs, runCommand, unit})
|
|
194
|
-
|
|
195
|
-
const generations = await appendGeneration({config, generation: {
|
|
196
|
-
activatedAt: now().toISOString(),
|
|
197
|
-
containerId: after.containerId,
|
|
198
|
-
image: targetImage,
|
|
199
|
-
previousImage,
|
|
200
|
-
releaseId,
|
|
201
|
-
status: "activated"
|
|
202
|
-
}, ledger, unit})
|
|
203
|
-
|
|
204
|
-
return {
|
|
205
|
-
containerId: after.containerId,
|
|
206
|
-
error: undefined,
|
|
207
|
-
generations,
|
|
208
|
-
image: targetImage,
|
|
209
|
-
previousImage,
|
|
210
|
-
releaseId,
|
|
211
|
-
rollback: undefined,
|
|
212
|
-
stage: "activation",
|
|
213
|
-
status: "rolled",
|
|
214
|
-
unitId: unit.id
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
if (!unit.rollback.enabled || previousImage === null || previousImage === targetImage) {
|
|
219
|
-
return failedRoll({containerId: null, error: activationFailure, ledger, previousImage, releaseId, stage: "activation", targetImage, unit})
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
const rollbackRecreateFailure = await recreateUnitContainer({composeArgs, previousImage, runCommand, targetImage: previousImage, unit})
|
|
223
|
-
|
|
224
|
-
if (rollbackRecreateFailure) {
|
|
225
|
-
return failedRoll({
|
|
226
|
-
containerId: null,
|
|
227
|
-
error: `${activationFailure}; rollback to ${previousImage} failed: ${rollbackRecreateFailure}`,
|
|
228
|
-
ledger,
|
|
229
|
-
previousImage,
|
|
230
|
-
releaseId,
|
|
231
|
-
stage: "rollback_failed",
|
|
232
|
-
targetImage,
|
|
233
|
-
unit
|
|
234
|
-
})
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
const rollbackActivationFailure = await waitForActivation({composeArgs, context, proxyPort: config.proxy.port, runCommand, unit})
|
|
238
|
-
|
|
239
|
-
if (rollbackActivationFailure) {
|
|
240
|
-
return failedRoll({
|
|
241
|
-
containerId: null,
|
|
242
|
-
error: `${activationFailure}; rollback to ${previousImage} did not activate: ${rollbackActivationFailure}`,
|
|
243
|
-
ledger,
|
|
244
|
-
previousImage,
|
|
245
|
-
releaseId,
|
|
246
|
-
stage: "rollback_failed",
|
|
247
|
-
targetImage,
|
|
248
|
-
unit
|
|
249
|
-
})
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
const rollbackContainer = await inspectUnitContainer({composeArgs, runCommand, unit})
|
|
253
|
-
const generations = await appendGeneration({config, generation: {
|
|
254
|
-
activatedAt: now().toISOString(),
|
|
255
|
-
containerId: rollbackContainer.containerId,
|
|
256
|
-
image: previousImage,
|
|
257
|
-
previousImage: targetImage,
|
|
258
|
-
releaseId,
|
|
259
|
-
status: "rolled_back"
|
|
260
|
-
}, ledger, unit})
|
|
261
|
-
|
|
262
|
-
return {
|
|
263
|
-
containerId: rollbackContainer.containerId,
|
|
264
|
-
error: activationFailure,
|
|
265
|
-
generations,
|
|
266
|
-
image: rollbackContainer.image ?? previousImage,
|
|
267
|
-
previousImage: targetImage,
|
|
268
|
-
releaseId,
|
|
269
|
-
rollback: {error: undefined, image: previousImage, status: "rolled_back"},
|
|
270
|
-
stage: "rollback",
|
|
271
|
-
status: "rolled_back",
|
|
272
|
-
unitId: unit.id
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
/**
|
|
277
|
-
* Rolls a Compose unit back to an earlier image through the same drain-first path used by a
|
|
278
|
-
* forward roll.
|
|
279
|
-
* @param {object} args - Options.
|
|
280
|
-
* @param {RollbridgeConfig} args.config - Rollbridge config.
|
|
281
|
-
* @param {string} [args.image] - Explicit earlier image; defaults to the retained previous one.
|
|
282
|
-
* @param {(socketPath: string) => Promise<import("./daemon.js").ControlSocketInspection>} [args.inspectSocket] - Control socket probe.
|
|
283
|
-
* @param {() => Date} [args.now] - Clock used for generation ids.
|
|
284
|
-
* @param {CommandRunner} [args.runCommand] - Command runner.
|
|
285
|
-
* @param {(args: {command: Record<string, JsonValue>, path: string}) => Promise<Record<string, JsonValue>>} [args.sendCommand] - Control command sender.
|
|
286
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
287
|
-
* @returns {Promise<ComposeRollResult>} Roll outcome.
|
|
288
|
-
*/
|
|
289
|
-
export async function rollbackComposeUnit({config, image, inspectSocket, now, runCommand, sendCommand, unit}) {
|
|
290
|
-
const ledger = await readComposeLedger({config, unit})
|
|
291
|
-
const targetImage = image ?? rollbackImageForUnit(ledger)
|
|
292
|
-
|
|
293
|
-
if (targetImage === undefined) {
|
|
294
|
-
throw new Error(`No earlier image is retained for Compose unit "${unit.id}"; pass --image <reference> to roll it back explicitly.`)
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
return await rollComposeUnit({config, image: targetImage, inspectSocket, now, runCommand, sendCommand, unit})
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
/**
|
|
301
|
-
* Reports the retained generations and the live container state for one Compose unit.
|
|
302
|
-
* @param {object} args - Options.
|
|
303
|
-
* @param {RollbridgeConfig} args.config - Rollbridge config.
|
|
304
|
-
* @param {CommandRunner} [args.runCommand] - Command runner.
|
|
305
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
306
|
-
* @returns {Promise<{container: ContainerState, generations: ComposeGeneration[], ledgerPath: string, unitId: string}>} Unit status.
|
|
307
|
-
*/
|
|
308
|
-
export async function composeUnitStatus({config, runCommand = runComposeCommand, unit}) {
|
|
309
|
-
const composeArgs = composeGlobalArgs(unit)
|
|
310
|
-
const ledger = await readComposeLedger({config, unit})
|
|
311
|
-
|
|
312
|
-
return {
|
|
313
|
-
container: await inspectUnitContainer({composeArgs, runCommand, unit}),
|
|
314
|
-
generations: ledger.generations,
|
|
315
|
-
ledgerPath: composeLedgerPath(config),
|
|
316
|
-
unitId: unit.id
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
/**
|
|
321
|
-
* @param {RollbridgeConfig} config - Rollbridge config.
|
|
322
|
-
* @returns {string} Ledger file path for Compose unit generations.
|
|
323
|
-
*/
|
|
324
|
-
export function composeLedgerPath(config) {
|
|
325
|
-
return config.composeStatePath ?? path.resolve(process.cwd(), `rollbridge-${config.application}.compose.state.json`)
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
/**
|
|
329
|
-
* Runs the unit's quiescence hook, which is what stops the unit admitting new work before it is
|
|
330
|
-
* recreated.
|
|
331
|
-
* @param {object} args - Options.
|
|
332
|
-
* @param {import("./template.js").TemplateContext} args.context - Template context.
|
|
333
|
-
* @param {CommandRunner} args.runCommand - Command runner.
|
|
334
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
335
|
-
* @returns {Promise<string | undefined>} Failure message, or undefined when quiesced.
|
|
336
|
-
*/
|
|
337
|
-
async function runDrain({context, runCommand, unit}) {
|
|
338
|
-
if (unit.drain.command === undefined) return undefined
|
|
339
|
-
|
|
340
|
-
const result = await runCommand({
|
|
341
|
-
command: renderTemplate(unit.drain.command, context),
|
|
342
|
-
cwd: unit.projectDirectory,
|
|
343
|
-
timeoutMs: unit.drain.timeoutMs
|
|
344
|
-
})
|
|
345
|
-
|
|
346
|
-
if (result.code === 0) return undefined
|
|
347
|
-
|
|
348
|
-
return `Compose unit "${unit.id}" could not be quiesced (${result.stderr.trim() || `exit ${result.code}`}); keeping the current container running and not recreating it`
|
|
349
|
-
}
|
|
350
|
-
|
|
351
|
-
/**
|
|
352
|
-
* Polls the unit's in-flight gate until it reports that accepted work has settled.
|
|
353
|
-
* @param {object} args - Options.
|
|
354
|
-
* @param {import("./template.js").TemplateContext} args.context - Template context.
|
|
355
|
-
* @param {CommandRunner} args.runCommand - Command runner.
|
|
356
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
357
|
-
* @returns {Promise<string | undefined>} Failure message, or undefined when settled.
|
|
358
|
-
*/
|
|
359
|
-
async function waitForInFlightWork({context, runCommand, unit}) {
|
|
360
|
-
if (unit.inFlight.command === undefined) return undefined
|
|
361
|
-
|
|
362
|
-
const command = renderTemplate(unit.inFlight.command, context)
|
|
363
|
-
const deadline = Date.now() + unit.inFlight.timeoutMs
|
|
364
|
-
let lastError = "no attempts"
|
|
365
|
-
|
|
366
|
-
while (Date.now() <= deadline) {
|
|
367
|
-
const result = await runCommand({command, cwd: unit.projectDirectory, timeoutMs: Math.max(1000, Math.min(unit.inFlight.intervalMs * 10, unit.inFlight.timeoutMs))})
|
|
368
|
-
|
|
369
|
-
if (result.code === 0) return undefined
|
|
370
|
-
|
|
371
|
-
lastError = result.stderr.trim() || `exit ${result.code}`
|
|
372
|
-
await sleep(Math.min(unit.inFlight.intervalMs, Math.max(0, deadline - Date.now())))
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
return `Compose unit "${unit.id}" still had in-flight work after ${unit.inFlight.timeoutMs}ms (${lastError}); keeping the current container running and not recreating it`
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
/**
|
|
379
|
-
* Recreates the unit's container with the target image through a generated Compose override, so
|
|
380
|
-
* the compose file itself is never rewritten.
|
|
381
|
-
* @param {object} args - Options.
|
|
382
|
-
* @param {string[]} args.composeArgs - Compose global arguments.
|
|
383
|
-
* @param {string | null} args.previousImage - Image currently serving, recorded for diagnostics.
|
|
384
|
-
* @param {CommandRunner} args.runCommand - Command runner.
|
|
385
|
-
* @param {string} args.targetImage - Image the unit is rolled to.
|
|
386
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
387
|
-
* @returns {Promise<string | undefined>} Failure message, or undefined when recreated.
|
|
388
|
-
*/
|
|
389
|
-
async function recreateUnitContainer({composeArgs, previousImage, runCommand, targetImage, unit}) {
|
|
390
|
-
const overridePath = overrideComposePath(unit)
|
|
391
|
-
|
|
392
|
-
await fs.writeFile(overridePath, overrideComposeContents({targetImage, unit}), {mode: 0o600})
|
|
393
|
-
|
|
394
|
-
try {
|
|
395
|
-
const result = await runCommand({
|
|
396
|
-
args: [...composeArgs, "-f", overridePath, "up", "-d", "--no-deps", "--force-recreate", unit.service],
|
|
397
|
-
cwd: unit.projectDirectory,
|
|
398
|
-
timeoutMs: unit.recreateTimeoutMs
|
|
399
|
-
})
|
|
400
|
-
|
|
401
|
-
if (result.code === 0) return undefined
|
|
402
|
-
|
|
403
|
-
return `Compose recreate failed for unit "${unit.id}" (image ${targetImage}, previous ${previousImage ?? "none"}): ${result.stderr.trim() || `exit ${result.code}`}`
|
|
404
|
-
} finally {
|
|
405
|
-
await fs.rm(overridePath, {force: true})
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
/**
|
|
410
|
-
* Activation gate: the recreated container must be running and healthy, must not front the
|
|
411
|
-
* Rollbridge proxy port, and any configured readiness probe must pass.
|
|
412
|
-
* @param {object} args - Options.
|
|
413
|
-
* @param {string[]} args.composeArgs - Compose global arguments.
|
|
414
|
-
* @param {import("./template.js").TemplateContext} args.context - Template context.
|
|
415
|
-
* @param {number} args.proxyPort - Rollbridge proxy port the unit must not front.
|
|
416
|
-
* @param {CommandRunner} args.runCommand - Command runner.
|
|
417
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
418
|
-
* @returns {Promise<string | undefined>} Failure message, or undefined when activated.
|
|
419
|
-
*/
|
|
420
|
-
async function waitForActivation({composeArgs, context, proxyPort, runCommand, unit}) {
|
|
421
|
-
if (unit.activation.startDelayMs > 0) await sleep(unit.activation.startDelayMs)
|
|
422
|
-
|
|
423
|
-
const deadline = Date.now() + unit.activation.timeoutMs
|
|
424
|
-
let lastError = "no attempts"
|
|
425
|
-
|
|
426
|
-
while (Date.now() <= deadline) {
|
|
427
|
-
const unready = await activationBlocker({composeArgs, context, proxyPort, runCommand, unit})
|
|
428
|
-
|
|
429
|
-
if (unready === undefined) return undefined
|
|
430
|
-
|
|
431
|
-
lastError = unready
|
|
432
|
-
await sleep(Math.min(unit.activation.intervalMs, Math.max(0, deadline - Date.now())))
|
|
433
|
-
}
|
|
434
|
-
|
|
435
|
-
return `Compose unit "${unit.id}" did not activate within ${unit.activation.timeoutMs}ms (${lastError})`
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
/**
|
|
439
|
-
* @param {object} args - Options.
|
|
440
|
-
* @param {string[]} args.composeArgs - Compose global arguments.
|
|
441
|
-
* @param {import("./template.js").TemplateContext} args.context - Template context.
|
|
442
|
-
* @param {number} args.proxyPort - Rollbridge proxy port the unit must not front.
|
|
443
|
-
* @param {CommandRunner} args.runCommand - Command runner.
|
|
444
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
445
|
-
* @returns {Promise<string | undefined>} Why the unit is not ready yet, or undefined when ready.
|
|
446
|
-
*/
|
|
447
|
-
async function activationBlocker({composeArgs, context, proxyPort, runCommand, unit}) {
|
|
448
|
-
const container = await inspectUnitContainer({composeArgs, runCommand, unit})
|
|
449
|
-
|
|
450
|
-
if (container.containerId === null) return "the service has no container yet"
|
|
451
|
-
if (container.state !== "running") return `container ${container.containerId} is ${container.state || "not running"}`
|
|
452
|
-
if (unit.activation.healthcheck && container.healthStatus !== "" && container.healthStatus !== "healthy") return `container ${container.containerId} health is ${container.healthStatus}`
|
|
453
|
-
|
|
454
|
-
const frontingError = proxyFrontingError({container, proxyPort, unit})
|
|
455
|
-
|
|
456
|
-
if (frontingError) return frontingError
|
|
457
|
-
|
|
458
|
-
if (unit.activation.readyCommand !== undefined) {
|
|
459
|
-
const result = await runCommand({command: renderTemplate(unit.activation.readyCommand, context), cwd: unit.projectDirectory, timeoutMs: DOCKER_INSPECT_TIMEOUT_MS})
|
|
460
|
-
|
|
461
|
-
if (result.code !== 0) return `readiness command failed: ${result.stderr.trim() || `exit ${result.code}`}`
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
if (unit.activation.readyUrl !== undefined) {
|
|
465
|
-
try {
|
|
466
|
-
const response = await fetch(unit.activation.readyUrl)
|
|
467
|
-
|
|
468
|
-
if (!response.ok) return `readiness URL ${unit.activation.readyUrl} returned HTTP ${response.status}`
|
|
469
|
-
} catch (error) {
|
|
470
|
-
return `readiness URL ${unit.activation.readyUrl} failed: ${error instanceof Error ? error.message : String(error)}`
|
|
471
|
-
}
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
return undefined
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
/**
|
|
478
|
-
* @param {object} args - Options.
|
|
479
|
-
* @param {string[]} args.composeArgs - Compose global arguments.
|
|
480
|
-
* @param {CommandRunner} args.runCommand - Command runner.
|
|
481
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
482
|
-
* @returns {Promise<ContainerState>} Live container state, or an absent marker.
|
|
483
|
-
*/
|
|
484
|
-
async function inspectUnitContainer({composeArgs, runCommand, unit}) {
|
|
485
|
-
const listed = await runCommand({args: [...composeArgs, "ps", "-q", unit.service], cwd: unit.projectDirectory, timeoutMs: DOCKER_INSPECT_TIMEOUT_MS})
|
|
486
|
-
|
|
487
|
-
if (listed.code !== 0) {
|
|
488
|
-
throw new Error(`Could not list container for Compose unit "${unit.id}": ${listed.stderr.trim() || `exit ${listed.code}`}`)
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
const containerId = listed.stdout.split("\n").map((line) => line.trim()).filter(Boolean).at(-1)
|
|
492
|
-
|
|
493
|
-
if (containerId === undefined) return {containerId: null, healthStatus: "", image: null, ports: [], state: "absent"}
|
|
494
|
-
|
|
495
|
-
const inspected = await runCommand({args: ["docker", "inspect", "--format", INSPECT_FORMAT, containerId], timeoutMs: DOCKER_INSPECT_TIMEOUT_MS})
|
|
496
|
-
|
|
497
|
-
if (inspected.code !== 0) {
|
|
498
|
-
throw new Error(`Could not inspect container ${containerId} for Compose unit "${unit.id}": ${inspected.stderr.trim() || `exit ${inspected.code}`}`)
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
const [image, state, healthStatus, ports, portBindings] = inspected.stdout.trim().split("|")
|
|
502
|
-
|
|
503
|
-
return {
|
|
504
|
-
containerId,
|
|
505
|
-
healthStatus: healthStatus ?? "",
|
|
506
|
-
image: image && image.length > 0 ? image : null,
|
|
507
|
-
ports: [...new Set([...publishedHostPorts(ports), ...publishedHostPorts(portBindings)])],
|
|
508
|
-
state: state ?? ""
|
|
509
|
-
}
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
/**
|
|
513
|
-
* @param {string | undefined} json - `docker inspect` port mapping JSON.
|
|
514
|
-
* @returns {number[]} Host ports the container publishes.
|
|
515
|
-
*/
|
|
516
|
-
function publishedHostPorts(json) {
|
|
517
|
-
if (!json || json === "null") return []
|
|
518
|
-
|
|
519
|
-
const parsed = parseJsonOrNull(json)
|
|
520
|
-
|
|
521
|
-
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return []
|
|
522
|
-
|
|
523
|
-
return Object.values(parsed).flatMap((bindings) => {
|
|
524
|
-
if (!Array.isArray(bindings)) return []
|
|
525
|
-
|
|
526
|
-
return bindings.flatMap((binding) => {
|
|
527
|
-
if (!binding || typeof binding !== "object" || Array.isArray(binding)) return []
|
|
528
|
-
|
|
529
|
-
const hostPort = binding.HostPort
|
|
530
|
-
|
|
531
|
-
return typeof hostPort === "string" && /^\d+$/.test(hostPort) ? [Number(hostPort)] : []
|
|
532
|
-
})
|
|
533
|
-
})
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
/**
|
|
537
|
-
* A Compose unit must never front the stable Rollbridge proxy port: rolling such a container
|
|
538
|
-
* would bounce the proxied entry point in place, which Rollbridge forbids for proxied processes.
|
|
539
|
-
* @param {object} args - Options.
|
|
540
|
-
* @param {ContainerState} args.container - Live container state for the unit.
|
|
541
|
-
* @param {number} args.proxyPort - Rollbridge proxy port.
|
|
542
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
543
|
-
* @returns {string | undefined} Refusal message when the container fronts the proxy, else undefined.
|
|
544
|
-
*/
|
|
545
|
-
function proxyFrontingError({container, proxyPort, unit}) {
|
|
546
|
-
if (container.containerId === null) return undefined
|
|
547
|
-
if (!container.ports.includes(proxyPort)) return undefined
|
|
548
|
-
|
|
549
|
-
return `Compose unit "${unit.id}" (container ${container.containerId}) publishes port ${proxyPort}, the Rollbridge proxy port; a Compose unit must never replace the proxied entry point in place`
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
/**
|
|
553
|
-
* @param {string} json - JSON text emitted by a command.
|
|
554
|
-
* @returns {JsonValue} Parsed value, or null when the text is not JSON.
|
|
555
|
-
*/
|
|
556
|
-
function parseJsonOrNull(json) {
|
|
557
|
-
try {
|
|
558
|
-
return /** @type {JsonValue} */ (JSON.parse(json))
|
|
559
|
-
} catch {
|
|
560
|
-
return null
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
|
|
564
|
-
/**
|
|
565
|
-
* Refuses a Compose roll while the daemon has an unresolved release generation transition, so a
|
|
566
|
-
* container roll can never race or bypass a proxied switch.
|
|
567
|
-
* @param {object} args - Options.
|
|
568
|
-
* @param {RollbridgeConfig} args.config - Rollbridge config.
|
|
569
|
-
* @param {(socketPath: string) => Promise<import("./daemon.js").ControlSocketInspection>} args.inspectSocket - Control socket probe.
|
|
570
|
-
* @param {(args: {command: Record<string, JsonValue>, path: string}) => Promise<Record<string, JsonValue>>} args.sendCommand - Control command sender.
|
|
571
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
572
|
-
* @returns {Promise<void>} Resolves when no transition is in flight.
|
|
573
|
-
*/
|
|
574
|
-
async function assertNoUnresolvedReleaseTransition({config, inspectSocket, sendCommand, unit}) {
|
|
575
|
-
let inspection
|
|
576
|
-
|
|
577
|
-
try {
|
|
578
|
-
inspection = await inspectSocket(config.control.path)
|
|
579
|
-
} catch {
|
|
580
|
-
return
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
if (!inspection.alive) return
|
|
584
|
-
|
|
585
|
-
if (inspection.application !== undefined && inspection.application !== config.application) {
|
|
586
|
-
throw new Error(`A Rollbridge daemon for "${inspection.application}" owns ${config.control.path}; refusing to roll Compose unit "${unit.id}" for "${config.application}".`)
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
const status = await sendCommand({command: {command: "status", includeLogs: false}, path: config.control.path})
|
|
590
|
-
const transition = status.generationTransition
|
|
591
|
-
|
|
592
|
-
if (!transition || typeof transition !== "object" || Array.isArray(transition)) return
|
|
593
|
-
|
|
594
|
-
const phase = transition.phase
|
|
595
|
-
|
|
596
|
-
if (typeof phase === "string" && phase !== "committed") {
|
|
597
|
-
const candidateReleaseId = typeof transition.candidateReleaseId === "string" ? transition.candidateReleaseId : "unknown"
|
|
598
|
-
|
|
599
|
-
throw new Error(`Cannot roll Compose unit "${unit.id}" while release generation transition ${candidateReleaseId} is unresolved at ${phase}; resume the exact deploy first`)
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
/**
|
|
604
|
-
* @param {string} image - Image reference to roll to.
|
|
605
|
-
* @param {ComposeUnitConfig} unit - Compose unit config.
|
|
606
|
-
* @returns {void}
|
|
607
|
-
*/
|
|
608
|
-
function assertImageReference(image, unit) {
|
|
609
|
-
if (image.length === 0 || /\s/.test(image) || image.startsWith("-")) {
|
|
610
|
-
throw new Error(`Compose unit "${unit.id}" target image "${image}" is not a valid image reference`)
|
|
611
|
-
}
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
/**
|
|
615
|
-
* @param {object} args - Options.
|
|
616
|
-
* @param {string} args.application - Application name.
|
|
617
|
-
* @param {string[]} args.composeArgs - Compose global arguments.
|
|
618
|
-
* @param {string} args.releaseId - Roll generation id.
|
|
619
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
620
|
-
* @returns {import("./template.js").TemplateContext} Template context for the unit's hook commands.
|
|
621
|
-
*/
|
|
622
|
-
function templateContext({application, composeArgs, releaseId, unit}) {
|
|
623
|
-
return {
|
|
624
|
-
application,
|
|
625
|
-
composeArgs: composeArgs.join(" "),
|
|
626
|
-
composeFile: path.resolve(unit.projectDirectory, unit.composeFile),
|
|
627
|
-
env: {...process.env},
|
|
628
|
-
projectDirectory: unit.projectDirectory,
|
|
629
|
-
releaseId,
|
|
630
|
-
service: unit.service,
|
|
631
|
-
unitId: unit.id
|
|
632
|
-
}
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
/**
|
|
636
|
-
* @param {ComposeUnitConfig} unit - Compose unit config.
|
|
637
|
-
* @returns {string} Path of the generated override file for one roll.
|
|
638
|
-
*/
|
|
639
|
-
function overrideComposePath(unit) {
|
|
640
|
-
return path.join(unit.projectDirectory, `.rollbridge-${unit.id}.override.yml`)
|
|
641
|
-
}
|
|
642
|
-
|
|
643
|
-
/**
|
|
644
|
-
* @param {object} args - Options.
|
|
645
|
-
* @param {string} args.targetImage - Image to pin for this roll.
|
|
646
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
647
|
-
* @returns {string} Compose override pinning the image and the stop grace period.
|
|
648
|
-
*/
|
|
649
|
-
function overrideComposeContents({targetImage, unit}) {
|
|
650
|
-
const stopGracePeriod = `${Math.max(1, Math.round(unit.stopGracePeriodMs / 1000))}s`
|
|
651
|
-
|
|
652
|
-
return [
|
|
653
|
-
"# Generated by Rollbridge for this roll; do not edit or commit it.",
|
|
654
|
-
"services:",
|
|
655
|
-
` ${yamlQuote(unit.service)}:`,
|
|
656
|
-
` image: ${yamlQuote(targetImage)}`,
|
|
657
|
-
` stop_grace_period: ${stopGracePeriod}`,
|
|
658
|
-
""
|
|
659
|
-
].join("\n")
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
/**
|
|
663
|
-
* @param {string} value - Scalar to emit as a quoted YAML string.
|
|
664
|
-
* @returns {string} Quoted scalar.
|
|
665
|
-
*/
|
|
666
|
-
function yamlQuote(value) {
|
|
667
|
-
return `"${value.replace(/\\/g, "\\\\").replace(/"/g, "\\\"")}"`
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
/**
|
|
671
|
-
* @param {object} args - Options.
|
|
672
|
-
* @param {Omit<ComposeGeneration, "unitId">} args.generation - Generation to record.
|
|
673
|
-
* @param {ComposeLedger} args.ledger - Current ledger.
|
|
674
|
-
* @param {RollbridgeConfig} args.config - Rollbridge config.
|
|
675
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
676
|
-
* @returns {Promise<ComposeGeneration[]>} Retained generations after the append.
|
|
677
|
-
*/
|
|
678
|
-
async function appendGeneration({config, generation, ledger, unit}) {
|
|
679
|
-
const generations = [...ledger.generations, {...generation, unitId: unit.id}].slice(-GENERATION_HISTORY_LIMIT)
|
|
680
|
-
|
|
681
|
-
await writeState(composeLedgerPath(config), {generations, unitId: unit.id})
|
|
682
|
-
|
|
683
|
-
return generations
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
/**
|
|
687
|
-
* @param {object} args - Options.
|
|
688
|
-
* @param {string | null} args.containerId - Container still serving.
|
|
689
|
-
* @param {string} args.error - Failure reason.
|
|
690
|
-
* @param {ComposeLedger} args.ledger - Ledger to keep unchanged.
|
|
691
|
-
* @param {string | null} args.previousImage - Image still selected for rollback.
|
|
692
|
-
* @param {string} args.releaseId - Roll generation id.
|
|
693
|
-
* @param {string} args.stage - Stage that failed.
|
|
694
|
-
* @param {string} args.targetImage - Image the roll was attempting.
|
|
695
|
-
* @param {ComposeUnitConfig} args.unit - Compose unit config.
|
|
696
|
-
* @returns {ComposeRollResult} Failed roll report; the ledger is left untouched.
|
|
697
|
-
*/
|
|
698
|
-
function failedRoll({containerId, error, ledger, previousImage, releaseId, stage, targetImage, unit}) {
|
|
699
|
-
return {
|
|
700
|
-
containerId,
|
|
701
|
-
error,
|
|
702
|
-
generations: ledger.generations,
|
|
703
|
-
image: previousImage ?? targetImage,
|
|
704
|
-
previousImage,
|
|
705
|
-
releaseId,
|
|
706
|
-
rollback: undefined,
|
|
707
|
-
stage,
|
|
708
|
-
status: "failed",
|
|
709
|
-
unitId: unit.id
|
|
710
|
-
}
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
/**
|
|
714
|
-
* @param {JsonValue} value - Candidate ledger entry.
|
|
715
|
-
* @param {string} unitId - Compose unit the entry must belong to.
|
|
716
|
-
* @returns {value is ComposeGeneration} True when the entry is a usable generation record.
|
|
717
|
-
*/
|
|
718
|
-
function isGeneration(value, unitId) {
|
|
719
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return false
|
|
720
|
-
|
|
721
|
-
const generation = /** @type {Record<string, JsonValue>} */ (value)
|
|
722
|
-
|
|
723
|
-
return typeof generation.image === "string" && typeof generation.releaseId === "string" && typeof generation.activatedAt === "string" &&
|
|
724
|
-
generation.unitId === unitId && (generation.status === "activated" || generation.status === "rolled_back")
|
|
725
|
-
}
|