rollbridge 0.1.73 → 0.1.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -10
- package/TODO.md +13 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/changelog.d/20261106-release-dependency-boot-budgets.md +32 -0
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +5 -0
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/changelog.d/20261115-wedged-generation-escalation.md +44 -0
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +30 -0
- package/changelog.d/20261116-config-json-canonical-shape.md +11 -0
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +29 -0
- package/changelog.d/20261116-guardian-version-handoff.md +37 -0
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +7 -0
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +1 -0
- package/changelog.d/20261117-activation-path-drain-deadline.md +35 -0
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +7 -0
- package/changelog.d/20261118-bounded-control-client-close.md +4 -0
- package/changelog.d/20261118-incumbent-retirement-authority-guard.md +1 -0
- package/changelog.d/20261118-owner-retirement-no-guardian-bound.md +19 -0
- package/changelog.d/20261118-shutdown-retirement-bound-restored.md +31 -0
- package/docs/cli.md +128 -3
- package/docs/compose-services.md +222 -0
- package/docs/config.md +177 -6
- package/docs/troubleshooting.md +97 -6
- package/docs/velocious.md +1 -1
- package/docs/workers.md +11 -0
- package/examples/tensorbuzz.com.js +3 -2
- package/package.json +1 -1
- package/src/cli.js +69 -3
- package/src/compose-unit.js +725 -0
- package/src/config.js +346 -8
- package/src/control-client.js +32 -5
- package/src/daemon.js +448 -76
- package/src/doctor.js +74 -0
- package/src/drain-policy.js +252 -0
- package/src/guardian-client.js +92 -9
- package/src/health.js +99 -8
- package/src/managed-process.js +197 -16
- package/src/process-guardian.js +212 -22
- package/src/process-memory.js +62 -2
- package/src/process-order.js +26 -0
- package/src/release-group.js +452 -39
- package/tensorbuzz.yml +20 -1
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +160 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/config-validation.test.js +135 -5
- package/test/control-protocol.test.js +22 -0
- package/test/drain-policy.test.js +328 -0
- package/test/fixtures/deferred-boot-app.js +106 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +149 -5
- package/test/health.test.js +74 -1
- package/test/managed-process.test.js +260 -1
- package/test/owner-recovery.test.js +151 -20
- package/test/owner-replacement.test.js +176 -15
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +447 -2
- package/test/rollbridge.test.js +774 -72
- package/test/shutdown-completion.test.js +119 -0
- package/test/shutdown-retirement-bound.test.js +162 -0
- package/test/support/process.js +3 -1
- package/test/tensorbuzz-config.test.js +228 -0
package/src/config.js
CHANGED
|
@@ -6,6 +6,7 @@ import {createRequire} from "node:module"
|
|
|
6
6
|
import os from "node:os"
|
|
7
7
|
import path from "node:path"
|
|
8
8
|
import {pathToFileURL} from "node:url"
|
|
9
|
+
import {processStartOrder} from "./process-order.js"
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
@@ -17,19 +18,52 @@ import {pathToFileURL} from "node:url"
|
|
|
17
18
|
* @typedef {{activateCommand?: string, activateTimeoutMs?: number, drainCommand?: string, drainTimeoutMs: number, quietCommand?: string, reactivateCommand?: string, stopCommand?: string}} LifecycleConfig
|
|
18
19
|
* @typedef {number | "indefinite"} StopTimeoutMs
|
|
19
20
|
* @typedef {"persistent" | "handoff"} ServiceDeployStrategy
|
|
20
|
-
* @typedef {{cwd?: string, deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
|
|
21
|
+
* @typedef {{cwd?: string, dependencyTimeoutMs: number, dependsOn: string[], deployStrategy: ServiceDeployStrategy, env: Record<string, string>, gracefulStopMs: StopTimeoutMs, health?: HealthConfig, id: string, lifecycle: LifecycleConfig, memory?: MemoryConfig, nonBlockingDrain: boolean, outputLines: number, policy: ProcessPolicy, port?: PortRange, replicas: number, restart: RestartConfig, restartDelayMs: number, stopSignal: string, command: string}} ProcessConfig
|
|
21
22
|
* @typedef {{group?: number | string, mode?: number, owner?: number | string, path: string}} ControlConfig
|
|
22
|
-
* @typedef {{drainTimeoutMs: number, forceStopTimeoutMs: number, healthPath: string, healthTimeoutMs: number, host: string, port: number, upstreamHost: string}} ProxyConfig
|
|
23
|
+
* @typedef {{drainTimeoutMs: number, forceStopTimeoutMs: number, healthPath: string, healthTimeoutMs: number, host: string, nonBlockingDrainTimeoutMs: number, port: number, upstreamHost: string}} ProxyConfig
|
|
23
24
|
* @typedef {{includes: string[], name: string}} LegacyTakeoverProcessConfig
|
|
24
25
|
* @typedef {{forceStopTimeoutMs: number, processes: LegacyTakeoverProcessConfig[], screens: string[]}} LegacyTakeoverConfig
|
|
25
|
-
* @typedef {{escalateWedgedGenerations
|
|
26
|
+
* @typedef {{escalateWedgedGenerations: boolean, keep: number, maxAgeMs: number, maxConcurrentDrainingReleaseGroups: number}} ReleaseRetentionConfig
|
|
26
27
|
* @typedef {{reconnectGraceMs: number}} OwnerRecoveryConfig
|
|
27
|
-
* @typedef {{
|
|
28
|
+
* @typedef {{command: string | undefined, timeoutMs: number}} ComposeDrainConfig
|
|
29
|
+
* @typedef {{command: string | undefined, intervalMs: number, timeoutMs: number}} ComposeInFlightConfig
|
|
30
|
+
* @typedef {{healthcheck: boolean, intervalMs: number, readyCommand: string | undefined, readyUrl: string | undefined, startDelayMs: number, timeoutMs: number}} ComposeActivationConfig
|
|
31
|
+
* @typedef {{enabled: boolean}} ComposeRollbackConfig
|
|
32
|
+
* @typedef {{activation: ComposeActivationConfig, composeFile: string, drain: ComposeDrainConfig, envFile: string | undefined, id: string, image: string, inFlight: ComposeInFlightConfig, projectName: string | undefined, projectDirectory: string, recreateTimeoutMs: number, rollback: ComposeRollbackConfig, service: string, stopGracePeriodMs: number}} ComposeUnitConfig
|
|
33
|
+
* @typedef {{application: string, compose?: ComposeUnitConfig[], composeStatePath?: string, control: ControlConfig, legacyTakeover?: LegacyTakeoverConfig, ownerRecovery?: OwnerRecoveryConfig, processes: ProcessConfig[], proxy: ProxyConfig, releaseRetention: ReleaseRetentionConfig, statePath?: string}} RollbridgeConfig
|
|
28
34
|
* @typedef {{fix: string, message: string}} ConfigIssue
|
|
29
35
|
*/
|
|
30
36
|
|
|
31
37
|
const PROCESS_POLICIES = new Set(["proxied", "companion", "singleton", "service"])
|
|
32
38
|
const DEFAULT_CONFIG_FILENAMES = ["rollbridge.js"]
|
|
39
|
+
// Budget for one process waiting for a dependency's port, separate from the health budget.
|
|
40
|
+
// Measured on TensorBuzz production (server3, 2026-11-06): the candidate's
|
|
41
|
+
// background-jobs-main took 11817ms (14:33:39.175 start -> 14:33:50.992 listening) before
|
|
42
|
+
// the web process could reach its own boot, and the production command chain already waited
|
|
43
|
+
// 30000ms per hop. 30000ms keeps that dependency budget, so the health deadline measures
|
|
44
|
+
// only the app boot.
|
|
45
|
+
const DEFAULT_DEPENDENCY_TIMEOUT_MS = 30000
|
|
46
|
+
// Budget for one process to answer its health path for the first time, measured from its
|
|
47
|
+
// launch. Measured on TensorBuzz production (server3, 2026-11-06): the 30000ms release
|
|
48
|
+
// health deadline killed two candidate web processes mid-boot. Release 20261106142602
|
|
49
|
+
// started web at 14:33:39.205, was still printing attachment storage root migration lines
|
|
50
|
+
// at 14:34:08.842, and got SIGTERM at 14:34:09.492 without ever logging a listening line,
|
|
51
|
+
// so the app's own boot alone exceeded 30000ms; release 20261106144454 was killed 15373ms
|
|
52
|
+
// after its web process started, before reaching those lines. 120000ms is the budget the
|
|
53
|
+
// same app has been published on since (config/rollbridge.production.js, merged
|
|
54
|
+
// 2026-11-06) and is made the default so a slow boot is not a trap every project has to
|
|
55
|
+
// know to configure. A candidate that never becomes healthy still fails the deploy at the
|
|
56
|
+
// deadline while the incumbent keeps serving; the budget only moves when the gate reports.
|
|
57
|
+
const DEFAULT_HEALTH_TIMEOUT_MS = 120000
|
|
58
|
+
// Bound on how many release groups may drain at once before a new handoff is refused.
|
|
59
|
+
// Measured on TensorBuzz production (2026-11-14/15, shared/log/rollbridge.log): 16 releases
|
|
60
|
+
// drained in 18-114 minutes with a median of ~37 minutes, while deploys arrived every
|
|
61
|
+
// 3-7 minutes (7 traffic switches in 61 minutes on 2026-11-14). A bound of 1 — the value a
|
|
62
|
+
// consumer gate effectively had — is unsatisfiable at that cadence, so every deploy adds one
|
|
63
|
+
// more draining generation. 37 / 3 = 12 is the backlog the measured drain median produces at
|
|
64
|
+
// the fastest measured cadence, so this default admits normal operation and refuses only
|
|
65
|
+
// growth beyond it. 0 disables the bound.
|
|
66
|
+
const DEFAULT_MAX_CONCURRENT_DRAINING_RELEASE_GROUPS = 12
|
|
33
67
|
const commonJsRequire = createRequire(import.meta.url)
|
|
34
68
|
|
|
35
69
|
/**
|
|
@@ -72,6 +106,7 @@ export async function loadConfig(configPath) {
|
|
|
72
106
|
|
|
73
107
|
config.control.path = path.resolve(configDirectory, config.control.path)
|
|
74
108
|
if (config.statePath) config.statePath = path.resolve(configDirectory, config.statePath)
|
|
109
|
+
if (config.composeStatePath) config.composeStatePath = path.resolve(configDirectory, config.composeStatePath)
|
|
75
110
|
return config
|
|
76
111
|
}
|
|
77
112
|
|
|
@@ -125,6 +160,32 @@ export function normalizeConfig(rawConfig, configPath = process.cwd()) {
|
|
|
125
160
|
return config
|
|
126
161
|
}
|
|
127
162
|
|
|
163
|
+
/**
|
|
164
|
+
* Drops every undefined-valued property so a normalized config is identical to its own JSON form.
|
|
165
|
+
*
|
|
166
|
+
* Owner replacement hands the candidate daemon its release configs through the guardian's JSON
|
|
167
|
+
* socket, and JSON serialization drops undefined-valued keys. The inherited config then compares
|
|
168
|
+
* unequal to a freshly loaded one under `isDeepStrictEqual` even when the config file never
|
|
169
|
+
* changed, and the live-reload fence aborts the next deploy. Measured in production on 2026-11-16
|
|
170
|
+
* (tensorbuzz/production): after a failed activation was compensated inside a replacement daemon,
|
|
171
|
+
* the following deploy failed with "Config changes to control cannot be applied live" while both
|
|
172
|
+
* config files were byte-identical and hashed to the same `ownerConfigDigest` — that digest is
|
|
173
|
+
* JSON-based, so the daemon held two contradictory notions of the same config and only the
|
|
174
|
+
* JSON-shaped one survived the handoff.
|
|
175
|
+
* @param {JsonValue} value - Normalized config value.
|
|
176
|
+
* @returns {JsonValue} The same value with every undefined-valued property removed.
|
|
177
|
+
*/
|
|
178
|
+
function withoutUndefinedProperties(value) {
|
|
179
|
+
if (Array.isArray(value)) return value.map(withoutUndefinedProperties)
|
|
180
|
+
if (isPlainObject(value)) {
|
|
181
|
+
return Object.fromEntries(Object.entries(value)
|
|
182
|
+
.filter(([, entryValue]) => entryValue !== undefined)
|
|
183
|
+
.map(([key, entryValue]) => [key, withoutUndefinedProperties(entryValue)]))
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
return value
|
|
187
|
+
}
|
|
188
|
+
|
|
128
189
|
/**
|
|
129
190
|
* Validates a raw config object and collects every issue instead of throwing on the first one.
|
|
130
191
|
* @param {JsonValue} rawConfig - Parsed config.
|
|
@@ -158,13 +219,32 @@ export function validateConfig(rawConfig, configPath = process.cwd()) {
|
|
|
158
219
|
const ownerRecoverySource = source.ownerRecovery === undefined || source.ownerRecovery === null ? undefined : objectAt(source.ownerRecovery, "ownerRecovery", issues)
|
|
159
220
|
const reconnectGraceMs = ownerRecoverySource ? normalizeNumber(ownerRecoverySource.reconnectGraceMs, "ownerRecovery.reconnectGraceMs", issues, {default: 30000}) : 30000
|
|
160
221
|
const ownerRecovery = ownerRecoverySource ? {reconnectGraceMs: nonNegativeOrDefault(reconnectGraceMs, "ownerRecovery.reconnectGraceMs", issues, 30000, true)} : undefined
|
|
222
|
+
const compose = normalizeComposeUnits(source.compose, issues)
|
|
223
|
+
const composeStatePath = compose.length === 0 && source.composeStatePath === undefined && source.composeStatePath !== null
|
|
224
|
+
? undefined
|
|
225
|
+
: normalizeString(source.composeStatePath, "composeStatePath", issues, {default: path.resolve(path.dirname(configPath), `rollbridge-${application}.compose.state.json`)})
|
|
161
226
|
|
|
162
227
|
if (ownerRecovery && !statePath) issues.push({fix: "Configure statePath when ownerRecovery is enabled.", message: "ownerRecovery requires statePath"})
|
|
163
228
|
|
|
164
229
|
validateProcessSet(processes, issues)
|
|
165
230
|
validateActivationLifecycle(processes, ownerRecovery, statePath, issues)
|
|
231
|
+
validateComposeUnits(compose, processes, issues)
|
|
166
232
|
|
|
167
|
-
return {
|
|
233
|
+
return {
|
|
234
|
+
config: /** @type {RollbridgeConfig} */ (withoutUndefinedProperties(/** @type {JsonValue} */ ({
|
|
235
|
+
application,
|
|
236
|
+
...(compose.length > 0 ? {compose} : {}),
|
|
237
|
+
...(composeStatePath ? {composeStatePath} : {}),
|
|
238
|
+
control,
|
|
239
|
+
legacyTakeover,
|
|
240
|
+
ownerRecovery,
|
|
241
|
+
processes,
|
|
242
|
+
proxy,
|
|
243
|
+
releaseRetention,
|
|
244
|
+
statePath
|
|
245
|
+
}))),
|
|
246
|
+
issues
|
|
247
|
+
}
|
|
168
248
|
}
|
|
169
249
|
|
|
170
250
|
/**
|
|
@@ -179,8 +259,9 @@ function normalizeProxy(source, issues) {
|
|
|
179
259
|
drainTimeoutMs: normalizeNumber(source.drainTimeoutMs, "proxy.drainTimeoutMs", issues, {default: 60000}),
|
|
180
260
|
forceStopTimeoutMs: normalizeNumber(source.forceStopTimeoutMs, "proxy.forceStopTimeoutMs", issues, {default: 10000}),
|
|
181
261
|
healthPath: normalizeString(source.healthPath, "proxy.healthPath", issues, {default: "/ping"}),
|
|
182
|
-
healthTimeoutMs: normalizeNumber(source.healthTimeoutMs, "proxy.healthTimeoutMs", issues, {default:
|
|
262
|
+
healthTimeoutMs: normalizeNumber(source.healthTimeoutMs, "proxy.healthTimeoutMs", issues, {default: DEFAULT_HEALTH_TIMEOUT_MS}),
|
|
183
263
|
host,
|
|
264
|
+
nonBlockingDrainTimeoutMs: normalizeNumber(source.nonBlockingDrainTimeoutMs, "proxy.nonBlockingDrainTimeoutMs", issues, {default: 60000}),
|
|
184
265
|
port: normalizeNumber(source.port, "proxy.port", issues, {default: 8182}),
|
|
185
266
|
upstreamHost: normalizeString(source.upstreamHost, "proxy.upstreamHost", issues, {default: defaultUpstreamHost(host)})
|
|
186
267
|
}
|
|
@@ -207,7 +288,7 @@ function normalizeProcess(value, index, proxy, issues) {
|
|
|
207
288
|
if (!isPlainObject(value)) {
|
|
208
289
|
issues.push({fix: `Define processes[${index}] as a mapping with id, policy, and command.`, message: `processes[${index}] must be an object`})
|
|
209
290
|
|
|
210
|
-
return {command: "", cwd: undefined, deployStrategy: "persistent", env: {}, gracefulStopMs: proxy.forceStopTimeoutMs, health: undefined, id: "", lifecycle: {drainTimeoutMs: 0}, memory: undefined, nonBlockingDrain: false, outputLines: 50, policy: "companion", port: undefined, replicas: 1, restart: defaultRestartConfig(), restartDelayMs: 1000, stopSignal: "SIGTERM"}
|
|
291
|
+
return {command: "", cwd: undefined, dependencyTimeoutMs: DEFAULT_DEPENDENCY_TIMEOUT_MS, dependsOn: [], deployStrategy: "persistent", env: {}, gracefulStopMs: proxy.forceStopTimeoutMs, health: undefined, id: "", lifecycle: {drainTimeoutMs: 0}, memory: undefined, nonBlockingDrain: false, outputLines: 50, policy: "companion", port: undefined, replicas: 1, restart: defaultRestartConfig(), restartDelayMs: 1000, stopSignal: "SIGTERM"}
|
|
211
292
|
}
|
|
212
293
|
|
|
213
294
|
const source = value
|
|
@@ -215,6 +296,8 @@ function normalizeProcess(value, index, proxy, issues) {
|
|
|
215
296
|
return {
|
|
216
297
|
command: normalizeString(source.command, `processes[${index}].command`, issues),
|
|
217
298
|
cwd: source.cwd === undefined ? undefined : normalizeString(source.cwd, `processes[${index}].cwd`, issues),
|
|
299
|
+
dependencyTimeoutMs: normalizeDependencyTimeoutMs(source.dependencyTimeoutMs, `processes[${index}].dependencyTimeoutMs`, issues),
|
|
300
|
+
dependsOn: normalizeDependsOn(source.dependsOn, `processes[${index}].dependsOn`, issues),
|
|
218
301
|
deployStrategy: normalizeDeployStrategy(source.deployStrategy, `processes[${index}].deployStrategy`, issues),
|
|
219
302
|
env: normalizeEnv(source.env, `processes[${index}].env`, issues),
|
|
220
303
|
gracefulStopMs: normalizeStopTimeout(source.gracefulStopMs, `processes[${index}].gracefulStopMs`, proxy.forceStopTimeoutMs, issues),
|
|
@@ -518,6 +601,180 @@ function normalizeLegacyTakeoverProcess(value, index, issues) {
|
|
|
518
601
|
}]
|
|
519
602
|
}
|
|
520
603
|
|
|
604
|
+
/**
|
|
605
|
+
* @param {JsonValue} value - Raw Compose unit list.
|
|
606
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
607
|
+
* @returns {ComposeUnitConfig[]} Normalized Compose units (empty when the key is absent).
|
|
608
|
+
*/
|
|
609
|
+
function normalizeComposeUnits(value, issues) {
|
|
610
|
+
if (value === undefined || value === null) return []
|
|
611
|
+
|
|
612
|
+
if (!Array.isArray(value)) {
|
|
613
|
+
issues.push({fix: "Set compose to a list of Compose service units, or remove the key.", message: "compose must be an array"})
|
|
614
|
+
|
|
615
|
+
return []
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
if (value.length === 0) {
|
|
619
|
+
issues.push({fix: "Define at least one compose unit, or remove the compose key.", message: "compose must define at least one unit"})
|
|
620
|
+
|
|
621
|
+
return []
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
return value.map((unitSource, index) => normalizeComposeUnit(unitSource, index, issues))
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* @param {JsonValue} value - Raw Compose unit.
|
|
629
|
+
* @param {number} index - Unit index.
|
|
630
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
631
|
+
* @returns {ComposeUnitConfig} Normalized Compose unit.
|
|
632
|
+
*/
|
|
633
|
+
function normalizeComposeUnit(value, index, issues) {
|
|
634
|
+
const key = `compose[${index}]`
|
|
635
|
+
|
|
636
|
+
if (!isPlainObject(value)) {
|
|
637
|
+
issues.push({fix: `Define ${key} as a mapping with id, service, image, and projectDirectory.`, message: `${key} must be an object`})
|
|
638
|
+
|
|
639
|
+
return {
|
|
640
|
+
activation: {healthcheck: true, intervalMs: 1000, readyCommand: undefined, readyUrl: undefined, startDelayMs: 0, timeoutMs: 60000},
|
|
641
|
+
composeFile: "compose.yml",
|
|
642
|
+
drain: {command: undefined, timeoutMs: 0},
|
|
643
|
+
envFile: undefined,
|
|
644
|
+
id: "",
|
|
645
|
+
image: "",
|
|
646
|
+
inFlight: {command: undefined, intervalMs: 1000, timeoutMs: 0},
|
|
647
|
+
projectName: undefined,
|
|
648
|
+
projectDirectory: "",
|
|
649
|
+
recreateTimeoutMs: 120000,
|
|
650
|
+
rollback: {enabled: true},
|
|
651
|
+
service: "",
|
|
652
|
+
stopGracePeriodMs: 10000
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
const source = value
|
|
657
|
+
const stopGracePeriodMs = normalizePositiveNumber(source.stopGracePeriodMs, `${key}.stopGracePeriodMs`, issues, 10000)
|
|
658
|
+
const drainSource = source.drain === undefined || source.drain === null ? undefined : objectAt(source.drain, `${key}.drain`, issues, {})
|
|
659
|
+
const inFlightSource = source.inFlight === undefined || source.inFlight === null ? undefined : objectAt(source.inFlight, `${key}.inFlight`, issues, {})
|
|
660
|
+
const activationSource = source.activation === undefined || source.activation === null ? {} : objectAt(source.activation, `${key}.activation`, issues, {})
|
|
661
|
+
const rollbackSource = source.rollback === undefined || source.rollback === null ? undefined : objectAt(source.rollback, `${key}.rollback`, issues, {})
|
|
662
|
+
|
|
663
|
+
/** @type {ComposeUnitConfig} */
|
|
664
|
+
const unit = {
|
|
665
|
+
activation: {
|
|
666
|
+
healthcheck: normalizeBoolean(activationSource.healthcheck, `${key}.activation.healthcheck`, issues, true),
|
|
667
|
+
intervalMs: normalizePositiveNumber(activationSource.intervalMs, `${key}.activation.intervalMs`, issues, 1000),
|
|
668
|
+
readyCommand: activationSource.readyCommand === undefined ? undefined : normalizeString(activationSource.readyCommand, `${key}.activation.readyCommand`, issues, {nonEmpty: true}),
|
|
669
|
+
readyUrl: activationSource.readyUrl === undefined ? undefined : normalizeString(activationSource.readyUrl, `${key}.activation.readyUrl`, issues, {nonEmpty: true}),
|
|
670
|
+
startDelayMs: normalizeStartDelayMs(activationSource.startDelayMs, `${key}.activation.startDelayMs`, issues),
|
|
671
|
+
timeoutMs: normalizePositiveNumber(activationSource.timeoutMs, `${key}.activation.timeoutMs`, issues, 60000)
|
|
672
|
+
},
|
|
673
|
+
composeFile: normalizeString(source.composeFile, `${key}.composeFile`, issues, {default: "compose.yml"}),
|
|
674
|
+
drain: {
|
|
675
|
+
command: drainSource?.command === undefined ? undefined : normalizeString(drainSource.command, `${key}.drain.command`, issues, {nonEmpty: true}),
|
|
676
|
+
timeoutMs: normalizeNumber(drainSource?.timeoutMs, `${key}.drain.timeoutMs`, issues, {default: 60000})
|
|
677
|
+
},
|
|
678
|
+
envFile: source.envFile === undefined || source.envFile === null ? undefined : normalizeString(source.envFile, `${key}.envFile`, issues, {nonEmpty: true}),
|
|
679
|
+
id: normalizeString(source.id, `${key}.id`, issues),
|
|
680
|
+
image: normalizeString(source.image, `${key}.image`, issues),
|
|
681
|
+
inFlight: {
|
|
682
|
+
command: inFlightSource?.command === undefined ? undefined : normalizeString(inFlightSource.command, `${key}.inFlight.command`, issues, {nonEmpty: true}),
|
|
683
|
+
intervalMs: normalizePositiveNumber(inFlightSource?.intervalMs, `${key}.inFlight.intervalMs`, issues, 1000),
|
|
684
|
+
timeoutMs: normalizeNumber(inFlightSource?.timeoutMs, `${key}.inFlight.timeoutMs`, issues, {default: 0})
|
|
685
|
+
},
|
|
686
|
+
projectName: source.projectName === undefined || source.projectName === null ? undefined : normalizeString(source.projectName, `${key}.projectName`, issues, {nonEmpty: true}),
|
|
687
|
+
projectDirectory: normalizeString(source.projectDirectory, `${key}.projectDirectory`, issues),
|
|
688
|
+
recreateTimeoutMs: normalizePositiveNumber(source.recreateTimeoutMs, `${key}.recreateTimeoutMs`, issues, 120000),
|
|
689
|
+
rollback: {enabled: normalizeBoolean(rollbackSource?.enabled, `${key}.rollback.enabled`, issues, true)},
|
|
690
|
+
service: normalizeString(source.service, `${key}.service`, issues),
|
|
691
|
+
stopGracePeriodMs
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
if (unit.drain.command !== undefined && unit.drain.timeoutMs <= 0) {
|
|
695
|
+
issues.push({fix: `Set ${key}.drain.timeoutMs to a positive number of milliseconds; with 0 the quiescence step is skipped and ${key}.drain.command never runs.`, message: `${key}.drain.command requires a positive ${key}.drain.timeoutMs`})
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
if (unit.inFlight.command !== undefined && unit.inFlight.timeoutMs <= 0) {
|
|
699
|
+
issues.push({fix: `Set ${key}.inFlight.timeoutMs to how long the roll may wait for in-flight work, e.g. 120000; with 0 the gate is skipped and ${key}.inFlight.command never runs.`, message: `${key}.inFlight.command requires a positive ${key}.inFlight.timeoutMs`})
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
if (unit.image.length > 0 && !isComposeImageReference(unit.image)) {
|
|
703
|
+
issues.push({fix: `Set ${key}.image to an image reference pinned to a tag or digest, e.g. "registry.local/socketduct-gateway:1.4.0".`, message: `${key}.image "${unit.image}" is not a valid image reference`})
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
return unit
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
/**
|
|
710
|
+
* @param {JsonValue} value - Raw positive-number value.
|
|
711
|
+
* @param {string} key - Config key.
|
|
712
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
713
|
+
* @param {number} fallback - Default and fallback value.
|
|
714
|
+
* @returns {number} Positive number, or the fallback when invalid.
|
|
715
|
+
*/
|
|
716
|
+
function normalizePositiveNumber(value, key, issues, fallback) {
|
|
717
|
+
const number = normalizeNumber(value, key, issues, {default: fallback})
|
|
718
|
+
|
|
719
|
+
if (!(number > 0)) {
|
|
720
|
+
issues.push({fix: `Set ${key} to a positive number of milliseconds, e.g. ${fallback}.`, message: `${key} must be a positive number`})
|
|
721
|
+
|
|
722
|
+
return fallback
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
return number
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
/**
|
|
729
|
+
* @param {string} image - Image reference.
|
|
730
|
+
* @returns {boolean} True when the reference is a usable Docker image name.
|
|
731
|
+
*/
|
|
732
|
+
function isComposeImageReference(image) {
|
|
733
|
+
if (/\s/.test(image) || image.startsWith("-")) return false
|
|
734
|
+
|
|
735
|
+
const [name, digest] = image.split("@")
|
|
736
|
+
|
|
737
|
+
if (name.length === 0) return false
|
|
738
|
+
if (digest === undefined) return true
|
|
739
|
+
|
|
740
|
+
return /^sha256:[a-f0-9]{64}$/.test(digest)
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
/**
|
|
744
|
+
* Validates cross-unit rules: unique unit ids, no collision with process ids, and the fields a
|
|
745
|
+
* drain-first roll needs.
|
|
746
|
+
* @param {ComposeUnitConfig[]} compose - Normalized Compose units.
|
|
747
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
748
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
749
|
+
* @returns {void}
|
|
750
|
+
*/
|
|
751
|
+
function validateComposeUnits(compose, processes, issues) {
|
|
752
|
+
if (compose.length === 0) return
|
|
753
|
+
|
|
754
|
+
const processIds = new Set(processes.map((processConfig) => processConfig.id))
|
|
755
|
+
const seenIds = /** @type {Set<string>} */ (new Set())
|
|
756
|
+
|
|
757
|
+
for (const unit of compose) {
|
|
758
|
+
if (!unit.id) continue
|
|
759
|
+
|
|
760
|
+
if (seenIds.has(unit.id)) {
|
|
761
|
+
issues.push({fix: `Give each compose unit a unique id; "${unit.id}" is used more than once.`, message: `Duplicate compose unit id: ${unit.id}`})
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
seenIds.add(unit.id)
|
|
765
|
+
|
|
766
|
+
if (processIds.has(unit.id)) {
|
|
767
|
+
issues.push({fix: `Rename compose unit "${unit.id}" or the process with the same id; Rollbridge targets both by id.`, message: `Compose unit id "${unit.id}" collides with a process id`})
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
for (const [index, unit] of compose.entries()) {
|
|
772
|
+
if (!unit.service) issues.push({fix: `Set compose[${index}].service to the Compose service name to roll.`, message: `compose[${index}].service is required`})
|
|
773
|
+
if (!unit.projectDirectory) issues.push({fix: `Set compose[${index}].projectDirectory to the directory holding the Compose file.`, message: `compose[${index}].projectDirectory is required`})
|
|
774
|
+
if (!unit.image) issues.push({fix: `Set compose[${index}].image to the target image reference.`, message: `compose[${index}].image is required`})
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
521
778
|
/**
|
|
522
779
|
* @param {JsonValue} value - Raw stop signal.
|
|
523
780
|
* @param {string} key - Config key.
|
|
@@ -566,6 +823,34 @@ function normalizeDeployStrategy(value, key, issues) {
|
|
|
566
823
|
return "persistent"
|
|
567
824
|
}
|
|
568
825
|
|
|
826
|
+
/**
|
|
827
|
+
* @param {JsonValue} value - Raw dependency list.
|
|
828
|
+
* @param {string} key - Config key.
|
|
829
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
830
|
+
* @returns {string[]} Ids of the processes this process waits for before it starts.
|
|
831
|
+
*/
|
|
832
|
+
function normalizeDependsOn(value, key, issues) {
|
|
833
|
+
return normalizeStringList(value, key, issues)
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
/**
|
|
837
|
+
* @param {JsonValue} value - Raw dependency wait budget.
|
|
838
|
+
* @param {string} key - Config key.
|
|
839
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
840
|
+
* @returns {number} Milliseconds this process may spend waiting for its dependency ports.
|
|
841
|
+
*/
|
|
842
|
+
function normalizeDependencyTimeoutMs(value, key, issues) {
|
|
843
|
+
const timeoutMs = normalizeNumber(value, key, issues, {default: DEFAULT_DEPENDENCY_TIMEOUT_MS})
|
|
844
|
+
|
|
845
|
+
if (timeoutMs < 1) {
|
|
846
|
+
issues.push({fix: `Set ${key} to a positive number of milliseconds, e.g. 30000, or omit it for the default.`, message: `${key} must be a positive number`})
|
|
847
|
+
|
|
848
|
+
return DEFAULT_DEPENDENCY_TIMEOUT_MS
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
return timeoutMs
|
|
852
|
+
}
|
|
853
|
+
|
|
569
854
|
/**
|
|
570
855
|
* @param {JsonValue} value - Raw output retention value.
|
|
571
856
|
* @param {string} key - Config key.
|
|
@@ -626,11 +911,13 @@ function normalizeBoolean(value, key, issues, fallback) {
|
|
|
626
911
|
function normalizeReleaseRetention(source, issues) {
|
|
627
912
|
const keep = normalizeNumber(source.keep, "releaseRetention.keep", issues, {default: 10})
|
|
628
913
|
const maxAgeMs = normalizeNumber(source.maxAgeMs, "releaseRetention.maxAgeMs", issues, {default: 0})
|
|
914
|
+
const maxConcurrentDrainingReleaseGroups = normalizeNumber(source.maxConcurrentDrainingReleaseGroups, "releaseRetention.maxConcurrentDrainingReleaseGroups", issues, {default: DEFAULT_MAX_CONCURRENT_DRAINING_RELEASE_GROUPS})
|
|
629
915
|
|
|
630
916
|
return {
|
|
631
917
|
escalateWedgedGenerations: normalizeBoolean(source.escalateWedgedGenerations, "releaseRetention.escalateWedgedGenerations", issues, false),
|
|
632
918
|
keep: nonNegativeOrDefault(keep, "releaseRetention.keep", issues, 10, true),
|
|
633
|
-
maxAgeMs: nonNegativeOrDefault(maxAgeMs, "releaseRetention.maxAgeMs", issues, 0, false)
|
|
919
|
+
maxAgeMs: nonNegativeOrDefault(maxAgeMs, "releaseRetention.maxAgeMs", issues, 0, false),
|
|
920
|
+
maxConcurrentDrainingReleaseGroups: nonNegativeOrDefault(maxConcurrentDrainingReleaseGroups, "releaseRetention.maxConcurrentDrainingReleaseGroups", issues, DEFAULT_MAX_CONCURRENT_DRAINING_RELEASE_GROUPS, true)
|
|
634
921
|
}
|
|
635
922
|
}
|
|
636
923
|
|
|
@@ -751,6 +1038,57 @@ function validateProcessSet(processes, issues) {
|
|
|
751
1038
|
|
|
752
1039
|
issues.push({fix: `Add a port range to the proxied process "${processConfig.id || "(unnamed)"}", e.g. port: {from: 18000, to: 18099}.`, message: `Proxied process ${processConfig.id || "(unnamed)"} must define a port range`})
|
|
753
1040
|
}
|
|
1041
|
+
|
|
1042
|
+
validateProcessDependencies(processes, issues)
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
/**
|
|
1046
|
+
* Validates `dependsOn`: a dependency must name a ported process that Rollbridge starts
|
|
1047
|
+
* before the dependent. Waiting is done on the dependency's allocated port, and an edge that
|
|
1048
|
+
* points backwards in the start order can never be satisfied, which also rules out cycles.
|
|
1049
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
1050
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
1051
|
+
* @returns {void}
|
|
1052
|
+
*/
|
|
1053
|
+
function validateProcessDependencies(processes, issues) {
|
|
1054
|
+
const declared = new Map(processes.filter((processConfig) => processConfig.id).map((processConfig) => [processConfig.id, processConfig]))
|
|
1055
|
+
const startOrder = new Map(processStartOrder(processes, {includePersistentServices: true}).map((processConfig, index) => [processConfig.id, index]))
|
|
1056
|
+
|
|
1057
|
+
for (const processConfig of processes) {
|
|
1058
|
+
if (!processConfig.id || processConfig.dependsOn.length === 0) continue
|
|
1059
|
+
|
|
1060
|
+
const dependentIndex = startOrder.get(processConfig.id)
|
|
1061
|
+
|
|
1062
|
+
for (const dependencyId of processConfig.dependsOn) {
|
|
1063
|
+
if (dependencyId === processConfig.id) {
|
|
1064
|
+
issues.push({fix: `Remove "${processConfig.id}" from its own dependsOn.`, message: `Process "${processConfig.id}" cannot depend on itself`})
|
|
1065
|
+
continue
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
const dependency = declared.get(dependencyId)
|
|
1069
|
+
|
|
1070
|
+
if (!dependency) {
|
|
1071
|
+
issues.push({fix: `Name a configured process in "${processConfig.id}".dependsOn; "${dependencyId}" is not one of: ${[...declared.keys()].join(", ")}.`, message: `Process "${processConfig.id}" depends on unknown process "${dependencyId}"`})
|
|
1072
|
+
continue
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
if (dependency.policy === "singleton") {
|
|
1076
|
+
issues.push({fix: `Remove "${dependencyId}" from "${processConfig.id}".dependsOn; a singleton is not part of a release generation, so Rollbridge cannot gate a release start on it.`, message: `Process "${processConfig.id}" cannot depend on singleton "${dependencyId}"`})
|
|
1077
|
+
continue
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
if (!dependency.port) {
|
|
1081
|
+
issues.push({fix: `Give "${dependencyId}" a port, or remove it from "${processConfig.id}".dependsOn; Rollbridge waits for a dependency on its allocated port.`, message: `Process "${dependencyId}" has no port, so "${processConfig.id}" cannot wait for it`})
|
|
1082
|
+
continue
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
const dependencyIndex = startOrder.get(dependencyId)
|
|
1086
|
+
|
|
1087
|
+
if (dependencyIndex === undefined || dependentIndex === undefined || dependencyIndex > dependentIndex) {
|
|
1088
|
+
issues.push({fix: `Move "${dependencyId}" before "${processConfig.id}" in processes[]; Rollbridge starts persistent services, then handoff services, companions, and the proxied process.`, message: `Process "${processConfig.id}" depends on "${dependencyId}", which Rollbridge starts after it`})
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
754
1092
|
}
|
|
755
1093
|
|
|
756
1094
|
/**
|
package/src/control-client.js
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
import net from "node:net"
|
|
4
4
|
|
|
5
|
+
const DEFAULT_CONTROL_COMMAND_TIMEOUT_MS = 30000
|
|
6
|
+
|
|
5
7
|
/**
|
|
6
8
|
* @typedef {import("./json.js").JsonValue} JsonValue
|
|
7
9
|
*/
|
|
@@ -11,15 +13,40 @@ import net from "node:net"
|
|
|
11
13
|
* @param {object} args - Options.
|
|
12
14
|
* @param {Record<string, JsonValue>} args.command - Command payload.
|
|
13
15
|
* @param {string} args.path - Control socket path.
|
|
16
|
+
* @param {number} [args.timeoutMs] - Maximum time to wait for a response.
|
|
14
17
|
* @returns {Promise<Record<string, JsonValue>>} Response payload.
|
|
15
18
|
*/
|
|
16
|
-
export async function sendControlCommand({command, path}) {
|
|
19
|
+
export async function sendControlCommand({command, path, timeoutMs = DEFAULT_CONTROL_COMMAND_TIMEOUT_MS}) {
|
|
17
20
|
return await new Promise((resolve, reject) => {
|
|
18
21
|
const socket = net.createConnection(path)
|
|
19
22
|
let buffer = ""
|
|
23
|
+
let settled = false
|
|
24
|
+
const timeout = setTimeout(() => {
|
|
25
|
+
socket.destroy()
|
|
26
|
+
settleReject(new Error(`Rollbridge control command ${String(command.command)} on ${path} timed out after ${timeoutMs}ms`))
|
|
27
|
+
}, timeoutMs)
|
|
28
|
+
|
|
29
|
+
/** @param {Error} error - Command failure. */
|
|
30
|
+
const settleReject = (error) => {
|
|
31
|
+
if (settled) return
|
|
32
|
+
settled = true
|
|
33
|
+
clearTimeout(timeout)
|
|
34
|
+
reject(error)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** @param {Record<string, JsonValue>} response - Command response. */
|
|
38
|
+
const settleResolve = (response) => {
|
|
39
|
+
if (settled) return
|
|
40
|
+
settled = true
|
|
41
|
+
clearTimeout(timeout)
|
|
42
|
+
resolve(response)
|
|
43
|
+
}
|
|
20
44
|
|
|
21
45
|
socket.setEncoding("utf8")
|
|
22
|
-
socket.once("error",
|
|
46
|
+
socket.once("error", settleReject)
|
|
47
|
+
socket.once("close", () => {
|
|
48
|
+
if (!settled) settleReject(new Error(`Rollbridge control connection ${path} closed before its response`))
|
|
49
|
+
})
|
|
23
50
|
socket.on("data", (chunk) => {
|
|
24
51
|
buffer += chunk
|
|
25
52
|
const newlineIndex = buffer.indexOf("\n")
|
|
@@ -33,16 +60,16 @@ export async function sendControlCommand({command, path}) {
|
|
|
33
60
|
response = JSON.parse(line)
|
|
34
61
|
} catch (error) {
|
|
35
62
|
socket.destroy()
|
|
36
|
-
|
|
63
|
+
settleReject(new Error(`Invalid Rollbridge control response from ${path}`, {cause: error}))
|
|
37
64
|
return
|
|
38
65
|
}
|
|
39
66
|
|
|
40
67
|
socket.end()
|
|
41
68
|
|
|
42
69
|
if (response.status === "error") {
|
|
43
|
-
|
|
70
|
+
settleReject(new Error(String(response.error || "Unknown Rollbridge error")))
|
|
44
71
|
} else {
|
|
45
|
-
|
|
72
|
+
settleResolve(response)
|
|
46
73
|
}
|
|
47
74
|
})
|
|
48
75
|
socket.write(`${JSON.stringify(command)}\n`)
|