rollbridge 0.1.60 → 0.1.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/changelog.d/20261109-path-filtered-builds.md +11 -0
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +26 -0
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +27 -0
- package/docs/config.md +2 -2
- package/docs/troubleshooting.md +5 -2
- package/docs/velocious.md +1 -1
- package/examples/tensorbuzz.com.js +1 -1
- package/package.json +1 -1
- package/src/config.js +13 -1
- package/tensorbuzz.yml +17 -0
- package/test/config-validation.test.js +49 -0
- package/test/owner-replacement.test.js +22 -2
- package/test/rollbridge.test.js +77 -17
- package/test/tensorbuzz-config.test.js +228 -0
package/README.md
CHANGED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
### Added
|
|
2
|
+
|
|
3
|
+
- `tensorbuzz.yml` declares a `paths` filter for the CI lane: `"**"` with a quoted
|
|
4
|
+
`!**/*.md` negation, so a delivery that changes only Markdown is recorded as a
|
|
5
|
+
terminal `skipped` build instead of installing the dependency tree and running
|
|
6
|
+
`npm run all-checks`. 31 of the repository's 114 tracked paths are Markdown and
|
|
7
|
+
no gate input is among them — `test/tensorbuzz-config.test.js` asserts that
|
|
8
|
+
partition over every tracked path, that every negation is quoted, and that the
|
|
9
|
+
catch-all stays first so a later rule cannot re-admit what a negation carved out.
|
|
10
|
+
`tensorbuzz.yml` itself, an unavailable or capped changed-file list, and a manual
|
|
11
|
+
rebuild still run the lane.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
### Changed
|
|
2
|
+
|
|
3
|
+
- `proxy.healthTimeoutMs` now defaults to **120000** instead of 30000, so a release
|
|
4
|
+
gets a boot budget above the measured production startup without every project
|
|
5
|
+
having to configure one. Measured on TensorBuzz production (server3) 2026-11-06:
|
|
6
|
+
release `20261106142602` started its `web` process at 14:33:39.205, was still
|
|
7
|
+
printing `attachment storage root migration` lines at 14:34:08.842, and was
|
|
8
|
+
SIGTERMed at 14:34:09.492 without ever logging a listening line — the app's own
|
|
9
|
+
boot alone exceeded the 30 000 ms deadline, after the dependency wait had already
|
|
10
|
+
consumed about 11 817 ms of it. Release `20261106144454` was killed 15 373 ms after
|
|
11
|
+
its `web` process started, before reaching those lines. 120 000 ms is the budget the
|
|
12
|
+
same release has published on since `config/rollbridge.production.js` set it on
|
|
13
|
+
2026-11-06.
|
|
14
|
+
- A process that configures no `health` block inherits that budget from
|
|
15
|
+
`proxy.healthTimeoutMs`; setting either level still wins over the default. The
|
|
16
|
+
documentation examples now show 120 000 rather than the value that failed.
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- A release whose app needs more than 30 seconds to boot publishes while an
|
|
21
|
+
incumbent generation is live. `test/rollbridge.test.js` now deploys a candidate
|
|
22
|
+
that answers `/ping` 32 seconds after launch on the default budget and asserts
|
|
23
|
+
`release candidate health ready` records `bootMs`, `healthGateMs`, and
|
|
24
|
+
`deadlineMs: 120000`, with the incumbent still serving `/release` during the boot.
|
|
25
|
+
The health gate is not weakened: a candidate that never becomes healthy still
|
|
26
|
+
fails the deploy when its deadline expires and the incumbent keeps serving.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
### Fixed
|
|
2
|
+
|
|
3
|
+
- `test/owner-replacement.test.js` "failed partial upgrade resumes an incumbent retired release
|
|
4
|
+
drain" no longer fails on a busy runner. Its whole-fixture budget was `5000ms` while the scenario
|
|
5
|
+
spawns six child processes (daemon, a `worker` and a proxied `web` for each of two releases, plus a
|
|
6
|
+
partial owner-replacement guardian) and then waits for the retired drain to resume, so its cost
|
|
7
|
+
tracked host contention instead of the behaviour under test. Measured in the same invocation:
|
|
8
|
+
1.457s at load average 8.3 and 7.105s at load average 24.5, both with the drain completing — the
|
|
9
|
+
cap sat between the two, which is why the spec reported `Error: Timed out after 5000ms` naming no
|
|
10
|
+
stage while nothing in the drain stalled. The budget is now `PARTIAL_DRAIN_FIXTURE_TIMEOUT_MS`
|
|
11
|
+
(20000ms, 2.8x the measured worst case) with the measurement cited at its declaration.
|
|
12
|
+
- The same spec now asserts the resumed drain in the retired release's own status record
|
|
13
|
+
(`drainStartedAt` and `stoppedAt`, ordered) instead of only the stdout line its wait is keyed on,
|
|
14
|
+
so a release cleared without draining its processes cannot pass. No product behavior changed and no
|
|
15
|
+
health gate was weakened: the waits stay event-gated, so a release that never resumes its drain
|
|
16
|
+
still fails.
|
|
17
|
+
- `test/rollbridge.test.js` "candidate activation quiesces the old jobs generation before a blocked
|
|
18
|
+
singleton replacement completes" no longer depends on inotify to reach its assertions. It awaited an
|
|
19
|
+
`fs.watch` event with no initial read and no fallback, so in a runtime whose filesystem delivers no
|
|
20
|
+
events it hung to the runner's ambient 60s default and named no stage. Measured here: a minimal
|
|
21
|
+
`fs.watch` receives 0 events for a file written by an external process, on overlayfs `/tmp` and on
|
|
22
|
+
ext3 `/data`, with `max_user_watches` at 1048576 — the container, not the product. The wait is now on
|
|
23
|
+
the durable marker the old generation writes (`SINGLETON_QUIET_MARKER_TIMEOUT_MS = 20000`, spec
|
|
24
|
+
measured at 1.740s), naming the marker path if it ever expires. The three assertions are unchanged
|
|
25
|
+
and now execute everywhere. The suite's other `fs.watch` helpers (`waitForState`,
|
|
26
|
+
`waitForLifecycleEvent`) already read the file before watching, which is why only this spec was
|
|
27
|
+
affected.
|
package/docs/config.md
CHANGED
|
@@ -77,7 +77,7 @@ to let a deploy group talk to the daemon.
|
|
|
77
77
|
| `proxy.port` | number | `8182` | Stable port Nginx (or another front end) points at. |
|
|
78
78
|
| `proxy.upstreamHost` | string | `proxy.host`, or `"127.0.0.1"` when `proxy.host` is `0.0.0.0`/`::` | Host Rollbridge uses for release health checks and proxy targets. |
|
|
79
79
|
| `proxy.healthPath` | string | `"/ping"` | Default health-check path for proxied processes. |
|
|
80
|
-
| `proxy.healthTimeoutMs` | number | `
|
|
80
|
+
| `proxy.healthTimeoutMs` | number | `120000` | Default boot budget for proxied processes: how long a release may take to answer its health path for the first time. Measured on TensorBuzz production (server3, 2026-11-06): the web app was still initialising when a `30000` deadline expired, so two deploys were killed mid-boot; `120000` is the budget the same release publishes on today. Waiting for a `dependsOn` service is not charged to it. |
|
|
81
81
|
| `proxy.drainTimeoutMs` | number | `60000` | How long to drain HTTP/WebSocket connections before stopping the retired proxied process. Expiry never stops an independently draining jobs generation. |
|
|
82
82
|
| `proxy.forceStopTimeoutMs` | number | `10000` | Default per-process graceful-stop timeout (`SIGTERM`, then `SIGKILL`). |
|
|
83
83
|
|
|
@@ -445,7 +445,7 @@ release). Set `health: false` to disable it.
|
|
|
445
445
|
| Field | Type | Default | Description |
|
|
446
446
|
| --- | --- | --- | --- |
|
|
447
447
|
| `health.path` | string | `proxy.healthPath` | HTTP path probed on the process's port. |
|
|
448
|
-
| `health.timeoutMs` | number | `proxy.healthTimeoutMs` | Total time to wait for the first healthy response. |
|
|
448
|
+
| `health.timeoutMs` | number | `proxy.healthTimeoutMs` | Total time to wait for the first healthy response, measured from the process launch. Only the app's own boot is charged to it — a `dependsOn` wait runs on `dependencyTimeoutMs` before this window opens. |
|
|
449
449
|
| `health.intervalMs` | number | `250` | Delay between probes. |
|
|
450
450
|
| `health.startDelayMs` | non-negative number | `0` | Wait this long after the process starts before the first probe (runs before the `timeoutMs` window). |
|
|
451
451
|
|
package/docs/troubleshooting.md
CHANGED
|
@@ -70,8 +70,11 @@ plus a `release process dependency ready` event shows the wait Rollbridge did on
|
|
|
70
70
|
the dependency's own budget.
|
|
71
71
|
|
|
72
72
|
**Fix.** Make the proxied command bind `{{port}}` and serve `health.path` with a
|
|
73
|
-
2xx status.
|
|
74
|
-
|
|
73
|
+
2xx status. The boot budget is `proxy.healthTimeoutMs` (`120000` by default), which
|
|
74
|
+
every process inherits unless it sets `health.timeoutMs` itself, and it measures only
|
|
75
|
+
the app's own boot. For an app that needs longer than the default, raise
|
|
76
|
+
`health.timeoutMs` above the measured boot time (or set `health.startDelayMs` so
|
|
77
|
+
probing begins after the app is up). For a
|
|
75
78
|
same-release service dependency, declare `dependsOn` (see
|
|
76
79
|
[`processes[].dependsOn`](config.md#processesdependson)) so the wait runs on
|
|
77
80
|
`dependencyTimeoutMs` instead of eating the boot budget.
|
package/docs/velocious.md
CHANGED
package/package.json
CHANGED
package/src/config.js
CHANGED
|
@@ -43,6 +43,18 @@ const DEFAULT_CONFIG_FILENAMES = ["rollbridge.js"]
|
|
|
43
43
|
// 30000ms per hop. 30000ms keeps that dependency budget, so the health deadline measures
|
|
44
44
|
// only the app boot.
|
|
45
45
|
const DEFAULT_DEPENDENCY_TIMEOUT_MS = 30000
|
|
46
|
+
// Budget for one process to answer its health path for the first time, measured from its
|
|
47
|
+
// launch. Measured on TensorBuzz production (server3, 2026-11-06): the 30000ms release
|
|
48
|
+
// health deadline killed two candidate web processes mid-boot. Release 20261106142602
|
|
49
|
+
// started web at 14:33:39.205, was still printing attachment storage root migration lines
|
|
50
|
+
// at 14:34:08.842, and got SIGTERM at 14:34:09.492 without ever logging a listening line,
|
|
51
|
+
// so the app's own boot alone exceeded 30000ms; release 20261106144454 was killed 15373ms
|
|
52
|
+
// after its web process started, before reaching those lines. 120000ms is the budget the
|
|
53
|
+
// same app has been published on since (config/rollbridge.production.js, merged
|
|
54
|
+
// 2026-11-06) and is made the default so a slow boot is not a trap every project has to
|
|
55
|
+
// know to configure. A candidate that never becomes healthy still fails the deploy at the
|
|
56
|
+
// deadline while the incumbent keeps serving; the budget only moves when the gate reports.
|
|
57
|
+
const DEFAULT_HEALTH_TIMEOUT_MS = 120000
|
|
46
58
|
const commonJsRequire = createRequire(import.meta.url)
|
|
47
59
|
|
|
48
60
|
/**
|
|
@@ -212,7 +224,7 @@ function normalizeProxy(source, issues) {
|
|
|
212
224
|
drainTimeoutMs: normalizeNumber(source.drainTimeoutMs, "proxy.drainTimeoutMs", issues, {default: 60000}),
|
|
213
225
|
forceStopTimeoutMs: normalizeNumber(source.forceStopTimeoutMs, "proxy.forceStopTimeoutMs", issues, {default: 10000}),
|
|
214
226
|
healthPath: normalizeString(source.healthPath, "proxy.healthPath", issues, {default: "/ping"}),
|
|
215
|
-
healthTimeoutMs: normalizeNumber(source.healthTimeoutMs, "proxy.healthTimeoutMs", issues, {default:
|
|
227
|
+
healthTimeoutMs: normalizeNumber(source.healthTimeoutMs, "proxy.healthTimeoutMs", issues, {default: DEFAULT_HEALTH_TIMEOUT_MS}),
|
|
216
228
|
host,
|
|
217
229
|
port: normalizeNumber(source.port, "proxy.port", issues, {default: 8182}),
|
|
218
230
|
upstreamHost: normalizeString(source.upstreamHost, "proxy.upstreamHost", issues, {default: defaultUpstreamHost(host)})
|
package/tensorbuzz.yml
CHANGED
|
@@ -1,3 +1,20 @@
|
|
|
1
|
+
# TensorBuzz build config for Rollbridge.
|
|
2
|
+
#
|
|
3
|
+
# `paths` (STOE-1179): this gate installs the dependency tree and runs typecheck, ESLint and the
|
|
4
|
+
# whole Velocious suite, and every one of those reads code, `examples/`, and the package/tooling
|
|
5
|
+
# manifests — no gate input is Markdown. A delivery that changes nothing but Markdown therefore
|
|
6
|
+
# has nothing for this lane to check, and TensorBuzz records it as a terminal `skipped` build
|
|
7
|
+
# instead of spending a container and ~3 minutes on it. `test/tensorbuzz-config.test.js` asserts
|
|
8
|
+
# the split against every tracked path so a later edit cannot carve a gate input out of the lane.
|
|
9
|
+
#
|
|
10
|
+
# Ordered rules, last match wins, so the negation has to come after `"**"`. Quote every negation:
|
|
11
|
+
# an unquoted `!x` is a YAML tag, and the config validator fails the whole file rather than
|
|
12
|
+
# filtering anything. The lane still runs whenever `tensorbuzz.yml` itself changes, whenever the
|
|
13
|
+
# changed-file list is missing, empty, or capped by the host, and on every manual rebuild.
|
|
14
|
+
paths:
|
|
15
|
+
- "**"
|
|
16
|
+
- "!**/*.md"
|
|
17
|
+
|
|
1
18
|
before_script:
|
|
2
19
|
- npm install
|
|
3
20
|
script:
|
|
@@ -587,6 +587,55 @@ test("validateConfig defaults health.startDelayMs to 0, accepts an override, and
|
|
|
587
587
|
expect(negative.issues.some((issue) => issue.message === "processes[0].health.startDelayMs must be a non-negative number")).toBeTruthy()
|
|
588
588
|
})
|
|
589
589
|
|
|
590
|
+
test("validateConfig gives an unconfigured proxied process the measured production boot budget", () => {
|
|
591
|
+
/**
|
|
592
|
+
* @param {Record<string, import("../src/json.js").JsonValue>} [processConfig] - Extra keys for the proxied process.
|
|
593
|
+
* @returns {import("../src/config.js").RollbridgeConfig} Normalized config.
|
|
594
|
+
*/
|
|
595
|
+
const normalized = (processConfig = {}) => validateConfig({
|
|
596
|
+
application: "demo",
|
|
597
|
+
control: {path: "/tmp/demo.sock"},
|
|
598
|
+
processes: [
|
|
599
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}, ...processConfig}
|
|
600
|
+
],
|
|
601
|
+
proxy: {host: "127.0.0.1", port: 8182}
|
|
602
|
+
}).config
|
|
603
|
+
|
|
604
|
+
const defaults = normalized()
|
|
605
|
+
|
|
606
|
+
expect(defaults.proxy.healthTimeoutMs).toBe(120000)
|
|
607
|
+
// The per-process gate inherits the release budget, so a project that configures no health
|
|
608
|
+
// timeout at all gets a deadline above the TensorBuzz boot measured on 2026-11-06 instead of
|
|
609
|
+
// the 30000ms that killed two production deploys.
|
|
610
|
+
expect(defaults.processes[0].health?.timeoutMs).toBe(120000)
|
|
611
|
+
expect(defaults.processes[0].health?.intervalMs).toBe(250)
|
|
612
|
+
|
|
613
|
+
const releaseBudget = normalized({health: {path: "/healthz"}})
|
|
614
|
+
|
|
615
|
+
expect(releaseBudget.proxy.healthTimeoutMs).toBe(120000)
|
|
616
|
+
expect(releaseBudget.processes[0].health?.path).toBe("/healthz")
|
|
617
|
+
expect(releaseBudget.processes[0].health?.timeoutMs).toBe(120000)
|
|
618
|
+
|
|
619
|
+
// Both configuration levels still win over the default: the release-level budget and the
|
|
620
|
+
// per-process budget, which is what a project with a known boot time sets.
|
|
621
|
+
const configuredProxy = validateConfig({
|
|
622
|
+
application: "demo",
|
|
623
|
+
control: {path: "/tmp/demo.sock"},
|
|
624
|
+
processes: [
|
|
625
|
+
{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
|
|
626
|
+
],
|
|
627
|
+
proxy: {healthTimeoutMs: 45000, host: "127.0.0.1", port: 8182}
|
|
628
|
+
}).config
|
|
629
|
+
|
|
630
|
+
expect(configuredProxy.proxy.healthTimeoutMs).toBe(45000)
|
|
631
|
+
expect(configuredProxy.processes[0].health?.timeoutMs).toBe(45000)
|
|
632
|
+
|
|
633
|
+
const perProcess = normalized({health: {path: "/ping", timeoutMs: 900}})
|
|
634
|
+
|
|
635
|
+
expect(perProcess.proxy.healthTimeoutMs).toBe(120000)
|
|
636
|
+
expect(perProcess.processes[0].health?.timeoutMs).toBe(900)
|
|
637
|
+
})
|
|
638
|
+
|
|
590
639
|
test("validateConfig defaults dependsOn and its own wait budget, and rejects dependencies it cannot wait on", () => {
|
|
591
640
|
/**
|
|
592
641
|
* @param {import("../src/json.js").JsonValue} processes - Processes under test.
|
|
@@ -22,6 +22,15 @@ const binPath = path.join(repoRoot, "bin", "rollbridge")
|
|
|
22
22
|
const dummyAppPath = path.join(repoRoot, "test", "fixtures", "dummy-app.js")
|
|
23
23
|
const legacyDaemonPath = path.join(repoRoot, "test", "fixtures", "pre-split3-daemon-runner.js")
|
|
24
24
|
const partialGuardianPath = path.join(repoRoot, "test", "fixtures", "partial-owner-replacement-process-guardian.js")
|
|
25
|
+
// Budget for the retired-drain fixture below. It spawns six child processes (a daemon, a worker and
|
|
26
|
+
// a proxied web process for each of two releases, plus a partial owner-replacement guardian) and then
|
|
27
|
+
// waits for the incumbent's retired drain to resume, so its cost tracks host contention, not the
|
|
28
|
+
// behaviour under test. Measured on the same machine in the same invocation: 1.457s at load average
|
|
29
|
+
// 8.3 and 7.105s at load average 24.5, both with the drain completing. The previous 5000ms cap sat
|
|
30
|
+
// between those two numbers, which is why the spec reddened on a busy runner while nothing in the
|
|
31
|
+
// drain actually stalled. 20000ms is 2.8x the measured worst case; the wait stays event-gated, so a
|
|
32
|
+
// release that never resumes its drain still fails, and now fails naming the release record.
|
|
33
|
+
const PARTIAL_DRAIN_FIXTURE_TIMEOUT_MS = 20000
|
|
25
34
|
|
|
26
35
|
test("partial owner-replacement guardian crosses the authenticated legacy bridge before retired-owner commit", async () => {
|
|
27
36
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-"))
|
|
@@ -324,7 +333,7 @@ test("partial guardian replacement persists the coordinator only after ownership
|
|
|
324
333
|
}
|
|
325
334
|
})
|
|
326
335
|
|
|
327
|
-
test("failed partial upgrade resumes an incumbent retired release drain", {timeoutMs:
|
|
336
|
+
test("failed partial upgrade resumes an incumbent retired release drain", {timeoutMs: PARTIAL_DRAIN_FIXTURE_TIMEOUT_MS}, async () => {
|
|
328
337
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-drain-"))
|
|
329
338
|
const socketPath = path.join(root, "rollbridge.sock")
|
|
330
339
|
const statePath = path.join(root, "state.json")
|
|
@@ -382,7 +391,18 @@ test("failed partial upgrade resumes an incumbent retired release drain", {timeo
|
|
|
382
391
|
retainedConnection.destroy()
|
|
383
392
|
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
384
393
|
await releaseDrained
|
|
385
|
-
|
|
394
|
+
const drainedStatus = await sendControlCommand({command: {command: "status"}, path: socketPath})
|
|
395
|
+
|
|
396
|
+
expect(releaseState(drainedStatus, "v1")).toBe("stopped")
|
|
397
|
+
// The resumed drain must also be visible in the retired release's own record, not only as the
|
|
398
|
+
// stdout line the wait is keyed on: a release cleared without draining its processes leaves no
|
|
399
|
+
// drainStartedAt/stoppedAt pair behind.
|
|
400
|
+
const drainedRelease = /** @type {{drainStartedAt?: string, releaseId: string, stoppedAt?: string}[]} */ (drainedStatus.releases)
|
|
401
|
+
.find(({releaseId}) => releaseId === "v1")
|
|
402
|
+
|
|
403
|
+
expect(typeof drainedRelease?.drainStartedAt).toBe("string")
|
|
404
|
+
expect(typeof drainedRelease?.stoppedAt).toBe("string")
|
|
405
|
+
expect(Date.parse(/** @type {string} */ (drainedRelease?.stoppedAt))).toBeGreaterThanOrEqual(Date.parse(/** @type {string} */ (drainedRelease?.drainStartedAt)))
|
|
386
406
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
|
|
387
407
|
|
|
388
408
|
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
package/test/rollbridge.test.js
CHANGED
|
@@ -272,6 +272,47 @@ test("the dependency wait is charged to its own budget, not to the proxied boot
|
|
|
272
272
|
}
|
|
273
273
|
})
|
|
274
274
|
|
|
275
|
+
test("a release whose app boots past the old 30000ms window publishes on the default boot budget while the incumbent serves", {timeoutMs: 180000}, async () => {
|
|
276
|
+
const fixture = await createFixture()
|
|
277
|
+
// Measured on TensorBuzz production (server3, 2026-11-06): the candidate web process was
|
|
278
|
+
// still initialising when the 30000ms release health deadline expired and was killed at
|
|
279
|
+
// 30287ms without ever logging a listening line. 32000ms reproduces a boot the old default
|
|
280
|
+
// killed; nothing here configures a health budget, so the release runs on the default.
|
|
281
|
+
const lateBootMs = 32000
|
|
282
|
+
const incumbentConfig = deferredBootConfig(fixture, {dependencyTimeoutMs: 30000, serviceBootMs: 0, webBootMs: 0})
|
|
283
|
+
const configPath = await writeConfigFile(incumbentConfig, fixture.root)
|
|
284
|
+
/** @type {Array<{data?: Record<string, import("../src/json.js").JsonValue>, message: string}>} */
|
|
285
|
+
const logs = []
|
|
286
|
+
const daemon = new RollbridgeDaemon({config: incumbentConfig, configPath, logger: (message, data = {}) => logs.push({data, message})})
|
|
287
|
+
|
|
288
|
+
await daemon.start()
|
|
289
|
+
|
|
290
|
+
try {
|
|
291
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
292
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
293
|
+
|
|
294
|
+
await writeConfigFile(deferredBootConfig(fixture, {dependencyTimeoutMs: 30000, serviceBootMs: 0, webBootMs: lateBootMs}), fixture.root)
|
|
295
|
+
const deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
296
|
+
|
|
297
|
+
// The candidate is still booting a second later: the incumbent keeps serving traffic.
|
|
298
|
+
await new Promise((resolve) => setTimeout(resolve, 1000))
|
|
299
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
300
|
+
await deployment
|
|
301
|
+
|
|
302
|
+
const ready = logs.find((entry) => entry.message === "release candidate health ready" && entry.data?.releaseId === "v2")
|
|
303
|
+
|
|
304
|
+
if (!ready?.data) throw new Error("expected the v2 boot time to be recorded in the event log")
|
|
305
|
+
expect(Number(ready.data.bootMs)).toBeGreaterThanOrEqual(lateBootMs)
|
|
306
|
+
expect(Number(ready.data.healthGateMs)).toBeGreaterThanOrEqual(lateBootMs)
|
|
307
|
+
expect(Number(ready.data.deadlineMs)).toBe(120000)
|
|
308
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
309
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
310
|
+
} finally {
|
|
311
|
+
await daemon.shutdown()
|
|
312
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
313
|
+
}
|
|
314
|
+
})
|
|
315
|
+
|
|
275
316
|
test("a dependency that never opens fails the deploy on its own budget and keeps the incumbent serving", async () => {
|
|
276
317
|
const fixture = await createFixture()
|
|
277
318
|
const initialConfig = deferredBootConfig(fixture, {dependencyTimeoutMs: 5000, serviceBootMs: 0, webBootMs: 0, webHealthTimeoutMs: 900})
|
|
@@ -362,6 +403,13 @@ test("singleton processes restart without overlap during deploy", async () => {
|
|
|
362
403
|
}
|
|
363
404
|
})
|
|
364
405
|
|
|
406
|
+
// Budget for waiting on the old jobs generation's quiet marker in the spec below. Measured with the
|
|
407
|
+
// marker read directly: the spec completes in 1.740s on this host, so this is not a boot budget but a
|
|
408
|
+
// generous ceiling on a handoff that already happened; the number only decides how long a genuinely
|
|
409
|
+
// stuck quiescence takes to report itself, and it now reports the marker path instead of the runner's
|
|
410
|
+
// ambient 60s default.
|
|
411
|
+
const SINGLETON_QUIET_MARKER_TIMEOUT_MS = 20000
|
|
412
|
+
|
|
365
413
|
test("candidate activation quiesces the old jobs generation before a blocked singleton replacement completes", async () => {
|
|
366
414
|
const fixture = await createFixture({handoffService: true, handoffServiceQuiet: true, includeSingleton: true, webDependsOnService: true})
|
|
367
415
|
const singletonGatePath = path.join(fixture.root, "singleton-replacement.gate")
|
|
@@ -384,24 +432,27 @@ test("candidate activation quiesces the old jobs generation before a blocked sin
|
|
|
384
432
|
try {
|
|
385
433
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
386
434
|
|
|
387
|
-
const abortController = new AbortController()
|
|
388
|
-
const changes = fs.watch(fixture.root, {signal: abortController.signal})
|
|
389
|
-
const singletonReplacementBlocked = (async () => {
|
|
390
|
-
try {
|
|
391
|
-
for await (const change of changes) {
|
|
392
|
-
if (change.filename === path.basename(singletonQuietPath)) return
|
|
393
|
-
}
|
|
394
|
-
} finally {
|
|
395
|
-
abortController.abort()
|
|
396
|
-
}
|
|
397
|
-
})()
|
|
398
|
-
|
|
399
435
|
deployPromise = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
400
436
|
let deploySettled = false
|
|
401
437
|
|
|
402
438
|
void deployPromise.then(() => { deploySettled = true }, () => { deploySettled = true })
|
|
403
|
-
|
|
404
|
-
|
|
439
|
+
|
|
440
|
+
// Wait on the durable marker the old jobs generation writes, not on an inotify event for it.
|
|
441
|
+
// fs.watch surfaces nothing in runtimes whose filesystem does not deliver events (measured in
|
|
442
|
+
// this container: a minimal fs.watch receives 0 events for a file written by an external
|
|
443
|
+
// process, on both overlayfs /tmp and ext3 /data, with max_user_watches at 1048576), and an
|
|
444
|
+
// unguarded event loop turns that into a bare runner timeout naming no stage. Reading the
|
|
445
|
+
// marker is the same observation with a fallback the event never had, and the three assertions
|
|
446
|
+
// below are unchanged: candidate traffic active, deploy still pending, old generation quiesced.
|
|
447
|
+
const quietDeadline = Date.now() + SINGLETON_QUIET_MARKER_TIMEOUT_MS
|
|
448
|
+
let sawQuietMarker = false
|
|
449
|
+
|
|
450
|
+
while (Date.now() < quietDeadline && !sawQuietMarker) {
|
|
451
|
+
sawQuietMarker = await fs.readFile(singletonQuietPath, "utf8").then((content) => content.includes("quiet"), () => false)
|
|
452
|
+
if (!sawQuietMarker) await new Promise((resolve) => setTimeout(resolve, 10))
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
if (!sawQuietMarker) throw new Error(`Old jobs generation never wrote its quiet marker within ${SINGLETON_QUIET_MARKER_TIMEOUT_MS}ms: ${singletonQuietPath}`)
|
|
405
456
|
|
|
406
457
|
// Candidate traffic must already be active.
|
|
407
458
|
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
@@ -2294,16 +2345,23 @@ async function createFixture(options = {}) {
|
|
|
2294
2345
|
|
|
2295
2346
|
/**
|
|
2296
2347
|
* Config where both processes run the deferred-boot fixture, so a measured boot budget can be
|
|
2297
|
-
* compared against a dependency wait that must not be charged to it.
|
|
2348
|
+
* compared against a dependency wait that must not be charged to it. Omitting
|
|
2349
|
+
* webHealthTimeoutMs leaves both the per-process and the release health budget unconfigured, so
|
|
2350
|
+
* the normalized config carries the default boot budget instead of a test-chosen one.
|
|
2298
2351
|
* @param {{config: import("../src/config.js").RollbridgeConfig, root: string}} fixture - Fixture.
|
|
2299
|
-
* @param {{dependencyTimeoutMs: number, serviceBootMs: number, webBootMs: number, webHealthTimeoutMs
|
|
2352
|
+
* @param {{dependencyTimeoutMs: number, serviceBootMs: number, webBootMs: number, webHealthTimeoutMs?: number}} options - Budgets.
|
|
2300
2353
|
* @returns {import("../src/config.js").RollbridgeConfig} Config for the deferred-boot processes.
|
|
2301
2354
|
*/
|
|
2302
2355
|
function deferredBootConfig(fixture, options) {
|
|
2303
2356
|
const appCommand = `${JSON.stringify(process.execPath)} ${JSON.stringify(deferredBootAppPath)}`
|
|
2357
|
+
/** @type {Partial<import("../src/config.js").ProxyConfig>} */
|
|
2358
|
+
const proxyWithoutHealthBudget = structuredClone(fixture.config.proxy)
|
|
2359
|
+
|
|
2360
|
+
delete proxyWithoutHealthBudget.healthTimeoutMs
|
|
2304
2361
|
|
|
2305
2362
|
return normalizeConfig({
|
|
2306
2363
|
...fixture.config,
|
|
2364
|
+
...(options.webHealthTimeoutMs === undefined ? {proxy: proxyWithoutHealthBudget} : {}),
|
|
2307
2365
|
processes: [
|
|
2308
2366
|
{
|
|
2309
2367
|
command: appCommand,
|
|
@@ -2318,7 +2376,9 @@ function deferredBootConfig(fixture, options) {
|
|
|
2318
2376
|
dependsOn: ["beacon"],
|
|
2319
2377
|
dependencyTimeoutMs: options.dependencyTimeoutMs,
|
|
2320
2378
|
env: {ROLLBRIDGE_BOOT_MS: String(options.webBootMs), ROLLBRIDGE_DEPENDENCY_PORT: "{{ports.beacon}}"},
|
|
2321
|
-
|
|
2379
|
+
...(options.webHealthTimeoutMs === undefined
|
|
2380
|
+
? {}
|
|
2381
|
+
: {health: {intervalMs: 50, path: "/ping", timeoutMs: options.webHealthTimeoutMs}}),
|
|
2322
2382
|
id: "web",
|
|
2323
2383
|
policy: "proxied",
|
|
2324
2384
|
port: {from: 0, to: 0}
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {execFile} from "node:child_process"
|
|
4
|
+
import fs from "node:fs/promises"
|
|
5
|
+
import path from "node:path"
|
|
6
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
7
|
+
import {fileURLToPath} from "node:url"
|
|
8
|
+
import {promisify} from "node:util"
|
|
9
|
+
|
|
10
|
+
describe("tensorbuzz-config", () => {
|
|
11
|
+
|
|
12
|
+
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
|
|
13
|
+
const execFileAsync = promisify(execFile)
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @typedef {object} PathRule
|
|
17
|
+
* @property {boolean} negated True when the rule carves the paths it matches out of the lane.
|
|
18
|
+
* @property {string} pattern Rule body without its YAML quotes and without the `!` prefix.
|
|
19
|
+
* @property {RegExp} regexp Matcher compiled from the rule body.
|
|
20
|
+
* @property {string} raw Rule exactly as declared in tensorbuzz.yml, quotes included.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* @typedef {object} PathDecision
|
|
25
|
+
* @property {string} filePath Repo-relative path the decision was taken on.
|
|
26
|
+
* @property {string} matchedPattern Last rule that matched, empty when no rule matched.
|
|
27
|
+
* @property {boolean} skipped True when the last matching rule negates the path.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* @typedef {object} GateInput
|
|
32
|
+
* @property {string[]} readPaths Paths `npm run all-checks` opens: the suite and the linters read
|
|
33
|
+
* these, so carving any of them out of the lane would skip a check that the change can break.
|
|
34
|
+
* @property {string[]} skippedPaths Paths the lane must not need: Markdown the gate never opens.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
const GATE_INPUTS = [
|
|
38
|
+
"src/daemon.js",
|
|
39
|
+
"src/process-guardian.js",
|
|
40
|
+
"test/rollbridge.test.js",
|
|
41
|
+
"test/fixtures/dummy-app.js",
|
|
42
|
+
"bin/rollbridge",
|
|
43
|
+
"examples/tensorbuzz.com.js",
|
|
44
|
+
"package.json",
|
|
45
|
+
"package-lock.json",
|
|
46
|
+
"tsconfig.json",
|
|
47
|
+
"eslint.config.js",
|
|
48
|
+
".npmignore",
|
|
49
|
+
"LICENSE",
|
|
50
|
+
"compose.yml",
|
|
51
|
+
"tensorbuzz.yml"
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
const MARKDOWN_PATHS = [
|
|
55
|
+
"README.md",
|
|
56
|
+
"AGENTS.md",
|
|
57
|
+
"docs/tensorbuzz-runbook.md",
|
|
58
|
+
"docs/releasing.md",
|
|
59
|
+
"changelog.d/20261109-path-filtered-builds.md"
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Reads the root `paths` rules out of tensorbuzz.yml.
|
|
64
|
+
* @param {string} configText Full tensorbuzz.yml contents.
|
|
65
|
+
* @returns {PathRule[]} Rules in declaration order.
|
|
66
|
+
*/
|
|
67
|
+
function readPathRules(configText) {
|
|
68
|
+
const lines = configText.split("\n")
|
|
69
|
+
const start = lines.findIndex((line) => line.trim() === "paths:")
|
|
70
|
+
|
|
71
|
+
if (start < 0) return []
|
|
72
|
+
|
|
73
|
+
/** @type {PathRule[]} */
|
|
74
|
+
const rules = []
|
|
75
|
+
|
|
76
|
+
for (const line of lines.slice(start + 1)) {
|
|
77
|
+
const entry = line.trim()
|
|
78
|
+
|
|
79
|
+
if (!entry.startsWith("- ")) break
|
|
80
|
+
if (!entry) continue
|
|
81
|
+
|
|
82
|
+
const raw = entry.slice(2).trim()
|
|
83
|
+
const quoted = raw.startsWith("\"") && raw.endsWith("\"") && raw.length > 1
|
|
84
|
+
const body = quoted ? raw.slice(1, -1) : raw
|
|
85
|
+
const negated = body.startsWith("!")
|
|
86
|
+
|
|
87
|
+
rules.push({
|
|
88
|
+
negated,
|
|
89
|
+
pattern: negated ? body.slice(1) : body,
|
|
90
|
+
raw,
|
|
91
|
+
regexp: compileRuleRegExp(negated ? body.slice(1) : body)
|
|
92
|
+
})
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return rules
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Compiles a `paths` glob the way TensorBuzz does, for the syntax this config declares: `**`
|
|
100
|
+
* crosses segments, `*` stays inside one segment, a rule containing `/` is anchored at the repo
|
|
101
|
+
* root while a bare name matches at any depth, and a matched directory covers everything below
|
|
102
|
+
* it. The `supported glob syntax` test keeps the config inside that subset.
|
|
103
|
+
* @param {string} pattern Rule body without the `!` prefix.
|
|
104
|
+
* @returns {RegExp} Anchored matcher for one repo-relative path.
|
|
105
|
+
*/
|
|
106
|
+
function compileRuleRegExp(pattern) {
|
|
107
|
+
const body = pattern.replace(/^\/+/, "").replace(/\/+$/, "")
|
|
108
|
+
const segments = body.split("/")
|
|
109
|
+
let source = body.includes("/") ? "" : "(?:[^/]+/)*"
|
|
110
|
+
|
|
111
|
+
for (let index = 0; index < segments.length; index += 1) {
|
|
112
|
+
const segment = segments[index]
|
|
113
|
+
const isLast = index === segments.length - 1
|
|
114
|
+
|
|
115
|
+
if (segment === "**") {
|
|
116
|
+
source += isLast ? ".*" : "(?:.*/)?"
|
|
117
|
+
|
|
118
|
+
continue
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
source += segment.split("*").map(escapeForRegExp).join("[^/]*")
|
|
122
|
+
|
|
123
|
+
if (!isLast) source += "/"
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (!body.endsWith("**")) source += "(?:/.*)?"
|
|
127
|
+
|
|
128
|
+
return new RegExp(`^${source}$`)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* @param {string} text Literal text to make regexp-safe.
|
|
133
|
+
* @returns {string} Escaped text.
|
|
134
|
+
*/
|
|
135
|
+
function escapeForRegExp(text) {
|
|
136
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Applies ordered rules to one path; the last match wins, exactly like TensorBuzz.
|
|
141
|
+
* @param {{filePath: string, rules: PathRule[]}} args Path and the declared rules.
|
|
142
|
+
* @returns {PathDecision} Whether the lane runs for that path.
|
|
143
|
+
*/
|
|
144
|
+
function decidePath({filePath, rules}) {
|
|
145
|
+
/** @type {PathRule | null} */
|
|
146
|
+
let lastMatch = null
|
|
147
|
+
|
|
148
|
+
for (const rule of rules) {
|
|
149
|
+
if (rule.regexp.test(filePath)) lastMatch = rule
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
return {
|
|
153
|
+
filePath,
|
|
154
|
+
matchedPattern: lastMatch ? lastMatch.pattern : "",
|
|
155
|
+
skipped: lastMatch !== null && lastMatch.negated
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* @returns {Promise<PathRule[]>} Rules parsed from the repository's tensorbuzz.yml.
|
|
161
|
+
*/
|
|
162
|
+
async function declaredPathRules() {
|
|
163
|
+
return readPathRules(await fs.readFile(path.join(repoRoot, "tensorbuzz.yml"), "utf8"))
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
test("tensorbuzz.yml declares a paths block that defaults to running the lane", async () => {
|
|
167
|
+
const rules = await declaredPathRules()
|
|
168
|
+
|
|
169
|
+
expect(rules.length >= 2).toBeTruthy()
|
|
170
|
+
expect(rules[0].pattern).toBe("**")
|
|
171
|
+
expect(rules[0].negated).toBe(false)
|
|
172
|
+
|
|
173
|
+
const firstNegation = rules.findIndex((rule) => rule.negated)
|
|
174
|
+
|
|
175
|
+
expect(firstNegation > 0).toBeTruthy()
|
|
176
|
+
|
|
177
|
+
// A positive rule after a negation would re-admit every path the negation carved out, because
|
|
178
|
+
// the last match wins.
|
|
179
|
+
expect(rules.slice(firstNegation).filter((rule) => !rule.negated).map((rule) => rule.raw)).toEqual([])
|
|
180
|
+
})
|
|
181
|
+
|
|
182
|
+
test("every paths rule is a quoted glob the config validator accepts", async () => {
|
|
183
|
+
const rules = await declaredPathRules()
|
|
184
|
+
|
|
185
|
+
// An unquoted `!x` is a YAML tag, not a string, and the validator rejects the whole config.
|
|
186
|
+
expect(rules.map((rule) => rule.raw).filter((raw) => !raw.startsWith("\""))).toEqual([])
|
|
187
|
+
expect(rules.map((rule) => rule.pattern).filter((pattern) => !/^[A-Za-z0-9_./*-]+$/.test(pattern))).toEqual([])
|
|
188
|
+
expect(rules.map((rule) => rule.pattern).filter((pattern) => pattern.length < 1)).toEqual([])
|
|
189
|
+
})
|
|
190
|
+
|
|
191
|
+
test("a delivery that changes only Markdown is filtered out of the lane", async () => {
|
|
192
|
+
const rules = await declaredPathRules()
|
|
193
|
+
|
|
194
|
+
for (const filePath of MARKDOWN_PATHS) {
|
|
195
|
+
const decision = decidePath({filePath, rules})
|
|
196
|
+
|
|
197
|
+
expect({context: filePath, skipped: decision.skipped}).toEqual({context: filePath, skipped: true})
|
|
198
|
+
}
|
|
199
|
+
})
|
|
200
|
+
|
|
201
|
+
test("every path the gate reads stays inside the lane", async () => {
|
|
202
|
+
const rules = await declaredPathRules()
|
|
203
|
+
|
|
204
|
+
for (const filePath of GATE_INPUTS) {
|
|
205
|
+
const decision = decidePath({filePath, rules})
|
|
206
|
+
|
|
207
|
+
expect({context: filePath, matchedPattern: decision.matchedPattern, skipped: decision.skipped})
|
|
208
|
+
.toEqual({context: filePath, matchedPattern: "**", skipped: false})
|
|
209
|
+
}
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
test("the declared rules partition every tracked path between the lane and the skip set", async () => {
|
|
213
|
+
const rules = await declaredPathRules()
|
|
214
|
+
const {stdout} = await execFileAsync("git", ["ls-files"], {cwd: repoRoot})
|
|
215
|
+
const trackedPaths = stdout.split("\n").filter((line) => line.length > 0)
|
|
216
|
+
|
|
217
|
+
const skipped = trackedPaths.filter((filePath) => decidePath({filePath, rules}).skipped)
|
|
218
|
+
const runs = trackedPaths.filter((filePath) => !decidePath({filePath, rules}).skipped)
|
|
219
|
+
|
|
220
|
+
expect(skipped.length > 0).toBeTruthy()
|
|
221
|
+
expect(runs.length > 0).toBeTruthy()
|
|
222
|
+
|
|
223
|
+
// Only Markdown may be carved out: any other tracked path is a gate input the lane must run for.
|
|
224
|
+
expect(skipped.filter((filePath) => !filePath.endsWith(".md"))).toEqual([])
|
|
225
|
+
expect(runs.filter((filePath) => filePath.endsWith(".md"))).toEqual([])
|
|
226
|
+
})
|
|
227
|
+
|
|
228
|
+
})
|