rollbridge 0.1.70 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -63
- package/TODO.md +0 -13
- package/docs/cli.md +3 -128
- package/docs/config.md +6 -177
- package/docs/troubleshooting.md +6 -97
- package/docs/velocious.md +1 -1
- package/docs/workers.md +0 -11
- package/examples/tensorbuzz.com.js +2 -3
- package/package.json +1 -1
- package/src/cli.js +2 -68
- package/src/config.js +8 -347
- package/src/daemon.js +65 -387
- package/src/doctor.js +0 -74
- package/src/guardian-client.js +8 -67
- package/src/health.js +8 -99
- package/src/managed-process.js +16 -197
- package/src/process-guardian.js +22 -212
- package/src/process-memory.js +2 -62
- package/src/release-group.js +37 -435
- package/tensorbuzz.yml +1 -20
- package/test/completion.test.js +2 -5
- package/test/config-validation.test.js +5 -135
- package/test/guardian-client.test.js +5 -119
- package/test/health.test.js +1 -74
- package/test/managed-process.test.js +1 -260
- package/test/owner-recovery.test.js +21 -125
- package/test/owner-replacement.test.js +14 -175
- package/test/process-memory.test.js +1 -47
- package/test/release-group.test.js +2 -408
- package/test/rollbridge.test.js +72 -737
- package/test/shutdown-completion.test.js +0 -52
- package/test/support/process.js +1 -3
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
- package/changelog.d/20261104-compose-service-units.md +0 -21
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
- package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
- package/changelog.d/20261109-path-filtered-builds.md +0 -11
- package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
- package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
- package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
- package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
- package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
- package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
- package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
- package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
- package/changelog.d/20261116-guardian-version-handoff.md +0 -37
- package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
- package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
- package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
- package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
- package/docs/compose-services.md +0 -222
- package/src/compose-unit.js +0 -725
- package/src/drain-policy.js +0 -252
- package/src/process-order.js +0 -26
- package/test/compose-config.test.js +0 -160
- package/test/compose-docker.test.js +0 -297
- package/test/compose-unit.test.js +0 -415
- package/test/drain-policy.test.js +0 -328
- package/test/fixtures/deferred-boot-app.js +0 -106
- package/test/fixtures/gated-app.js +0 -50
- package/test/tensorbuzz-config.test.js +0 -228
package/tensorbuzz.yml
CHANGED
|
@@ -1,23 +1,4 @@
|
|
|
1
|
-
# TensorBuzz build config for Rollbridge.
|
|
2
|
-
#
|
|
3
|
-
# `paths` (STOE-1179): this gate installs the dependency tree and runs typecheck, ESLint and the
|
|
4
|
-
# whole Velocious suite, and every one of those reads code, `examples/`, and the package/tooling
|
|
5
|
-
# manifests — no gate input is Markdown. A delivery that changes nothing but Markdown therefore
|
|
6
|
-
# has nothing for this lane to check, and TensorBuzz records it as a terminal `skipped` build
|
|
7
|
-
# instead of spending a container and ~3 minutes on it. `test/tensorbuzz-config.test.js` asserts
|
|
8
|
-
# the split against every tracked path so a later edit cannot carve a gate input out of the lane.
|
|
9
|
-
#
|
|
10
|
-
# Ordered rules, last match wins, so the negation has to come after `"**"`. Quote every negation:
|
|
11
|
-
# an unquoted `!x` is a YAML tag, and the config validator fails the whole file rather than
|
|
12
|
-
# filtering anything. The lane still runs whenever `tensorbuzz.yml` itself changes, whenever the
|
|
13
|
-
# changed-file list is missing, empty, or capped by the host, and on every manual rebuild.
|
|
14
|
-
paths:
|
|
15
|
-
- "**"
|
|
16
|
-
- "!**/*.md"
|
|
17
|
-
|
|
18
1
|
before_script:
|
|
19
2
|
- npm install
|
|
20
3
|
script:
|
|
21
|
-
|
|
22
|
-
# the TensorBuzz runner's workspace TMPDIR is deliberately long.
|
|
23
|
-
- TMPDIR=/tmp npm run all-checks
|
|
4
|
+
- npm run all-checks
|
package/test/completion.test.js
CHANGED
|
@@ -42,12 +42,10 @@ test("completion bash prints a sourceable script with commands and option flags"
|
|
|
42
42
|
|
|
43
43
|
expect(code).not.toBe(1)
|
|
44
44
|
expect(output).toMatch(/complete -F _rollbridge rollbridge/)
|
|
45
|
-
expect(output).toMatch(/compgen -W "daemon deploy rollback recover-generation-transition
|
|
45
|
+
expect(output).toMatch(/compgen -W "daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion"/)
|
|
46
46
|
// A command's own options are completed after the command.
|
|
47
47
|
expect(output).toMatch(/deploy\)\n\s+opts="[^"]*--release-path[^"]*"/)
|
|
48
48
|
expect(output).toMatch(/recover-generation-transition\)\n\s+opts="--config --release-path --release-id --revision --previous-release-id --accept-retired-incumbent"/)
|
|
49
|
-
expect(output).toMatch(/compose-roll\)\n\s+opts="--config --unit --image"/)
|
|
50
|
-
expect(output).toMatch(/compose-status\)\n\s+opts="--config --unit"/)
|
|
51
49
|
expect(output).toMatch(/ensure-daemon\)\n\s+opts="[^"]*--daemon-runtime-path[^"]*"/)
|
|
52
50
|
expect(output).toMatch(/restart\)\n\s+opts="[^"]*--policy[^"]*"/)
|
|
53
51
|
})
|
|
@@ -57,9 +55,8 @@ test("completion zsh prints a #compdef script with per-command options", async (
|
|
|
57
55
|
|
|
58
56
|
expect(output).toMatch(/^#compdef rollbridge/)
|
|
59
57
|
expect(output).toMatch(/compdef _rollbridge rollbridge/)
|
|
60
|
-
expect(output).toMatch(/commands=\(daemon deploy rollback recover-generation-transition
|
|
58
|
+
expect(output).toMatch(/commands=\(daemon deploy rollback recover-generation-transition ensure-daemon status stop restart shutdown validate doctor logs events predeploy-cleanup recover completion\)/)
|
|
61
59
|
expect(output).toMatch(/recover-generation-transition\) compadd -- --config --release-path --release-id --revision --previous-release-id --accept-retired-incumbent/)
|
|
62
|
-
expect(output).toMatch(/compose-rollback\) compadd -- --config --unit --image/)
|
|
63
60
|
expect(output).toMatch(/events\) compadd -- [^\n]*--limit/)
|
|
64
61
|
})
|
|
65
62
|
|
|
@@ -143,9 +143,7 @@ test("validateConfig defaults the restart policy, accepts overrides, and rejects
|
|
|
143
143
|
const defaulted = validateRestart(undefined)
|
|
144
144
|
|
|
145
145
|
expect(defaulted.issues).toEqual([])
|
|
146
|
-
|
|
147
|
-
// that crossed the guardian's JSON transfer is still equal to a freshly loaded one.
|
|
148
|
-
expect(defaulted.config.processes[0].restart).toEqual({backoffFactor: 1, maxDelayMs: 0, windowMs: 0})
|
|
146
|
+
expect(defaulted.config.processes[0].restart).toEqual({backoffFactor: 1, maxDelayMs: 0, maxRestarts: undefined, windowMs: 0})
|
|
149
147
|
|
|
150
148
|
const custom = validateRestart({backoffFactor: 2, maxDelayMs: 30000, maxRestarts: 5, windowMs: 60000})
|
|
151
149
|
|
|
@@ -589,108 +587,6 @@ test("validateConfig defaults health.startDelayMs to 0, accepts an override, and
|
|
|
589
587
|
expect(negative.issues.some((issue) => issue.message === "processes[0].health.startDelayMs must be a non-negative number")).toBeTruthy()
|
|
590
588
|
})
|
|
591
589
|
|
|
592
|
-
test("validateConfig gives an unconfigured proxied process the measured production boot budget", () => {
|
|
593
|
-
/**
|
|
594
|
-
* @param {Record<string, import("../src/json.js").JsonValue>} [processConfig] - Extra keys for the proxied process.
|
|
595
|
-
* @returns {import("../src/config.js").RollbridgeConfig} Normalized config.
|
|
596
|
-
*/
|
|
597
|
-
const normalized = (processConfig = {}) => validateConfig({
|
|
598
|
-
application: "demo",
|
|
599
|
-
control: {path: "/tmp/demo.sock"},
|
|
600
|
-
processes: [
|
|
601
|
-
{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}, ...processConfig}
|
|
602
|
-
],
|
|
603
|
-
proxy: {host: "127.0.0.1", port: 8182}
|
|
604
|
-
}).config
|
|
605
|
-
|
|
606
|
-
const defaults = normalized()
|
|
607
|
-
|
|
608
|
-
expect(defaults.proxy.healthTimeoutMs).toBe(120000)
|
|
609
|
-
// The per-process gate inherits the release budget, so a project that configures no health
|
|
610
|
-
// timeout at all gets a deadline above the TensorBuzz boot measured on 2026-11-06 instead of
|
|
611
|
-
// the 30000ms that killed two production deploys.
|
|
612
|
-
expect(defaults.processes[0].health?.timeoutMs).toBe(120000)
|
|
613
|
-
expect(defaults.processes[0].health?.intervalMs).toBe(250)
|
|
614
|
-
|
|
615
|
-
const releaseBudget = normalized({health: {path: "/healthz"}})
|
|
616
|
-
|
|
617
|
-
expect(releaseBudget.proxy.healthTimeoutMs).toBe(120000)
|
|
618
|
-
expect(releaseBudget.processes[0].health?.path).toBe("/healthz")
|
|
619
|
-
expect(releaseBudget.processes[0].health?.timeoutMs).toBe(120000)
|
|
620
|
-
|
|
621
|
-
// Both configuration levels still win over the default: the release-level budget and the
|
|
622
|
-
// per-process budget, which is what a project with a known boot time sets.
|
|
623
|
-
const configuredProxy = validateConfig({
|
|
624
|
-
application: "demo",
|
|
625
|
-
control: {path: "/tmp/demo.sock"},
|
|
626
|
-
processes: [
|
|
627
|
-
{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
|
|
628
|
-
],
|
|
629
|
-
proxy: {healthTimeoutMs: 45000, host: "127.0.0.1", port: 8182}
|
|
630
|
-
}).config
|
|
631
|
-
|
|
632
|
-
expect(configuredProxy.proxy.healthTimeoutMs).toBe(45000)
|
|
633
|
-
expect(configuredProxy.processes[0].health?.timeoutMs).toBe(45000)
|
|
634
|
-
|
|
635
|
-
const perProcess = normalized({health: {path: "/ping", timeoutMs: 900}})
|
|
636
|
-
|
|
637
|
-
expect(perProcess.proxy.healthTimeoutMs).toBe(120000)
|
|
638
|
-
expect(perProcess.processes[0].health?.timeoutMs).toBe(900)
|
|
639
|
-
})
|
|
640
|
-
|
|
641
|
-
test("validateConfig defaults dependsOn and its own wait budget, and rejects dependencies it cannot wait on", () => {
|
|
642
|
-
/**
|
|
643
|
-
* @param {import("../src/json.js").JsonValue} processes - Processes under test.
|
|
644
|
-
* @returns {{config: import("../src/config.js").RollbridgeConfig, issues: import("../src/config.js").ConfigIssue[]}} Validation result.
|
|
645
|
-
*/
|
|
646
|
-
const validateProcesses = (processes) => validateConfig({
|
|
647
|
-
application: "demo",
|
|
648
|
-
control: {path: "/tmp/demo.sock"},
|
|
649
|
-
processes,
|
|
650
|
-
proxy: {host: "127.0.0.1", port: 8182}
|
|
651
|
-
})
|
|
652
|
-
|
|
653
|
-
const base = [
|
|
654
|
-
{command: "run beacon", id: "beacon", policy: "service", port: 7330},
|
|
655
|
-
{command: "run jobs", dependsOn: ["beacon"], id: "jobs", policy: "service", port: {from: 7331, to: 7399}},
|
|
656
|
-
{command: "run web", dependsOn: ["jobs"], id: "web", policy: "proxied", port: {from: 18000, to: 18099}}
|
|
657
|
-
]
|
|
658
|
-
const valid = validateProcesses(base)
|
|
659
|
-
|
|
660
|
-
expect(valid.issues).toEqual([])
|
|
661
|
-
expect(valid.config.processes[0].dependsOn).toEqual([])
|
|
662
|
-
expect(valid.config.processes[1].dependsOn).toEqual(["beacon"])
|
|
663
|
-
expect(valid.config.processes[1].dependencyTimeoutMs).toBe(30000)
|
|
664
|
-
|
|
665
|
-
const custom = validateProcesses([...base.slice(0, 2), {...base[2], dependencyTimeoutMs: 60000}])
|
|
666
|
-
|
|
667
|
-
expect(custom.issues).toEqual([])
|
|
668
|
-
expect(custom.config.processes[2].dependencyTimeoutMs).toBe(60000)
|
|
669
|
-
|
|
670
|
-
const invalid = validateProcesses([
|
|
671
|
-
{command: "run web", dependsOn: ["missing", "web", "jobs", "worker"], dependencyTimeoutMs: 0, id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
672
|
-
{command: "run jobs", id: "jobs", policy: "singleton"},
|
|
673
|
-
{command: "run worker", id: "worker", policy: "companion"}
|
|
674
|
-
])
|
|
675
|
-
const messages = invalid.issues.map((issue) => issue.message)
|
|
676
|
-
|
|
677
|
-
expect(messages.some((message) => message === 'Process "web" depends on unknown process "missing"')).toBeTruthy()
|
|
678
|
-
expect(messages.some((message) => message === 'Process "web" cannot depend on itself')).toBeTruthy()
|
|
679
|
-
expect(messages.some((message) => message === 'Process "web" cannot depend on singleton "jobs"')).toBeTruthy()
|
|
680
|
-
expect(messages.some((message) => message === 'Process "worker" has no port, so "web" cannot wait for it')).toBeTruthy()
|
|
681
|
-
expect(messages.some((message) => message === "processes[0].dependencyTimeoutMs must be a positive number")).toBeTruthy()
|
|
682
|
-
|
|
683
|
-
const backwards = validateProcesses([
|
|
684
|
-
{command: "run web", dependsOn: "web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}},
|
|
685
|
-
{command: "run worker", dependsOn: ["web"], id: "worker", policy: "companion", port: 7001}
|
|
686
|
-
])
|
|
687
|
-
const backwardsMessages = backwards.issues.map((issue) => issue.message)
|
|
688
|
-
|
|
689
|
-
expect(backwardsMessages.some((message) => message === "processes[0].dependsOn must be an array")).toBeTruthy()
|
|
690
|
-
expect(backwardsMessages.some((message) => message === 'Process "worker" depends on "web", which Rollbridge starts after it')).toBeTruthy()
|
|
691
|
-
expect(backwards.issues.every((issue) => typeof issue.fix === "string" && issue.fix.length > 0)).toBe(true)
|
|
692
|
-
})
|
|
693
|
-
|
|
694
590
|
test("validateConfig defaults releaseRetention, accepts overrides, and rejects bad values", () => {
|
|
695
591
|
/**
|
|
696
592
|
* @param {import("../src/json.js").JsonValue} releaseRetention - Retention config under test, or undefined.
|
|
@@ -707,25 +603,17 @@ test("validateConfig defaults releaseRetention, accepts overrides, and rejects b
|
|
|
707
603
|
const defaulted = validateRetention(undefined)
|
|
708
604
|
|
|
709
605
|
expect(defaulted.issues).toEqual([])
|
|
710
|
-
expect(defaulted.config.releaseRetention).toEqual({
|
|
606
|
+
expect(defaulted.config.releaseRetention).toEqual({keep: 10, maxAgeMs: 0})
|
|
711
607
|
|
|
712
|
-
const custom = validateRetention({
|
|
608
|
+
const custom = validateRetention({keep: 3, maxAgeMs: 60000})
|
|
713
609
|
|
|
714
610
|
expect(custom.issues).toEqual([])
|
|
715
|
-
expect(custom.config.releaseRetention).toEqual({
|
|
716
|
-
|
|
717
|
-
// 0 is a valid bound: it disables the pre-activation draining-generation check.
|
|
718
|
-
const unbounded = validateRetention({maxConcurrentDrainingReleaseGroups: 0})
|
|
611
|
+
expect(custom.config.releaseRetention).toEqual({keep: 3, maxAgeMs: 60000})
|
|
719
612
|
|
|
720
|
-
|
|
721
|
-
expect(unbounded.config.releaseRetention.maxConcurrentDrainingReleaseGroups).toBe(0)
|
|
613
|
+
const invalid = validateRetention({keep: -1, maxAgeMs: -5})
|
|
722
614
|
|
|
723
|
-
const invalid = validateRetention({escalateWedgedGenerations: "yes", keep: -1, maxAgeMs: -5, maxConcurrentDrainingReleaseGroups: -2})
|
|
724
|
-
|
|
725
|
-
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.escalateWedgedGenerations must be a boolean")).toBeTruthy()
|
|
726
615
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.keep must be a non-negative integer")).toBeTruthy()
|
|
727
616
|
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxAgeMs must be a non-negative number")).toBeTruthy()
|
|
728
|
-
expect(invalid.issues.some((issue) => issue.message === "releaseRetention.maxConcurrentDrainingReleaseGroups must be a non-negative integer")).toBeTruthy()
|
|
729
617
|
})
|
|
730
618
|
|
|
731
619
|
test("validateConfig leaves statePath unset by default, accepts a string, and rejects non-strings", () => {
|
|
@@ -773,24 +661,6 @@ test("ownerRecovery requires durable state and a non-negative integer reconnecti
|
|
|
773
661
|
expect(invalidGrace.issues.some((issue) => issue.message === "ownerRecovery.reconnectGraceMs must be a non-negative integer")).toBeTruthy()
|
|
774
662
|
})
|
|
775
663
|
|
|
776
|
-
test("a normalized config is identical to its own JSON form, so the guardian transfer cannot change it", () => {
|
|
777
|
-
const {config, issues} = validateConfig({
|
|
778
|
-
application: "demo",
|
|
779
|
-
control: {path: "/tmp/demo.sock"},
|
|
780
|
-
processes: [{command: "run web", id: "web", policy: "proxied", port: {from: 18000, to: 18099}}],
|
|
781
|
-
proxy: {host: "127.0.0.1", port: 8182}
|
|
782
|
-
}, "/srv/app/rollbridge.js")
|
|
783
|
-
|
|
784
|
-
expect(issues).toEqual([])
|
|
785
|
-
|
|
786
|
-
// Owner replacement publishes release configs to the candidate daemon over the guardian's JSON
|
|
787
|
-
// socket, which drops undefined-valued properties. A config that is not its own JSON form therefore
|
|
788
|
-
// arrives at the candidate unequal to the config file it was started from, and the live-reload fence
|
|
789
|
-
// aborts the next deploy with "Config changes to control cannot be applied live" even though the
|
|
790
|
-
// file never changed — measured in production on 2026-11-16 (tensorbuzz/production).
|
|
791
|
-
expect(config).toEqual(JSON.parse(JSON.stringify(config)))
|
|
792
|
-
})
|
|
793
|
-
|
|
794
664
|
test("normalizeConfig throws an aggregated error listing every issue", async () => {
|
|
795
665
|
const normalization = Promise.resolve().then(() => normalizeConfig({
|
|
796
666
|
application: "demo",
|
|
@@ -7,39 +7,19 @@ import os from "node:os"
|
|
|
7
7
|
import path from "node:path"
|
|
8
8
|
import {describe, expect, test} from "@velocious/testing"
|
|
9
9
|
import {fileURLToPath} from "node:url"
|
|
10
|
-
import
|
|
11
|
-
import GuardianClient, {usableSocketPath} from "../src/guardian-client.js"
|
|
10
|
+
import GuardianClient from "../src/guardian-client.js"
|
|
12
11
|
import {waitForProcessExit} from "./support/process.js"
|
|
13
12
|
|
|
14
13
|
describe("guardian-client", () => {
|
|
15
14
|
|
|
16
|
-
const commonJsRequire = createRequire(import.meta.url)
|
|
17
|
-
const packageVersion = /** @type {{version: string}} */ (commonJsRequire("../package.json")).version
|
|
18
15
|
const legacyGuardianPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "pre-split3-process-guardian.js")
|
|
19
16
|
const recoveryOwnerPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "guardian-recovery-owner.js")
|
|
20
17
|
|
|
21
|
-
test("guardian maps an overlong configured socket path to a stable short path", async () => {
|
|
22
|
-
const root = await fs.mkdtemp(path.join(os.tmpdir(), `rollbridge-${"x".repeat(120)}-`))
|
|
23
|
-
const configuredPath = path.join(root, "guardian.sock")
|
|
24
|
-
const client = new GuardianClient({socketPath: configuredPath, token: "long-path-capability"})
|
|
25
|
-
|
|
26
|
-
try {
|
|
27
|
-
expect(Buffer.byteLength(configuredPath)).toBeGreaterThan(107)
|
|
28
|
-
expect(Buffer.byteLength(client.socketPath)).toBeLessThanOrEqual(107)
|
|
29
|
-
expect(client.socketPath).not.toBe(configuredPath)
|
|
30
|
-
await client.launch()
|
|
31
|
-
expect(await client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1, guardianVersion: packageVersion})
|
|
32
|
-
} finally {
|
|
33
|
-
await cleanupGuardian({client, root})
|
|
34
|
-
}
|
|
35
|
-
})
|
|
36
|
-
|
|
37
18
|
test("guardian bootstrap capability is absent from process argv", async () => {
|
|
38
19
|
const fixture = await createGuardian()
|
|
39
|
-
const packageVersion = JSON.parse(await fs.readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version
|
|
40
20
|
|
|
41
21
|
try {
|
|
42
|
-
expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1
|
|
22
|
+
expect(await fixture.client.capabilities()).toEqual({daemonRecovery: 1, generationReactivation: 1})
|
|
43
23
|
const commandLine = await fs.readFile(`/proc/${fixture.client.pid}/cmdline`, "utf8")
|
|
44
24
|
const environment = await fs.readFile(`/proc/${fixture.client.pid}/environ`, "utf8")
|
|
45
25
|
const status = await fs.readFile(`/proc/${fixture.client.pid}/status`, "utf8")
|
|
@@ -78,100 +58,6 @@ test("guardian inventory removes only an exact owned provenance", async () => {
|
|
|
78
58
|
}
|
|
79
59
|
})
|
|
80
60
|
|
|
81
|
-
test("a refused owner mutation names the held mutation, its process, its age and its blocker", async () => {
|
|
82
|
-
const fixture = await createGuardian()
|
|
83
|
-
|
|
84
|
-
try {
|
|
85
|
-
const mutationId = await fixture.client.beginOwnerMutation("restart", {ownerPid: process.pid, waitingFor: "the managed process to stop and start again"})
|
|
86
|
-
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/^Another owner mutation is already in progress: owner mutation "restart"; mutation [0-9a-f-]{36}; held by process \d+; since 20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\dZ; \d+s elapsed; waiting for the managed process to stop and start again; refused deploy$/)
|
|
87
|
-
await fixture.client.reportOwnerMutationWait(mutationId, "an idle forked runner that never exits")
|
|
88
|
-
await expect(fixture.client.beginOwnerMutation("deploy")).rejects.toThrow(/waiting for an idle forked runner that never exits/)
|
|
89
|
-
await fixture.client.endOwnerMutation(mutationId)
|
|
90
|
-
expect(typeof await fixture.client.beginOwnerMutation("deploy")).toBe("string")
|
|
91
|
-
} finally {
|
|
92
|
-
await cleanupGuardian(fixture)
|
|
93
|
-
}
|
|
94
|
-
})
|
|
95
|
-
|
|
96
|
-
test("an owner mutation that outruns its lease is released and the holder reports the expiry", async () => {
|
|
97
|
-
const fixture = await createGuardian()
|
|
98
|
-
|
|
99
|
-
try {
|
|
100
|
-
const wedged = await fixture.client.beginOwnerMutation("restart", {mutationTimeoutMs: 200, ownerPid: process.pid, waitingFor: "a drain that never returns"})
|
|
101
|
-
|
|
102
|
-
await new Promise((resolve) => setTimeout(resolve, 600))
|
|
103
|
-
const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
|
|
104
|
-
|
|
105
|
-
expect(typeof successor).toBe("string")
|
|
106
|
-
await expect(fixture.client.endOwnerMutation(wedged)).rejects.toThrow(/^Owner mutation restart exceeded its 200 ms lease and was released after \d+ ms; the mutation failed and its lease is gone$/)
|
|
107
|
-
await fixture.client.endOwnerMutation(successor)
|
|
108
|
-
} finally {
|
|
109
|
-
await cleanupGuardian(fixture)
|
|
110
|
-
}
|
|
111
|
-
})
|
|
112
|
-
|
|
113
|
-
test("a dead owner-mutation holder is reclaimed and the next mutation is admitted", async () => {
|
|
114
|
-
const fixture = await createGuardian()
|
|
115
|
-
const holder = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"])
|
|
116
|
-
|
|
117
|
-
try {
|
|
118
|
-
const reclaimed = fixture.client.waitForEvent("owner-mutation-reclaimed")
|
|
119
|
-
await fixture.client.beginOwnerMutation("restart", {ownerPid: holder.pid, waitingFor: "a process that will die"})
|
|
120
|
-
if (!holder.pid) throw new Error("Missing mutation holder PID")
|
|
121
|
-
holder.kill("SIGKILL")
|
|
122
|
-
await once(holder, "exit")
|
|
123
|
-
|
|
124
|
-
const successor = await fixture.client.beginOwnerMutation("deploy", {ownerPid: process.pid, waitingFor: "candidate activation"})
|
|
125
|
-
const event = await reclaimed
|
|
126
|
-
|
|
127
|
-
expect(typeof successor).toBe("string")
|
|
128
|
-
expect(event).toEqual(expect.objectContaining({code: "OWNER_MUTATION_RECLAIMED", event: "owner-mutation-reclaimed", operation: "restart", pid: holder.pid, reason: "owner mutation holder process is no longer alive"}))
|
|
129
|
-
expect((await fixture.client.guardianStatus()).version).toBe(packageVersion)
|
|
130
|
-
await fixture.client.endOwnerMutation(successor)
|
|
131
|
-
} finally {
|
|
132
|
-
if (holder.exitCode === null && holder.signalCode === null) holder.kill("SIGKILL")
|
|
133
|
-
await cleanupGuardian(fixture)
|
|
134
|
-
}
|
|
135
|
-
})
|
|
136
|
-
|
|
137
|
-
test("guardian lets the owner re-register a key whose registered child is gone, and still fences a live one", async () => {
|
|
138
|
-
const fixture = await createGuardian()
|
|
139
|
-
const liveInstance = fixture.client.process("live-generation", definition("live-generation"))
|
|
140
|
-
const deadInstance = fixture.client.process("dead-generation", definition("dead-generation"))
|
|
141
|
-
|
|
142
|
-
try {
|
|
143
|
-
await liveInstance.start()
|
|
144
|
-
await deadInstance.start()
|
|
145
|
-
const deadPid = (await deadInstance.status()).pid
|
|
146
|
-
|
|
147
|
-
if (!deadPid) throw new Error("Missing required fixture: dead-generation pid")
|
|
148
|
-
await deadInstance.stop()
|
|
149
|
-
await waitForProcessExit(deadPid)
|
|
150
|
-
const deadStatus = await deadInstance.status()
|
|
151
|
-
|
|
152
|
-
expect(deadStatus.state).toBe("stopped")
|
|
153
|
-
expect(deadStatus.pid).toBeUndefined()
|
|
154
|
-
|
|
155
|
-
// The wedge this guards against: a later deploy reconstructs the same key from a
|
|
156
|
-
// different release definition. The old child is conclusively gone, so the old
|
|
157
|
-
// provenance must not fence the new registration.
|
|
158
|
-
const nextInstance = fixture.client.process("dead-generation", {...definition("dead-generation"), env: {REPLACED_BY_LATER_DEPLOY: "1"}})
|
|
159
|
-
const reRegistered = nextInstance.status()
|
|
160
|
-
|
|
161
|
-
expect(reRegistered.state).toBe("stopped")
|
|
162
|
-
await nextInstance.start()
|
|
163
|
-
expect((await nextInstance.status()).state).toBe("running")
|
|
164
|
-
expect((await fixture.client.inventory()).some((entry) => entry.key === "dead-generation" && entry.provenance === nextInstance.provenance)).toBe(true)
|
|
165
|
-
|
|
166
|
-
// A live process with a different provenance stays fail-closed.
|
|
167
|
-
const hostileInstance = fixture.client.process("live-generation", {...definition("live-generation"), env: {HOSTILE: "1"}})
|
|
168
|
-
|
|
169
|
-
await expect(hostileInstance.start()).rejects.toThrow(/provenance mismatch/)
|
|
170
|
-
} finally {
|
|
171
|
-
await cleanupGuardian(fixture)
|
|
172
|
-
}
|
|
173
|
-
})
|
|
174
|
-
|
|
175
61
|
test("guardian runs a strict activation lifecycle command for the exact registered process", async () => {
|
|
176
62
|
const fixture = await createGuardian()
|
|
177
63
|
const activationPath = path.join(fixture.root, "activated")
|
|
@@ -956,7 +842,7 @@ test("replacement staging rejects owner state published after prepare", async ()
|
|
|
956
842
|
const prepared = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
957
843
|
|
|
958
844
|
await fixture.client.publishOwnerState({authority, snapshot: {activeReleaseId: "v2"}})
|
|
959
|
-
await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare
|
|
845
|
+
await expect(candidate.stageOwnerReplacement(prepared.replacementId, {authority: nextAuthority, snapshot: {activeReleaseId: "v1"}})).rejects.toThrow(/owner state changed after prepare/i)
|
|
960
846
|
await candidate.abortOwnerReplacement(prepared.replacementId)
|
|
961
847
|
const fresh = await candidate.prepareOwnerReplacement(authority, nextAuthority)
|
|
962
848
|
|
|
@@ -1792,7 +1678,7 @@ test("a legacy bridge remains discoverable when its candidate disconnects after
|
|
|
1792
1678
|
let replacement
|
|
1793
1679
|
|
|
1794
1680
|
try {
|
|
1795
|
-
const identity = {socketPath:
|
|
1681
|
+
const identity = {socketPath: path.join(fixture.root, "guardian-v2.sock"), token: "candidate-guardian-capability"}
|
|
1796
1682
|
|
|
1797
1683
|
upgraded = await fixture.client.upgradeLegacyGuardian({ownerState, ...identity})
|
|
1798
1684
|
const prepared = await upgraded.prepareOwnerReplacement(authority, nextAuthority)
|
|
@@ -1839,7 +1725,7 @@ async function createGuardian() {
|
|
|
1839
1725
|
async function createLegacyGuardian() {
|
|
1840
1726
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-legacy-guardian-"))
|
|
1841
1727
|
const token = "legacy-guardian-capability"
|
|
1842
|
-
const socketPath =
|
|
1728
|
+
const socketPath = path.join(root, "guardian.sock")
|
|
1843
1729
|
const child = spawn(process.execPath, [legacyGuardianPath, socketPath], {detached: true, stdio: ["ignore", "ignore", "ignore", "ipc"]})
|
|
1844
1730
|
|
|
1845
1731
|
await new Promise((resolve, reject) => {
|
package/test/health.test.js
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import http from "node:http"
|
|
4
|
-
import net from "node:net"
|
|
5
4
|
import {describe, expect, test} from "@velocious/testing"
|
|
6
|
-
import {waitForHealth
|
|
5
|
+
import {waitForHealth} from "../src/health.js"
|
|
7
6
|
|
|
8
7
|
describe("health", () => {
|
|
9
8
|
|
|
@@ -32,18 +31,6 @@ async function startHealthServer() {
|
|
|
32
31
|
}
|
|
33
32
|
}
|
|
34
33
|
|
|
35
|
-
/**
|
|
36
|
-
* @param {net.Server} server - Server to bind.
|
|
37
|
-
* @returns {Promise<number>} Ephemeral port it bound.
|
|
38
|
-
*/
|
|
39
|
-
async function bindPort(server) {
|
|
40
|
-
await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(undefined)))
|
|
41
|
-
|
|
42
|
-
const address = server.address()
|
|
43
|
-
|
|
44
|
-
return address && typeof address === "object" ? address.port : 0
|
|
45
|
-
}
|
|
46
|
-
|
|
47
34
|
test("waitForHealth delays the first probe by startDelayMs", async () => {
|
|
48
35
|
const server = await startHealthServer()
|
|
49
36
|
|
|
@@ -75,64 +62,4 @@ test("waitForHealth probes immediately when startDelayMs is 0", async () => {
|
|
|
75
62
|
await server.close()
|
|
76
63
|
}
|
|
77
64
|
})
|
|
78
|
-
|
|
79
|
-
test("waitForHealth names the deadline and the time it spent when the app never becomes healthy", async () => {
|
|
80
|
-
const server = http.createServer((_request, response) => {
|
|
81
|
-
response.writeHead(500)
|
|
82
|
-
response.end("bad")
|
|
83
|
-
})
|
|
84
|
-
|
|
85
|
-
try {
|
|
86
|
-
const port = await bindPort(server)
|
|
87
|
-
|
|
88
|
-
/** @type {import("../src/health.js").HealthCheckError} */
|
|
89
|
-
const failure = /** @type {import("../src/health.js").HealthCheckError} */ (await waitForHealth({
|
|
90
|
-
health: {intervalMs: 25, path: "/ping", startDelayMs: 0, timeoutMs: 300},
|
|
91
|
-
host: "127.0.0.1",
|
|
92
|
-
port,
|
|
93
|
-
processId: "web"
|
|
94
|
-
}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
95
|
-
|
|
96
|
-
expect(String(failure.message)).toMatch(/process web: \d+ probes, \d+ms elapsed of the 300ms health deadline/)
|
|
97
|
-
expect(failure.healthCheck?.deadlineMs).toBe(300)
|
|
98
|
-
expect({value: Boolean(failure.healthCheck && failure.healthCheck.elapsedMs >= 300), context: `expected the health gate to have spent its whole budget, was ${failure.healthCheck?.elapsedMs}ms`}).toMatchObject({value: true})
|
|
99
|
-
expect({value: Boolean(failure.healthCheck && failure.healthCheck.attempts >= 1), context: `expected at least one probe, was ${failure.healthCheck?.attempts}`}).toMatchObject({value: true})
|
|
100
|
-
expect(failure.healthCheck?.processId).toBe("web")
|
|
101
|
-
} finally {
|
|
102
|
-
await server.close()
|
|
103
|
-
}
|
|
104
|
-
})
|
|
105
|
-
|
|
106
|
-
test("waitForListeningPort waits for a dependency port on a budget of its own", async () => {
|
|
107
|
-
const server = net.createServer()
|
|
108
|
-
const port = await bindPort(server)
|
|
109
|
-
|
|
110
|
-
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
111
|
-
|
|
112
|
-
const waiting = waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 2000})
|
|
113
|
-
|
|
114
|
-
await new Promise((resolve) => setTimeout(resolve, 300))
|
|
115
|
-
await new Promise((resolve) => server.listen(port, "127.0.0.1", () => resolve(undefined)))
|
|
116
|
-
|
|
117
|
-
try {
|
|
118
|
-
const result = await waiting
|
|
119
|
-
|
|
120
|
-
expect({value: Boolean(result.attempts >= 2), context: `expected more than one connection attempt, was ${result.attempts}`}).toMatchObject({value: true})
|
|
121
|
-
expect({value: Boolean(result.elapsedMs >= 250), context: `expected the dependency wait to take ~300ms, was ${result.elapsedMs}ms`}).toMatchObject({value: true})
|
|
122
|
-
} finally {
|
|
123
|
-
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
124
|
-
}
|
|
125
|
-
})
|
|
126
|
-
|
|
127
|
-
test("waitForListeningPort names the budget it ran out on when the dependency never binds", async () => {
|
|
128
|
-
const server = net.createServer()
|
|
129
|
-
const port = await bindPort(server)
|
|
130
|
-
|
|
131
|
-
await new Promise((resolve) => server.close(() => resolve(undefined)))
|
|
132
|
-
|
|
133
|
-
/** @type {Error} */
|
|
134
|
-
const failure = /** @type {Error} */ (await waitForListeningPort({host: "127.0.0.1", intervalMs: 25, port, timeoutMs: 200}).catch((error) => error instanceof Error ? error : new Error(String(error))))
|
|
135
|
-
|
|
136
|
-
expect(String(failure.message)).toMatch(/did not accept a connection within 200ms/)
|
|
137
|
-
})
|
|
138
65
|
})
|