rollbridge 0.1.56 → 0.1.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -0
- package/TODO.md +12 -0
- package/changelog.d/20261013-recover-dead-guardian-state.md +1 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/docs/cli.md +70 -1
- package/docs/compose-services.md +220 -0
- package/docs/config.md +55 -0
- package/docs/troubleshooting.md +24 -0
- package/package.json +1 -1
- package/src/cli.js +82 -4
- package/src/compose-unit.js +725 -0
- package/src/config.js +201 -2
- package/src/daemon.js +23 -8
- package/src/doctor.js +67 -0
- package/src/guardian-client.js +13 -2
- package/src/process-guardian.js +137 -17
- package/src/recover.js +128 -8
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +158 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +70 -0
- package/test/recover.test.js +119 -0
- package/test/rollbridge.test.js +106 -6
package/test/rollbridge.test.js
CHANGED
|
@@ -23,6 +23,7 @@ const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
|
|
|
23
23
|
const memoryHogPath = path.join(currentDir, "fixtures", "memory-hog.js")
|
|
24
24
|
const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
|
|
25
25
|
const singletonAppPath = path.join(currentDir, "fixtures", "singleton-app.js")
|
|
26
|
+
const gatedAppPath = path.join(currentDir, "fixtures", "gated-app.js")
|
|
26
27
|
const linuxTest = process.platform === "linux" ? test : test.skip
|
|
27
28
|
|
|
28
29
|
test("a nonBlockingDrain worker stops immediately while its release is still draining", async () => {
|
|
@@ -751,6 +752,102 @@ test("candidate activation failure reports restoration failure and exact recover
|
|
|
751
752
|
}
|
|
752
753
|
})
|
|
753
754
|
|
|
755
|
+
test("recovery against a committed journal naming the live incumbent reports already_recovered and admits a rerun", async () => {
|
|
756
|
+
const fixture = await createFixture({gatedWeb: true, webHealthTimeoutMs: 500})
|
|
757
|
+
const daemon = await startDaemon(fixture.config)
|
|
758
|
+
|
|
759
|
+
try {
|
|
760
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
761
|
+
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
762
|
+
|
|
763
|
+
// v3 never becomes healthy: the deploy aborts before its own transition is journaled,
|
|
764
|
+
// so the journal still holds the committed transition that published v2, exactly like
|
|
765
|
+
// the TensorBuzz production incident.
|
|
766
|
+
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/health check failed/i)
|
|
767
|
+
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
768
|
+
expect(daemon.status().generationTransition?.candidateReleaseId).toBe("v2")
|
|
769
|
+
expect(daemon.status().generationTransition?.previousReleaseId).toBe("v1")
|
|
770
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
771
|
+
|
|
772
|
+
const exactRecovery = (overrides = {}) => sendControlCommand({
|
|
773
|
+
command: {
|
|
774
|
+
command: "recover-generation-transition",
|
|
775
|
+
previousReleaseId: "v2",
|
|
776
|
+
releaseId: "v3",
|
|
777
|
+
releasePath: fixture.root,
|
|
778
|
+
revision: "v3",
|
|
779
|
+
...overrides
|
|
780
|
+
},
|
|
781
|
+
path: fixture.config.control.path
|
|
782
|
+
})
|
|
783
|
+
|
|
784
|
+
// A fence naming a release other than the live incumbent is not a recoverable
|
|
785
|
+
// transition and keeps failing closed.
|
|
786
|
+
await expect(exactRecovery({previousReleaseId: "v1"})).rejects.toThrow(/not a safe failed pre-commit transition/i)
|
|
787
|
+
await expect(exactRecovery({previousReleaseId: "not-a-release"})).rejects.toThrow(/refusing stale recovery/i)
|
|
788
|
+
|
|
789
|
+
// Rampway derives the recovery fence from the release that is actually running.
|
|
790
|
+
const recovery = await exactRecovery()
|
|
791
|
+
|
|
792
|
+
expect(recovery.recoveryStatus).toBe("already_recovered")
|
|
793
|
+
expect(recovery.activeReleaseId).toBe("v2")
|
|
794
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
795
|
+
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
796
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
797
|
+
|
|
798
|
+
// The acceptance flow: the deploy is recovered by rerunning it, with no state edits.
|
|
799
|
+
await fs.writeFile(fixture.gatePath, "allow\n")
|
|
800
|
+
await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
|
|
801
|
+
expect(daemon.status().activeReleaseId).toBe("v3")
|
|
802
|
+
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
803
|
+
expect(daemon.status().generationTransition?.candidateReleaseId).toBe("v3")
|
|
804
|
+
expect(await fetchText(daemon, "/release")).toBe("v3")
|
|
805
|
+
} finally {
|
|
806
|
+
await daemon.shutdown()
|
|
807
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
808
|
+
}
|
|
809
|
+
})
|
|
810
|
+
|
|
811
|
+
test("recovery keeps the stale-fence guard failing closed for a genuinely in-flight transition", async () => {
|
|
812
|
+
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
813
|
+
const daemon = await startDaemon(fixture.config)
|
|
814
|
+
|
|
815
|
+
try {
|
|
816
|
+
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
817
|
+
const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
|
|
818
|
+
const reactivate = incumbentCoordinator?.reactivateStrict.bind(incumbentCoordinator)
|
|
819
|
+
|
|
820
|
+
if (!(incumbentCoordinator && reactivate)) throw new Error("Missing required fixture: incumbentCoordinator && reactivate")
|
|
821
|
+
incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
|
|
822
|
+
const deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
823
|
+
|
|
824
|
+
await expect(deployment).rejects.toThrow(/incumbent v1 restoration failed: incumbent restoration rejected/i)
|
|
825
|
+
|
|
826
|
+
// The failed candidate is still journaled in flight, so the fence must keep failing.
|
|
827
|
+
expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
|
|
828
|
+
|
|
829
|
+
const exactRecovery = (overrides = {}) => sendControlCommand({
|
|
830
|
+
command: {
|
|
831
|
+
command: "recover-generation-transition",
|
|
832
|
+
previousReleaseId: "v1",
|
|
833
|
+
releaseId: "v2",
|
|
834
|
+
releasePath: fixture.root,
|
|
835
|
+
revision: "v2",
|
|
836
|
+
...overrides
|
|
837
|
+
},
|
|
838
|
+
path: fixture.config.control.path
|
|
839
|
+
})
|
|
840
|
+
|
|
841
|
+
await expect(exactRecovery({previousReleaseId: "v2"})).rejects.toThrow(/refusing stale recovery/i)
|
|
842
|
+
await expect(exactRecovery({previousReleaseId: "not-a-release"})).rejects.toThrow(/refusing stale recovery/i)
|
|
843
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
844
|
+
expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
|
|
845
|
+
} finally {
|
|
846
|
+
await daemon.shutdown()
|
|
847
|
+
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
848
|
+
}
|
|
849
|
+
})
|
|
850
|
+
|
|
754
851
|
test("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
|
|
755
852
|
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true})
|
|
756
853
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -1973,8 +2070,8 @@ test("deploy can ensure the daemon before sending the release command", async ()
|
|
|
1973
2070
|
})
|
|
1974
2071
|
|
|
1975
2072
|
/**
|
|
1976
|
-
* @param {{companionReplicas?: number, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
|
|
1977
|
-
* @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
|
|
2073
|
+
* @param {{companionReplicas?: number, gatedWeb?: boolean, handoffService?: boolean, handoffServiceActivate?: boolean, handoffServiceActivateAmbiguousFailure?: boolean, handoffServiceActivateFailure?: boolean | string, handoffServiceQuiet?: boolean, handoffServiceQuietFailure?: boolean, includeCompanion?: boolean, includeService?: boolean, includeSingleton?: boolean, memoryLimitBytes?: number, nonBlockingDrainWorker?: boolean, persistState?: boolean, proxyHost?: string, singletonCwd?: string, webCommand?: string, webDependsOnService?: boolean, webHealthTimeoutMs?: number, workerReactivationFailure?: boolean, workerReactivationLifecycle?: boolean, workerStopDelayMs?: number}} [options] - Fixture options.
|
|
2074
|
+
* @returns {Promise<{activationGatePath: string, config: import("../src/config.js").RollbridgeConfig, gatePath: string, lifecycleLogPath: string, retirementGatePath: string, root: string, serviceLogPath: string, serviceQuietPath: string, singletonLogPath: string, statePath: string}>} Fixture data.
|
|
1978
2075
|
*/
|
|
1979
2076
|
async function createFixture(options = {}) {
|
|
1980
2077
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-test-"))
|
|
@@ -1985,6 +2082,7 @@ async function createFixture(options = {}) {
|
|
|
1985
2082
|
const retirementGatePath = path.join(root, "retirement.allow")
|
|
1986
2083
|
const singletonLogPath = path.join(root, "singleton.log")
|
|
1987
2084
|
const statePath = path.join(root, "rollbridge.state.json")
|
|
2085
|
+
const gatePath = path.join(root, "stc788-gate")
|
|
1988
2086
|
/** @type {Array<Record<string, import("../src/json.js").JsonValue>>} */
|
|
1989
2087
|
const processes = []
|
|
1990
2088
|
|
|
@@ -2054,9 +2152,11 @@ async function createFixture(options = {}) {
|
|
|
2054
2152
|
}
|
|
2055
2153
|
|
|
2056
2154
|
processes.push({
|
|
2057
|
-
command: options.webCommand || (options.
|
|
2058
|
-
? `${JSON.stringify(process.execPath)} ${JSON.stringify(
|
|
2059
|
-
:
|
|
2155
|
+
command: options.webCommand || (options.gatedWeb
|
|
2156
|
+
? `${JSON.stringify(process.execPath)} ${JSON.stringify(gatedAppPath)} --gate=${JSON.stringify(gatePath)}`
|
|
2157
|
+
: options.webDependsOnService
|
|
2158
|
+
? `${JSON.stringify(process.execPath)} ${JSON.stringify(dependentAppPath)}`
|
|
2159
|
+
: `${JSON.stringify(process.execPath)} ${JSON.stringify(dummyAppPath)}`),
|
|
2060
2160
|
health: {
|
|
2061
2161
|
intervalMs: 50,
|
|
2062
2162
|
path: "/ping",
|
|
@@ -2096,7 +2196,7 @@ async function createFixture(options = {}) {
|
|
|
2096
2196
|
...((options.persistState || options.handoffServiceActivate) ? {ownerRecovery: {reconnectGraceMs: 30000}, statePath} : {})
|
|
2097
2197
|
})
|
|
2098
2198
|
|
|
2099
|
-
return {activationGatePath, config, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
|
|
2199
|
+
return {activationGatePath, config, gatePath, lifecycleLogPath, retirementGatePath, root, serviceLogPath, serviceQuietPath, singletonLogPath, statePath}
|
|
2100
2200
|
}
|
|
2101
2201
|
|
|
2102
2202
|
/**
|