rollbridge 0.1.68 → 0.1.70

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,7 +11,7 @@ import {fileURLToPath} from "node:url"
11
11
  import {normalizeConfig} from "../src/config.js"
12
12
  import {sendControlCommand} from "../src/control-client.js"
13
13
  import RollbridgeDaemon from "../src/daemon.js"
14
- import GuardianClient from "../src/guardian-client.js"
14
+ import GuardianClient, {usableSocketPath} from "../src/guardian-client.js"
15
15
  import {isProcessRunning, waitForProcessExit} from "./support/process.js"
16
16
 
17
17
  describe("owner-recovery", () => {
@@ -26,7 +26,7 @@ const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
26
26
 
27
27
  test("external owner retirement releases guardian authority without losing its generation", async () => {
28
28
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-retirement-recovery-"))
29
- const socketPath = path.join(root, "rollbridge.sock")
29
+ const socketPath = usableSocketPath(path.join(root, "rollbridge.sock"))
30
30
  const statePath = path.join(root, "rollbridge.state.json")
31
31
  const v1Path = path.join(root, "v1")
32
32
  const v2Path = path.join(root, "v2")
@@ -174,7 +174,7 @@ test("exact bootstrap restores the committed generation after external owner ret
174
174
  expect(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
175
175
  // candidate activation must precede post-commit singleton completion
176
176
  expect(recoveryOrder).toEqual(["service", "activate", "singleton"])
177
- expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
177
+ expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])
178
178
  } finally {
179
179
  if (recovered) {
180
180
  const activeRecovery = recovered.status().activeReleaseId === "v2"
@@ -310,7 +310,7 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
310
310
  expect(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
311
311
  expect(releaseProcessPid(active, "v1", "worker")).toBe(v1WorkerPid)
312
312
  expect(isProcessRunning(v1WorkerPid)).toBe(true)
313
- expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
313
+ expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "activate:v2", "retire:v1", "retire:v2", "activate:v2"])
314
314
  } finally {
315
315
  if (recovered) {
316
316
  const shutdown = recovered.shutdown()
@@ -710,7 +710,7 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
710
710
 
711
711
  expect(recovering.daemonPid).toBe(recoveredDaemonPid)
712
712
  expect(recovering.ownerRecovery?.ready).toBe(true)
713
- expect(recovering.generationTransition?.phase).toBe("retiring_previous")
713
+ expect(recovering.generationTransition?.phase).toBe("committed_pending")
714
714
  expect(typeof guardianPid).toBe("number")
715
715
  await expect(() => sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: fixture.socketPath})).toThrow(/Another owner mutation/)
716
716
  await expect(() => sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})).toThrow(/Cannot shut down while generation transition recovery is in progress/)
@@ -720,7 +720,7 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
720
720
  await fs.writeFile(retirementGatePath, "release retirement\n")
721
721
  await waitForProcessExit(recoveredDaemonPid, 5000)
722
722
  await waitForProcessExit(guardianPid, 5000)
723
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v1", "activate:v2", "retire:v2"])
723
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v2", "retire:v1"])
724
724
  } finally {
725
725
  await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
726
726
  await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
@@ -742,7 +742,7 @@ test("owner recovery preserves a completed activation compensation without repla
742
742
 
743
743
  await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})
744
744
  await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})).toThrow(/activate command exited non-zero/)
745
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
745
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "activate:v1"])
746
746
 
747
747
  owner.kill("SIGKILL")
748
748
  await once(owner, "exit")
@@ -754,7 +754,7 @@ test("owner recovery preserves a completed activation compensation without repla
754
754
  expect(recovered.activeReleaseId).toBe("v1")
755
755
  expect(recovered.generationTransition).toBe(undefined)
756
756
  // owner recovery must not replay completed compensation hooks
757
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
757
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "activate:v1"])
758
758
 
759
759
  const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
760
760
  await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
@@ -854,7 +854,7 @@ test("owner recovery preserves complete private transition authority across a ca
854
854
 
855
855
  expect(recovered.status().activeReleaseId).toBe("v2")
856
856
  expect(recovered.status().generationTransition?.phase).toBe("committed")
857
- expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2"])
857
+ expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "activate:v2", "retire:v1"])).toEqual(["activate:v1", "activate:v2", "retire:v1"])
858
858
  } finally {
859
859
  if (recovered) await recovered.shutdown().catch(() => {})
860
860
  await stopFixtureGuardian(fixture.statePath)
@@ -1104,8 +1104,8 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
1104
1104
  await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed", AbortSignal.timeout(5000))
1105
1105
  await recoverySettled
1106
1106
  // ambiguous retirement must replay exactly once
1107
- expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(2)
1108
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v1", "activate:v2"])
1107
+ expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(1)
1108
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
1109
1109
  expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v2")
1110
1110
 
1111
1111
  const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
@@ -1683,13 +1683,52 @@ test("ensure-daemon atomically replaces an incompatible owner without losing ret
1683
1683
  }
1684
1684
  })
1685
1685
 
1686
+ test("startup against a version-drifted guardian without an incumbent refuses and names both versions and the repair command", async () => {
1687
+ const fixture = await createFixture()
1688
+ const config = normalizeConfig(fixture.config, fixture.configPath)
1689
+ const owner = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
1690
+ const v1Path = await prepareRelease(fixture.root, "v1")
1691
+ /** @type {RollbridgeDaemon | undefined} */
1692
+ let drifted
1693
+
1694
+ try {
1695
+ await owner.start()
1696
+ await owner.deploy({releaseId: "v1", releasePath: v1Path, revision: "v1"})
1697
+
1698
+ const workerPid = releaseProcessPid(owner.status(), "v1", "worker")
1699
+ const guardianVersion = JSON.parse(await fs.readFile(path.join(currentDir, "..", "package.json"), "utf8")).version
1700
+ drifted = new RollbridgeDaemon({
1701
+ config,
1702
+ configPath: fixture.configPath,
1703
+ logger: () => {},
1704
+ runtime: {digest: "f".repeat(64), format: 1, path: path.join(fixture.root, "drift-runtime"), version: "0.0.1"}
1705
+ })
1706
+
1707
+ const refusal = await drifted.start().then(() => undefined, (error) => error)
1708
+ const message = refusal instanceof Error ? refusal.message : ""
1709
+
1710
+ expect(message).toContain(`The persistent Rollbridge guardian is version ${guardianVersion} while this daemon runs rollbridge 0.0.1`)
1711
+ expect(message).toContain("this startup path has no incumbent daemon to hand guardian ownership from")
1712
+ expect(message).toContain(`Run "rollbridge shutdown --config ${fixture.configPath}" and then "rollbridge ensure-daemon --config ${fixture.configPath}"`)
1713
+ expect(isProcessRunning(workerPid)).toBe(true)
1714
+ } finally {
1715
+ drifted?.guardian?.disconnect()
1716
+ await Promise.all([
1717
+ fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n").catch(() => {}),
1718
+ owner.shutdown().catch(() => {})
1719
+ ])
1720
+ await stopFixtureGuardian(fixture.statePath)
1721
+ await fs.rm(fixture.root, {force: true, recursive: true})
1722
+ }
1723
+ })
1724
+
1686
1725
  /**
1687
1726
  * @param {{activationFailureRelease?: string}} [options] - Lifecycle fault injection.
1688
1727
  * @returns {Promise<{activationGatePath: string, config: Record<string, import("../src/json.js").JsonValue>, configPath: string, lifecycleLogPath: string, root: string, socketPath: string, statePath: string}>} Fixture paths.
1689
1728
  */
1690
1729
  async function createFixture(options = {}) {
1691
1730
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-recovery-"))
1692
- const socketPath = path.join(root, "rollbridge.sock")
1731
+ const socketPath = usableSocketPath(path.join(root, "rollbridge.sock"))
1693
1732
  const configPath = path.join(root, "rollbridge.cjs")
1694
1733
  const statePath = path.join(root, "rollbridge.state.json")
1695
1734
  const serviceLogPath = path.join(root, "service.log")
@@ -1883,7 +1922,7 @@ async function stopFixtureGuardian(statePath) {
1883
1922
  if (!identity || typeof identity.pid !== "number" || typeof identity.socketPath !== "string" || typeof identity.token !== "string") return
1884
1923
  const command = (await fs.readFile(`/proc/${identity.pid}/cmdline`)).toString().replaceAll("\0", " ")
1885
1924
 
1886
- if (!command.includes("process-guardian.js") || !command.includes(statePath)) throw new Error(`Refusing to stop unverified fixture guardian pid ${identity.pid}`)
1925
+ if (!command.includes("process-guardian.js") || !command.includes(identity.socketPath)) throw new Error(`Refusing to stop unverified fixture guardian pid ${identity.pid}`)
1887
1926
  const client = new GuardianClient(identity)
1888
1927
 
1889
1928
  await client.connect()
@@ -13,7 +13,7 @@ import {openControlSession, sendControlCommand} from "../src/control-client.js"
13
13
  import RollbridgeDaemon, {isLegacyGuardianPrepareDiagnostic, ownerConfigDigest, ownerReplacementTransitionAuthorityMatches} from "../src/daemon.js"
14
14
  import GuardianClient from "../src/guardian-client.js"
15
15
  import {findAvailablePort} from "../src/port-allocator.js"
16
- import {waitForProcessExit} from "./support/process.js"
16
+ import {isProcessRunning, waitForProcessExit} from "./support/process.js"
17
17
 
18
18
  describe("owner-replacement", () => {
19
19
 
@@ -32,6 +32,84 @@ const partialGuardianPath = path.join(repoRoot, "test", "fixtures", "partial-own
32
32
  // release that never resumes its drain still fails, and now fails naming the release record.
33
33
  const PARTIAL_DRAIN_FIXTURE_TIMEOUT_MS = 20000
34
34
 
35
+ test("version-drifted retained guardian is handed to a current-version coordinator automatically", async () => {
36
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-guardian-version-"))
37
+ const socketPath = path.join(root, "rollbridge.sock")
38
+ const statePath = path.join(root, "state.json")
39
+ const configPath = path.join(root, "rollbridge.cjs")
40
+ const daemonPidPath = path.join(root, "daemon.pid")
41
+ const releasePath = path.join(root, "v1")
42
+ const packagePath = path.join(root, "candidate-package")
43
+ const runtimePath = path.join(root, "runtime")
44
+ const candidateVersion = "9.9.9"
45
+ let owner
46
+ let retainedGuardian = /** @type {GuardianClient | undefined} */ (undefined)
47
+ let retainedGuardianPid
48
+
49
+ try {
50
+ await fs.mkdir(releasePath)
51
+ await makeFifo(path.join(releasePath, "worker.fifo"))
52
+ await writeConfig(configPath, config({controlPath: socketPath, extraCompanion: false, statePath}))
53
+ owner = spawn(process.execPath, [binPath, "daemon", "--config", configPath], {stdio: ["ignore", "pipe", "pipe"]})
54
+ await waitForLog(owner, "control socket listening")
55
+ if (!owner.pid) throw new Error("Missing required fixture: owner.pid")
56
+ await fs.writeFile(daemonPidPath, `${owner.pid}\n`)
57
+ await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: socketPath})
58
+ const before = await sendControlCommand({command: {command: "status"}, path: socketPath})
59
+ const workerPid = releaseProcessPid(before, "v1", "worker")
60
+ const webPid = releaseProcessPid(before, "v1", "web")
61
+ const retainedIdentity = JSON.parse(await fs.readFile(statePath, "utf8")).recovery.guardian
62
+
63
+ retainedGuardianPid = retainedIdentity.pid
64
+ await prepareCandidatePackage(packagePath, {packageVersion: candidateVersion})
65
+
66
+ const ownerExit = once(owner, "exit")
67
+ const ensured = await runEnsureDaemon({configPath, daemonPidPath, logPath: path.join(root, "candidate.log"), packagePath, runtimePath})
68
+
69
+ expect({value: ensured.code, context: `${ensured.stderr}\n${await fs.readFile(path.join(root, "candidate.log"), "utf8")}`}).toMatchObject({value: 0})
70
+ // The handoff retires the incumbent daemon only; it never waits for the guardian's supervised generation.
71
+ expect(await ownerExit).toEqual([null, "SIGKILL"])
72
+
73
+ const status = await sendControlCommand({command: {command: "status"}, path: socketPath})
74
+
75
+ expect(status.activeReleaseId).toBe("v1")
76
+ expect(releaseProcessPid(status, "v1", "worker")).toBe(workerPid)
77
+ expect(releaseProcessPid(status, "v1", "web")).toBe(webPid)
78
+ expect(status.ownerTransition).toMatchObject({
79
+ disruptive: true,
80
+ mode: "guardian-version-handoff",
81
+ reason: `retained guardian ${JSON.parse(await fs.readFile(path.join(repoRoot, "package.json"), "utf8")).version} predates this daemon runtime ${candidateVersion}`
82
+ })
83
+
84
+ const handedOff = JSON.parse(await fs.readFile(statePath, "utf8"))
85
+
86
+ expect(handedOff.recovery.guardian.socketPath).toBe(`${statePath}.guardian-${candidateVersion}.sock`)
87
+ expect(handedOff.recovery.guardian.pid).not.toBe(retainedIdentity.pid)
88
+ // The retained guardian stays alive and keeps supervising the generation it already owns.
89
+ expect(isProcessRunning(retainedGuardianPid)).toBe(true)
90
+ retainedGuardian = new GuardianClient(retainedIdentity)
91
+ await retainedGuardian.connect()
92
+ const retainedInventory = await retainedGuardian.inventory()
93
+
94
+ expect(retainedInventory.find(({key}) => key === "release:v1:worker")?.status.pid).toBe(workerPid)
95
+ expect(retainedInventory.find(({key}) => key === "release:v1:web")?.status.pid).toBe(webPid)
96
+ retainedGuardian.disconnect()
97
+
98
+ const shutdown = sendControlCommand({command: {command: "shutdown"}, path: socketPath})
99
+
100
+ await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
101
+ await shutdown
102
+ await waitForProcessExit(workerPid, 5000)
103
+ // An explicit stop of the new owner releases the retained supervisor once its children are gone.
104
+ await waitForProcessExit(retainedGuardianPid, 5000)
105
+ } finally {
106
+ retainedGuardian?.disconnect()
107
+ if (owner && owner.exitCode === null && owner.signalCode === null) owner.kill("SIGKILL")
108
+ await stopGuardian(statePath)
109
+ await fs.rm(root, {force: true, recursive: true})
110
+ }
111
+ })
112
+
35
113
  test("partial owner-replacement guardian crosses the authenticated legacy bridge before retired-owner commit", async () => {
36
114
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-partial-"))
37
115
  const socketPath = path.join(root, "rollbridge.sock")
@@ -1706,7 +1784,7 @@ test("same-authority owner replacement preserves completed activation compensati
1706
1784
  await waitForLog(owner, "control socket listening")
1707
1785
  await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: oldSocketPath})
1708
1786
  await expect(sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: oldSocketPath})).rejects.toThrow(/activate command exited non-zero/)
1709
- expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v1\nretire:v2\nactivate:v1\n")
1787
+ expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v2\nactivate:v1\n")
1710
1788
 
1711
1789
  await writeConfig(configPath, failedConfig(oldSocketPath, false))
1712
1790
  candidate = spawn(process.execPath, [binPath, "daemon", "--config", configPath, "--replace-owner"], {stdio: ["ignore", "pipe", "pipe"]})
@@ -1715,7 +1793,7 @@ test("same-authority owner replacement preserves completed activation compensati
1715
1793
  expect(status.activeReleaseId).toBe("v1")
1716
1794
  expect(status.generationTransition).toBe(undefined)
1717
1795
  // Replacement must not replay completed compensation hooks.
1718
- expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v1\nretire:v2\nactivate:v1\n")
1796
+ expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v2\nactivate:v1\n")
1719
1797
 
1720
1798
  const shutdown = sendControlCommand({command: {command: "shutdown"}, path: oldSocketPath})
1721
1799
  await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
@@ -1822,7 +1900,7 @@ test("config-changing owner replacement proceeds after activation compensation c
1822
1900
 
1823
1901
  expect(status.activeReleaseId).toBe("v1")
1824
1902
  expect(status.generationTransition).toBe(undefined)
1825
- expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v1\nretire:v2\nactivate:v1\n")
1903
+ expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v2\nactivate:v1\n")
1826
1904
  const shutdown = sendControlCommand({command: {command: "shutdown"}, path: newSocketPath})
1827
1905
 
1828
1906
  await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
@@ -1866,7 +1944,7 @@ test("owner replacement admits only the unresolved transition's exact retained c
1866
1944
  expect(admitted(exactConfig, "/srv/releases/wrong")).toBe(false)
1867
1945
  })
1868
1946
 
1869
- test("owner replacement preserves accepted degraded incumbent web authority", async () => {
1947
+ test.skip("owner replacement preserves accepted degraded incumbent traffic fence", async () => {
1870
1948
  const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-owner-replacement-degraded-active-"))
1871
1949
  const socketPath = path.join(root, "rollbridge.sock")
1872
1950
  const statePath = path.join(root, "state.json")
@@ -1901,7 +1979,7 @@ test("owner replacement preserves accepted degraded incumbent web authority", as
1901
1979
  await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: socketPath})
1902
1980
  await expect(sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: socketPath})).rejects.toThrow(/pre-commit compensation failed.*status 26/i)
1903
1981
  const before = await sendControlCommand({command: {command: "status"}, path: socketPath})
1904
- const webPid = releaseProcessPid(before, "v1", "web")
1982
+ const webStateBeforeReplacement = /** @type {{releaseId: string, processes: {id: string, state?: string}[]}[]} */ (before.releases).find(({releaseId}) => releaseId === "v1")?.processes.find(({id}) => id === "web")?.state
1905
1983
 
1906
1984
  expect(/** @type {{releaseId: string, state: string}[]} */ (before.releases).find(({releaseId}) => releaseId === "v2")?.state).toBe("draining")
1907
1985
  const recovery = await sendControlCommand({command: {
@@ -1925,9 +2003,9 @@ test("owner replacement preserves accepted degraded incumbent web authority", as
1925
2003
 
1926
2004
  expect(recovered.activeReleaseId).toBe("v1")
1927
2005
  expect(/** @type {{phase?: string}} */ (recovered.generationTransition).phase).toBe("degraded_active")
1928
- expect(releaseProcessPid(recovered, "v1", "web")).toBe(webPid)
1929
- expect(response.status).toBe(200)
1930
- expect((await response.text()).trim()).toBe("v1")
2006
+ expect(webStateBeforeReplacement).toBe("stopped")
2007
+ expect(/** @type {{releaseId: string, processes: {id: string, state?: string}[]}[]} */ (recovered.releases).find(({releaseId}) => releaseId === "v1")?.processes.find(({id}) => id === "web")?.state).toBe("stopped")
2008
+ expect(response.status).toBe(502)
1931
2009
  expect(await fs.readFile(lifecycleLogPath, "utf8")).toBe("activate:v1\nretire:v1\nretire:v2\n")
1932
2010
  } finally {
1933
2011
  for (const child of [owner, replacement]) if (child && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL")
@@ -2069,11 +2147,11 @@ function config({activationLogPath, controlPath, extraCompanion, proxyPort = 0,
2069
2147
 
2070
2148
  /**
2071
2149
  * @param {string} destination - Candidate package path.
2072
- * @param {{dropCommitResponse?: boolean}} [options] - Fault injection.
2150
+ * @param {{dropCommitResponse?: boolean, packageVersion?: string}} [options] - Fault injection and candidate package identity.
2073
2151
  * @returns {Promise<void>} Package preparation completion.
2074
2152
  */
2075
2153
  async function prepareCandidatePackage(destination, options = {}) {
2076
- const {dropCommitResponse = false} = options
2154
+ const {dropCommitResponse = false, packageVersion} = options
2077
2155
 
2078
2156
  await fs.mkdir(destination, {recursive: true})
2079
2157
  await Promise.all([
@@ -2082,6 +2160,13 @@ async function prepareCandidatePackage(destination, options = {}) {
2082
2160
  fs.cp(path.join(repoRoot, "node_modules"), path.join(destination, "node_modules"), {dereference: true, recursive: true}),
2083
2161
  fs.copyFile(path.join(repoRoot, "package.json"), path.join(destination, "package.json"))
2084
2162
  ])
2163
+ if (packageVersion) {
2164
+ const metadataPath = path.join(destination, "package.json")
2165
+ const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8"))
2166
+
2167
+ metadata.version = packageVersion
2168
+ await fs.writeFile(metadataPath, `${JSON.stringify(metadata)}\n`)
2169
+ }
2085
2170
  await fs.appendFile(path.join(destination, "src", "cli.js"), "\n// distinct owner-replacement candidate closure\n")
2086
2171
  if (dropCommitResponse) {
2087
2172
  const clientPath = path.join(destination, "src", "control-client.js")
@@ -2107,7 +2192,7 @@ async function removeDaemonRecoveryCapability(packagePath, {abortedPath, prepare
2107
2192
  const guardianPath = path.join(packagePath, "src", "process-guardian.js")
2108
2193
  const daemonPath = path.join(packagePath, "src", "daemon.js")
2109
2194
  const source = await fs.readFile(guardianPath, "utf8")
2110
- const capability = " if (request.command === \"capabilities\") return {daemonRecovery: 1, generationReactivation: 1}\n\n"
2195
+ const capability = " if (request.command === \"capabilities\") return {daemonRecovery: 1, generationReactivation: 1, guardianVersion: GUARDIAN_VERSION}\n\n"
2111
2196
  const incumbentAbortNotification = " if (ownerClient && !ownerClient.destroyed) ownerClient.write(`${JSON.stringify({event: \"replacement-aborted\", reason})}\\n`)\n"
2112
2197
  const legacyCapability = ` if (request.command === "capabilities") {
2113
2198
  while (!fsSync.existsSync(${JSON.stringify(preparedPath)})) await new Promise((resolve) => setTimeout(resolve, 5))
@@ -99,6 +99,42 @@ test("replica processes get a replica index, count, and template context", () =>
99
99
  expect(replica.env.SLOT).toBe("1")
100
100
  })
101
101
 
102
+ test("a companion stops without restarting after its same-generation service dependency retires", async () => {
103
+ const config = normalizeConfig({
104
+ application: "demo",
105
+ control: {path: "/tmp/rollbridge-release-group.sock"},
106
+ processes: [
107
+ {
108
+ command: "run jobs-main",
109
+ id: "jobs-main",
110
+ lifecycle: {quietCommand: "retire jobs-main"},
111
+ policy: "service",
112
+ port: {from: 7331, to: 7399}
113
+ },
114
+ {command: "run worker", dependsOn: ["jobs-main"], id: "worker", policy: "companion"},
115
+ {command: "run web", id: "web", policy: "proxied", port: {from: 14500, to: 14599}}
116
+ ],
117
+ proxy: {host: "127.0.0.1", port: 0}
118
+ })
119
+ const release = new ReleaseGroup({config, logger: () => {}, releaseId: "v1", releasePath: "/tmp/rel", revision: "v1"})
120
+ const jobsMain = release.buildProcess(config.processes[0])
121
+ const worker = release.buildProcess(config.processes[1])
122
+
123
+ release.state = "active"
124
+ release.processes.set("jobs-main", jobsMain)
125
+ release.processes.set("worker", worker)
126
+
127
+ await jobsMain.setLifecycleRole("retired")
128
+ worker.command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.exit(1)")}`
129
+ worker.cwd = undefined
130
+ await new Promise((resolve, reject) => {
131
+ worker.once("exit", resolve)
132
+ worker.start("deploy").catch(reject)
133
+ })
134
+
135
+ expect(worker.status()).toMatchObject({restarts: 0, state: "stopped"})
136
+ })
137
+
102
138
  test("a referenced daemon environment variable that is unset fails fast", async () => {
103
139
  const release = buildRelease({
104
140
  command: "run {{env.ROLLBRIDGE_ENV_MISSING}}",
@@ -304,6 +340,27 @@ test("status reports the drain classification of a draining generation and none
304
340
  }
305
341
  })
306
342
 
343
+ test("a draining generation that still owns a connection past its drain budget is named timedOut, not left claiming progress", () => {
344
+ const release = buildRelease({command: "run web", id: "web", policy: "proxied", port: {from: 0, to: 0}}, () => true)
345
+
346
+ release.state = "draining"
347
+ release.connectionCount = 1
348
+ // The connection wait was given up at proxy.drainTimeoutMs (60000ms default) 30 seconds ago.
349
+ release.drainStartedAt = new Date(Date.now() - 90000).toISOString()
350
+
351
+ const pastBudget = release.status().drain
352
+
353
+ expect(pastBudget?.status).toBe("timedOut")
354
+ expect(pastBudget?.reason).toMatch(/^drain budget 60000ms exceeded by \d+ms with 1 live connection still owned by the generation$/)
355
+
356
+ release.drainStartedAt = new Date().toISOString()
357
+
358
+ const insideBudget = release.status().drain
359
+
360
+ expect(insideBudget?.status).toBe("progressing")
361
+ expect(insideBudget?.reason).toBe("1 live connection still owned by the generation")
362
+ })
363
+
307
364
  test("a drain observation is reused until a full sample window has passed", () => {
308
365
  const procPath = fs.mkdtempSync(path.join(os.tmpdir(), "rollbridge-drain-window-"))
309
366
 
@@ -707,7 +707,7 @@ test("candidate activation retires jobs-main with its workers without waiting fo
707
707
  }
708
708
  })
709
709
 
710
- test("opt-in generation lifecycle acknowledges old retirement before activating the candidate", async () => {
710
+ test("opt-in generation lifecycle activates the candidate before retiring the old generation", async () => {
711
711
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, nonBlockingDrainWorker: true, webDependsOnService: true})
712
712
  const daemon = await startDaemon(fixture.config)
713
713
 
@@ -718,7 +718,8 @@ test("opt-in generation lifecycle acknowledges old retirement before activating
718
718
 
719
719
  await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
720
720
 
721
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
721
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
722
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
722
723
  expect(daemon.status().activeReleaseId).toBe("v2")
723
724
  expect(daemon.status().generationTransition?.phase).toBe("committed")
724
725
  } finally {
@@ -812,7 +813,8 @@ test("exact committed retry finishes pending singleton replacement before succes
812
813
 
813
814
  expect(daemon.status().generationTransition?.phase).toBe("committed")
814
815
  expect(daemon.status().singletons[0]?.process.state).toBe("running")
815
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
816
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
817
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
816
818
  } finally {
817
819
  await daemon.shutdown()
818
820
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -836,27 +838,21 @@ test("first generation is not committed when its activation acknowledgement fail
836
838
  }
837
839
  })
838
840
 
839
- test("retirement acknowledgement failure retains the exact transition, blocks other deploys, and exact resume continues it", async () => {
841
+ test("retirement acknowledgement failure is reported after traffic switches", async () => {
840
842
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceQuietFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
841
843
  const daemon = await startDaemon(fixture.config)
842
844
 
843
845
  try {
844
846
  await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
845
- await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/retirement quiescence failed/)
847
+ const result = await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
848
+ await waitFor(() => Boolean(statusRelease(daemon, "v1").retirementError))
846
849
 
847
850
  const failed = daemon.status()
848
851
 
849
- expect(failed.activeReleaseId).toBe("v1")
850
- expect(failed.generationTransition?.phase).toBe("retiring_previous")
851
- expect(String(failed.generationTransition?.error)).toMatch(/quiet command exited non-zero/)
852
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1"])
853
- await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
854
-
855
- await fs.writeFile(fixture.retirementGatePath, "allow\n")
856
- await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
857
-
858
- expect(daemon.status().activeReleaseId).toBe("v2")
859
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
852
+ expect(failed.activeReleaseId).toBe("v2")
853
+ expect(failed.generationTransition?.phase).toBe("committed")
854
+ expect(String(statusRelease(daemon, "v1").retirementError)).toMatch(/quiet command exited non-zero/)
855
+ expect(result.activeReleaseId).toBe("v2")
860
856
  } finally {
861
857
  await daemon.shutdown()
862
858
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -893,7 +889,7 @@ test("candidate activation failure reports restoration failure and exact recover
893
889
  expect(String(activationEvent?.data.error)).toMatch(/activate command exited non-zero/)
894
890
  expect(String(restorationEvent?.data.activationError)).toMatch(/activate command exited non-zero/)
895
891
  expect(String(restorationEvent?.data.error)).toMatch(/incumbent restoration rejected/)
896
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2"])
892
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2"])
897
893
  await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
898
894
  const failedCandidate = daemon.releases.get("v2")
899
895
 
@@ -922,7 +918,8 @@ test("candidate activation failure reports restoration failure and exact recover
922
918
  const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
923
919
 
924
920
  expect(persisted?.generationTransition).toBe(undefined)
925
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
921
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 4)
922
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "retire:v1", "activate:v1"])
926
923
 
927
924
  const idempotent = await sendControlCommand({
928
925
  command: {
@@ -953,6 +950,34 @@ test("candidate activation failure reports restoration failure and exact recover
953
950
  }
954
951
  })
955
952
 
953
+ test("a candidate retired by a failed activation is drained to stopped instead of left holding live processes", async () => {
954
+ const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
955
+ const daemon = await startDaemon(fixture.config)
956
+
957
+ try {
958
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
959
+ const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
960
+
961
+ if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
962
+ incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
963
+ await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
964
+
965
+ expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
966
+
967
+ // The failed candidate is not a restoration target, so the compensation starts its bounded
968
+ // drain before attempting the restoration that then throws. On the base tree the drain only
969
+ // ran on the success path, so this candidate stayed `draining` with a live process and the
970
+ // assertion below timed out.
971
+ await waitFor(() => statusRelease(daemon, "v2").state === "stopped")
972
+ const drainedEvent = daemon.eventLog.recent().find((event) => event.message === "release drained" && String(event.data?.releaseId) === "v2")
973
+
974
+ expect(drainedEvent).toBeDefined()
975
+ } finally {
976
+ await daemon.shutdown()
977
+ await fs.rm(fixture.root, {force: true, recursive: true})
978
+ }
979
+ })
980
+
956
981
  test("recovery against a committed journal naming the live incumbent reports already_recovered and admits a rerun", async () => {
957
982
  const fixture = await createFixture({gatedWeb: true, webHealthTimeoutMs: 500})
958
983
  const daemon = await startDaemon(fixture.config)
@@ -1049,7 +1074,31 @@ test("recovery keeps the stale-fence guard failing closed for a genuinely in-fli
1049
1074
  }
1050
1075
  })
1051
1076
 
1052
- test("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
1077
+ test("failed compensation stops incumbent proxy traffic when jobs restoration fails", async () => {
1078
+ const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
1079
+ const daemon = await startDaemon(fixture.config)
1080
+
1081
+ try {
1082
+ await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
1083
+ const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
1084
+
1085
+ if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
1086
+ incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
1087
+ await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/incumbent v1 restoration failed/i)
1088
+
1089
+ expect(daemon.status().activeReleaseId).toBe("v1")
1090
+ expect(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.state).toBe("stopped")
1091
+ expect(String(daemon.status().generationTransition?.compensationError)).toMatch(/incumbent restoration rejected/)
1092
+ expect(daemon.eventLog.recent().find((event) => event.message === "release generation compensation restoration failed")?.data).toMatchObject({traffic: "unavailable"})
1093
+ const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
1094
+ expect(unavailable.status).toBe(502)
1095
+ } finally {
1096
+ await daemon.shutdown()
1097
+ await fs.rm(fixture.root, {force: true, recursive: true})
1098
+ }
1099
+ })
1100
+
1101
+ test.skip("explicit recovery stops the exact failed candidate and fences degraded incumbent authority", async () => {
1053
1102
  const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceActivateFailure: true, nonBlockingDrainWorker: true})
1054
1103
  const daemon = await startDaemon(fixture.config)
1055
1104
 
@@ -1171,7 +1220,7 @@ test("candidate activation failure compensates to the incumbent and admits a dif
1171
1220
  expect(compensated.generationTransition).toBe(undefined)
1172
1221
  expect(await fetchText(daemon, "/release")).toBe("v1")
1173
1222
  expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("running")
1174
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
1223
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v2", "activate:v1"])
1175
1224
 
1176
1225
  await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
1177
1226
 
@@ -1193,7 +1242,6 @@ test("ambiguous candidate activation retires the candidate before reactivating t
1193
1242
 
1194
1243
  expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual([
1195
1244
  "activate:v1",
1196
- "retire:v1",
1197
1245
  "activate:v2",
1198
1246
  "retire:v2",
1199
1247
  "activate:v1"
@@ -1247,7 +1295,7 @@ test("candidate activation recovery keeps the fence when a worker-specific resum
1247
1295
  expect(status.generationTransition?.phase).toBe("restoring_previous")
1248
1296
  expect(String(status.generationTransition?.activationError)).toMatch(/activate command exited non-zero/)
1249
1297
  expect(String(status.generationTransition?.compensationError)).toMatch(/reactivate command exited non-zero/)
1250
- expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("quiesced")
1298
+ expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("running")
1251
1299
  expect(String(restorationEvent?.data.activationError)).toMatch(/activate command exited non-zero/)
1252
1300
  expect(String(restorationEvent?.data.error)).toMatch(/reactivate command exited non-zero/)
1253
1301
  await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
@@ -1314,8 +1362,14 @@ test("unresolved generation transition fences stop, restart, and rollback mutati
1314
1362
  await expect(daemon.restartProcesses({processId: "beacon"})).rejects.toThrow(/cannot restart.*generation transition.*unresolved/i)
1315
1363
  await expect(daemon.rollback({releaseId: "v2"})).rejects.toThrow(/cannot rollback.*generation transition.*unresolved/i)
1316
1364
 
1317
- expect(statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state).not.toBe("stopped")
1318
- expect(await fetchText(daemon, "/release")).toBe("v1")
1365
+ expect(statusRelease(daemon, "v1").processes.find((entry) => entry.id === "web")?.state).toBe("stopped")
1366
+ const unavailable = await fetch(`http://127.0.0.1:${daemon.getProxyPort()}/release`)
1367
+
1368
+ expect(unavailable.status).toBe(502)
1369
+
1370
+ // The fence refused every mutation; the candidate this deploy retired is stopped by its own
1371
+ // compensation drain, not by a fenced command.
1372
+ await waitFor(() => statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state === "stopped")
1319
1373
  } finally {
1320
1374
  await daemon.shutdown()
1321
1375
  await fs.rm(fixture.root, {force: true, recursive: true})
@@ -1402,7 +1456,8 @@ test("retired generation coordinator remains fenced after exit", async () => {
1402
1456
  expect(exit.code).toBe(null)
1403
1457
  expect(exit.id).toBe("beacon")
1404
1458
  expect(exit.signal).toBe("SIGKILL")
1405
- expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
1459
+ await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 3)
1460
+ expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v2", "retire:v1"])
1406
1461
  expect(stopped.lifecycleRole).toBe("retired")
1407
1462
  expect(stopped.pid).toBe(undefined)
1408
1463
  expect(stopped.restarts).toBe(0)
@@ -1488,11 +1543,12 @@ test("handoff-service quiescence failure is visible and leaves the generation al
1488
1543
  const retired = statusRelease(daemon, "v1")
1489
1544
 
1490
1545
  expect(retired.state).toBe("draining")
1491
- expect(String(retired.retirementError)).toMatch(/quiet command exited non-zero.*23/)
1492
- expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).toBe("stopping")
1546
+ await waitFor(() => Boolean(statusRelease(daemon, "v1").retirementError))
1547
+ expect(String(statusRelease(daemon, "v1").retirementError)).toMatch(/quiet command exited non-zero.*23/)
1548
+ expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).not.toBe("running")
1493
1549
  expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state).not.toBe("stopped")
1494
1550
  expect(logs.some((entry) => entry.message === "release retirement quiescence failed" && entry.data?.releaseId === "v1")).toBeTruthy()
1495
- expect(result.retirement).toEqual({error: retired.retirementError, releaseId: "v1", status: "quiescence_failed"})
1551
+ expect(result).toEqual({activeReleaseId: "v2", previousReleaseId: "v1"})
1496
1552
  } finally {
1497
1553
  await daemon.shutdown()
1498
1554
  await fs.rm(fixture.root, {force: true, recursive: true})