rollbridge 0.1.49 → 0.1.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -0
- package/README.md +5 -0
- package/changelog.d/20260909120000-velocious-testing.md +1 -0
- package/docs/cli.md +7 -1
- package/docs/generation-deployment-contract.md +9 -0
- package/eslint.config.js +8 -0
- package/package.json +3 -2
- package/src/cli.js +10 -2
- package/src/daemon.js +102 -12
- package/src/process-guardian.js +5 -1
- package/src/release-group.js +48 -1
- package/test/completion.test.js +18 -16
- package/test/config-examples.test.js +16 -17
- package/test/config-path.test.js +10 -11
- package/test/config-validation.test.js +163 -167
- package/test/control-protocol.test.js +75 -14
- package/test/daemon-bootstrap.test.js +104 -104
- package/test/daemon-runtime.test.js +17 -26
- package/test/doctor.test.js +51 -49
- package/test/event-log.test.js +13 -11
- package/test/guardian-client.test.js +160 -145
- package/test/health.test.js +6 -4
- package/test/logs.test.js +23 -17
- package/test/managed-process.test.js +96 -91
- package/test/owner-recovery.test.js +254 -239
- package/test/owner-replacement.test.js +228 -223
- package/test/package-metadata.test.js +48 -39
- package/test/port-allocator.test.js +13 -16
- package/test/predeploy-cleanup.test.js +12 -10
- package/test/process-memory.test.js +17 -15
- package/test/proxy.test.js +10 -8
- package/test/recover.test.js +30 -23
- package/test/release-group.test.js +16 -17
- package/test/release-retention.test.js +10 -8
- package/test/release-runtime-retention.test.js +31 -39
- package/test/rollbridge.test.js +388 -395
- package/test/shutdown-completion.test.js +51 -51
- package/test/state-store.test.js +10 -8
- package/test/system-ids.test.js +15 -13
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
-
import assert from "node:assert/strict"
|
|
4
3
|
import {spawn} from "node:child_process"
|
|
5
4
|
import {once} from "node:events"
|
|
6
5
|
import fs from "node:fs/promises"
|
|
7
6
|
import net from "node:net"
|
|
8
7
|
import os from "node:os"
|
|
9
8
|
import path from "node:path"
|
|
10
|
-
import test from "
|
|
9
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
11
10
|
import {fileURLToPath} from "node:url"
|
|
12
11
|
import {normalizeConfig} from "../src/config.js"
|
|
13
12
|
import {sendControlCommand} from "../src/control-client.js"
|
|
@@ -15,6 +14,8 @@ import RollbridgeDaemon from "../src/daemon.js"
|
|
|
15
14
|
import GuardianClient from "../src/guardian-client.js"
|
|
16
15
|
import {isProcessRunning, waitForProcessExit} from "./support/process.js"
|
|
17
16
|
|
|
17
|
+
describe("owner-recovery", () => {
|
|
18
|
+
|
|
18
19
|
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
|
19
20
|
const binPath = path.join(currentDir, "..", "bin", "rollbridge")
|
|
20
21
|
const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
|
|
@@ -72,21 +73,23 @@ test("external owner retirement releases guardian authority without losing its g
|
|
|
72
73
|
const v1 = recovered.releases.find(({releaseId}) => releaseId === "v1")
|
|
73
74
|
const v2 = recovered.releases.find(({releaseId}) => releaseId === "v2")
|
|
74
75
|
|
|
75
|
-
|
|
76
|
-
|
|
76
|
+
// the prestarted candidate must remain active
|
|
77
|
+
expect(recovered.activeReleaseId).toBe("v2")
|
|
78
|
+
expect(recovered.releaseReferences.sort((a, b) => a.releaseId.localeCompare(b.releaseId))).toEqual([
|
|
77
79
|
{releaseId: "v1", releasePath: v1Path},
|
|
78
80
|
{releaseId: "v2", releasePath: v2Path}
|
|
79
81
|
])
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
82
|
+
expect(v1?.state).toBe("draining")
|
|
83
|
+
expect(v1?.processes.find(({id}) => id === "worker")?.pid).toBe(v1WorkerPid)
|
|
84
|
+
expect(v1?.processes.find(({id}) => id === "worker")?.state).toBe("quiesced")
|
|
85
|
+
expect(v2?.state).toBe("active")
|
|
86
|
+
expect(v2?.processes.find(({id}) => id === "worker")?.pid).toBe(v2WorkerPid)
|
|
87
|
+
expect(v2?.processes.find(({id}) => id === "worker")?.state).toBe("running")
|
|
86
88
|
await waitForFile(path.join(v1Path, "drain-started"), 1000)
|
|
87
89
|
await fs.writeFile(path.join(v1Path, "drained"), "done\n")
|
|
88
90
|
await waitForProcessExit(v1WorkerPid, 1000)
|
|
89
|
-
|
|
91
|
+
// the active candidate worker must remain usable while the old generation drains
|
|
92
|
+
expect(isProcessRunning(v2WorkerPid)).toBe(true)
|
|
90
93
|
} finally {
|
|
91
94
|
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "drained"), "done\n").catch(() => {})))
|
|
92
95
|
await replacement?.shutdown().catch(() => {})
|
|
@@ -117,8 +120,8 @@ test("exact bootstrap restores the committed generation after external owner ret
|
|
|
117
120
|
...committed.singletons.map(({process}) => process.pid)
|
|
118
121
|
].filter((pid) => typeof pid === "number")
|
|
119
122
|
|
|
120
|
-
|
|
121
|
-
|
|
123
|
+
expect(committed.activeReleaseId).toBe("v2")
|
|
124
|
+
expect(committed.generationTransition?.phase).toBe("committed")
|
|
122
125
|
await retired.retireOwner({attestation: `sha256:${"a".repeat(64)}`})
|
|
123
126
|
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
124
127
|
await Promise.all(retiredPids.map((pid) => waitForProcessExit(pid, 3000)))
|
|
@@ -134,12 +137,10 @@ test("exact bootstrap restores the committed generation after external owner ret
|
|
|
134
137
|
const recoveredCandidate = recovered.releases.get("v2")
|
|
135
138
|
const recoveredService = recovered.services.get("beacon")
|
|
136
139
|
|
|
137
|
-
|
|
140
|
+
if (!recoveredCandidate || !recoveredService) throw new Error("Missing recovered candidate or service")
|
|
141
|
+
expect(recovered.singletons.get("singleton")).toBeTruthy()
|
|
138
142
|
recovered.serviceReleaseIds.set("beacon", "v1")
|
|
139
|
-
|
|
140
|
-
() => recovered?.assertCommittedBootstrapRecoveryReady(),
|
|
141
|
-
/service beacon belongs to retained release v1/
|
|
142
|
-
)
|
|
143
|
+
await expect(() => recovered?.assertCommittedBootstrapRecoveryReady()).toThrow(/service beacon belongs to retained release v1/)
|
|
143
144
|
recovered.serviceReleaseIds.set("beacon", "v2")
|
|
144
145
|
const activateGeneration = recoveredCandidate.activateGeneration.bind(recoveredCandidate)
|
|
145
146
|
const startService = recoveredService.start.bind(recoveredService)
|
|
@@ -150,7 +151,7 @@ test("exact bootstrap restores the committed generation after external owner ret
|
|
|
150
151
|
await activateGeneration()
|
|
151
152
|
}
|
|
152
153
|
recoveredService.start = async (...args) => {
|
|
153
|
-
|
|
154
|
+
expect(recovered?.generationTransition?.phase).toBe("restoring_committed")
|
|
154
155
|
recoveryOrder.push("service")
|
|
155
156
|
await startService(...args)
|
|
156
157
|
}
|
|
@@ -161,17 +162,19 @@ test("exact bootstrap restores the committed generation after external owner ret
|
|
|
161
162
|
await recovered.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
162
163
|
const active = recovered.status()
|
|
163
164
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
165
|
+
expect(active.activeReleaseId).toBe("v2")
|
|
166
|
+
expect(active.generationTransition?.phase).toBe("committed")
|
|
167
|
+
expect(active.releases.find(({releaseId}) => releaseId === "v1")?.state).toBe("draining")
|
|
168
|
+
expect(releaseProcessPid(active, "v1", "worker")).toBe(v1WorkerPid)
|
|
169
|
+
// the retained previous generation must keep draining
|
|
170
|
+
expect(isProcessRunning(v1WorkerPid)).toBe(true)
|
|
171
|
+
expect(active.releases.find(({releaseId}) => releaseId === "v2")?.state).toBe("active")
|
|
172
|
+
expect(active.releases.find(({releaseId}) => releaseId === "v2")?.processes.every(({pid, state}) => typeof pid === "number" && state === "running")).toBeTruthy()
|
|
173
|
+
expect(active.services.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
|
|
174
|
+
expect(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
|
|
175
|
+
// candidate activation must precede post-commit singleton completion
|
|
176
|
+
expect(recoveryOrder).toEqual(["service", "activate", "singleton"])
|
|
177
|
+
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
|
|
175
178
|
} finally {
|
|
176
179
|
if (recovered) {
|
|
177
180
|
const activeRecovery = recovered.status().activeReleaseId === "v2"
|
|
@@ -195,7 +198,7 @@ test("exact bootstrap restores a committed generation after its previous release
|
|
|
195
198
|
const worker = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
196
199
|
.find((processConfig) => processConfig.id === "worker")
|
|
197
200
|
|
|
198
|
-
|
|
201
|
+
if (!worker) throw new Error("Missing fixture worker")
|
|
199
202
|
const workerLifecycle = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (worker.lifecycle)
|
|
200
203
|
|
|
201
204
|
workerLifecycle.drainTimeoutMs = 500
|
|
@@ -219,7 +222,7 @@ test("exact bootstrap restores a committed generation after its previous release
|
|
|
219
222
|
.map(({pid}) => pid)
|
|
220
223
|
.filter((pid) => typeof pid === "number") || []
|
|
221
224
|
|
|
222
|
-
|
|
225
|
+
expect(committed.releases.some(({releaseId}) => releaseId === "v1")).toBe(false)
|
|
223
226
|
await retired.retireOwner({attestation: `sha256:${"d".repeat(64)}`})
|
|
224
227
|
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
225
228
|
await Promise.all(retiredPids.map((pid) => waitForProcessExit(pid, 3000)))
|
|
@@ -233,8 +236,8 @@ test("exact bootstrap restores a committed generation after its previous release
|
|
|
233
236
|
await recovered.start({exposeControl: false})
|
|
234
237
|
await recovered.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
235
238
|
|
|
236
|
-
|
|
237
|
-
|
|
239
|
+
expect(recovered.status().activeReleaseId).toBe("v2")
|
|
240
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
238
241
|
} finally {
|
|
239
242
|
if (recovered) {
|
|
240
243
|
const activeRecovery = recovered.status().activeReleaseId === "v2"
|
|
@@ -283,7 +286,7 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
|
|
|
283
286
|
await interrupted.updateGenerationTransition("restoring_committed")
|
|
284
287
|
const candidate = interrupted.releases.get("v2")
|
|
285
288
|
|
|
286
|
-
|
|
289
|
+
if (!candidate) throw new Error("Missing recovered candidate")
|
|
287
290
|
for (const processInstance of interrupted.services.values()) await processInstance.start("deploy")
|
|
288
291
|
await candidate.restartCommittedGeneration()
|
|
289
292
|
await interrupted.checkpointGenerationTransition()
|
|
@@ -291,8 +294,8 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
|
|
|
291
294
|
const restartedCandidatePids = restarted.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid)
|
|
292
295
|
const restartedServicePids = restarted.services.map(({process}) => process.pid)
|
|
293
296
|
|
|
294
|
-
|
|
295
|
-
|
|
297
|
+
expect(restarted.generationTransition?.phase).toBe("restoring_committed")
|
|
298
|
+
expect(restartedCandidatePids?.every((pid) => typeof pid === "number")).toBeTruthy()
|
|
296
299
|
await interrupted.retireCommittedOwner(undefined)
|
|
297
300
|
interrupted.guardian?.disconnect()
|
|
298
301
|
|
|
@@ -300,14 +303,14 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
|
|
|
300
303
|
await recovered.start({exposeControl: false})
|
|
301
304
|
const active = recovered.status()
|
|
302
305
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
306
|
+
expect(active.activeReleaseId).toBe("v2")
|
|
307
|
+
expect(active.generationTransition?.phase).toBe("committed")
|
|
308
|
+
expect(active.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid)).toEqual(restartedCandidatePids)
|
|
309
|
+
expect(active.services.map(({process}) => process.pid)).toEqual(restartedServicePids)
|
|
310
|
+
expect(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
|
|
311
|
+
expect(releaseProcessPid(active, "v1", "worker")).toBe(v1WorkerPid)
|
|
312
|
+
expect(isProcessRunning(v1WorkerPid)).toBe(true)
|
|
313
|
+
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
|
|
311
314
|
} finally {
|
|
312
315
|
if (recovered) {
|
|
313
316
|
const shutdown = recovered.shutdown()
|
|
@@ -358,39 +361,27 @@ test("committed bootstrap tuple mismatches fail closed without singletons", asyn
|
|
|
358
361
|
await recovered.start({exposeControl: false})
|
|
359
362
|
const owner = recovered
|
|
360
363
|
|
|
361
|
-
await
|
|
362
|
-
|
|
363
|
-
/only the exact same release, path, revision, and config authority/u
|
|
364
|
-
)
|
|
365
|
-
await assert.rejects(
|
|
366
|
-
() => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "wrong"}),
|
|
367
|
-
/only the exact same release, path, revision, and config authority/u
|
|
368
|
-
)
|
|
364
|
+
await expect(() => owner.deploy({releaseId: "v2", releasePath: wrongPath, revision: "v2"})).toThrow(/only the exact same release, path, revision, and config authority/u)
|
|
365
|
+
await expect(() => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "wrong"})).toThrow(/only the exact same release, path, revision, and config authority/u)
|
|
369
366
|
const changedConfig = structuredClone(fixture.config)
|
|
370
367
|
const jobs = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (changedConfig.processes)
|
|
371
368
|
.find((processConfig) => processConfig.id === "jobs")
|
|
372
369
|
|
|
373
|
-
|
|
370
|
+
if (!jobs) throw new Error("Missing fixture jobs service")
|
|
374
371
|
jobs.env = {.../** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobs.env), MISMATCHED_AUTHORITY: "true"}
|
|
375
372
|
await writeConfig(fixture.configPath, changedConfig)
|
|
376
|
-
await
|
|
377
|
-
() => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"}),
|
|
378
|
-
/only the exact same release, path, revision, and config authority/u
|
|
379
|
-
)
|
|
373
|
+
await expect(() => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})).toThrow(/only the exact same release, path, revision, and config authority/u)
|
|
380
374
|
await writeConfig(fixture.configPath, fixture.config)
|
|
381
|
-
await
|
|
382
|
-
() => owner.deploy({releaseId: "wrong", releasePath: wrongPath, revision: "wrong"}),
|
|
383
|
-
/only the exact same release, path, revision, and config authority/u
|
|
384
|
-
)
|
|
375
|
+
await expect(() => owner.deploy({releaseId: "wrong", releasePath: wrongPath, revision: "wrong"})).toThrow(/only the exact same release, path, revision, and config authority/u)
|
|
385
376
|
const preserved = owner.status()
|
|
386
377
|
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
378
|
+
expect(preserved.activeReleaseId).toBe(null)
|
|
379
|
+
expect(preserved.generationTransition?.candidateReleaseId).toBe("v2")
|
|
380
|
+
expect(preserved.generationTransition?.phase).toBe("committed")
|
|
381
|
+
expect(preserved.releases.find(({releaseId}) => releaseId === "v2")?.state).toBe("draining")
|
|
382
|
+
expect(releaseProcessPid(preserved, "v1", "worker")).toBe(v1WorkerPid)
|
|
383
|
+
expect(isProcessRunning(v1WorkerPid)).toBe(true)
|
|
384
|
+
expect(preserved.releases.some(({releaseId}) => releaseId === "wrong")).toBe(false)
|
|
394
385
|
|
|
395
386
|
await owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
396
387
|
await Promise.all([
|
|
@@ -399,8 +390,8 @@ test("committed bootstrap tuple mismatches fail closed without singletons", asyn
|
|
|
399
390
|
])
|
|
400
391
|
const afterIntentionalStop = await owner.deploy({releaseId: "wrong", releasePath: wrongPath, revision: "wrong"})
|
|
401
392
|
|
|
402
|
-
|
|
403
|
-
|
|
393
|
+
expect(afterIntentionalStop.activeReleaseId).toBe("wrong")
|
|
394
|
+
expect(owner.status().activeReleaseId).toBe("wrong")
|
|
404
395
|
} finally {
|
|
405
396
|
if (recovered) {
|
|
406
397
|
const activeReleaseId = recovered.status().activeReleaseId
|
|
@@ -439,8 +430,8 @@ test("replacement owner reconstructs one active and two draining generations aft
|
|
|
439
430
|
|
|
440
431
|
const before = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
441
432
|
|
|
442
|
-
|
|
443
|
-
|
|
433
|
+
expect(before.activeReleaseId).toBe("v3")
|
|
434
|
+
expect(before.releaseReferences.map((/** @type {{releaseId: string}} */ reference) => reference.releaseId)).toEqual(["v1", "v2", "v3"])
|
|
444
435
|
const generationEndpoints = before.releases.map((release) => ({
|
|
445
436
|
jobsPort: release.ports.jobs,
|
|
446
437
|
jobsState: release.processes.find((processStatus) => processStatus.id === "jobs")?.state,
|
|
@@ -448,7 +439,7 @@ test("replacement owner reconstructs one active and two draining generations aft
|
|
|
448
439
|
state: release.state
|
|
449
440
|
}))
|
|
450
441
|
|
|
451
|
-
|
|
442
|
+
expect({value: new Set(generationEndpoints.map(({jobsPort}) => jobsPort)).size, context: JSON.stringify(generationEndpoints)}).toMatchObject({value: 3})
|
|
452
443
|
owner.kill("SIGKILL")
|
|
453
444
|
await once(owner, "exit")
|
|
454
445
|
|
|
@@ -457,23 +448,24 @@ test("replacement owner reconstructs one active and two draining generations aft
|
|
|
457
448
|
|
|
458
449
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
459
450
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
451
|
+
expect(recovered.activeReleaseId).toBe("v3")
|
|
452
|
+
expect(recovered.releases.map((/** @type {{state: string}} */ release) => release.state)).toEqual(["draining", "draining", "active"])
|
|
453
|
+
expect(recovered.releaseReferences.map((/** @type {{releaseId: string}} */ reference) => reference.releaseId)).toEqual(["v1", "v2", "v3"])
|
|
454
|
+
expect(recovered.releases.map((release) => release.ports.jobs)).toEqual(before.releases.map((release) => release.ports.jobs))
|
|
455
|
+
expect(recovered.services[0]?.process.pid).toBe(before.services[0]?.process.pid)
|
|
456
|
+
expect(recovered.singletons[0]?.process.pid).toBe(before.singletons[0]?.process.pid)
|
|
466
457
|
|
|
467
458
|
const v4Path = path.join(fixture.root, "v4")
|
|
468
459
|
await fs.mkdir(v4Path)
|
|
469
460
|
const v4Gate = spawn("mkfifo", [path.join(v4Path, "worker.fifo")])
|
|
470
|
-
|
|
461
|
+
expect((await once(v4Gate, "exit"))[0]).toBe(0)
|
|
471
462
|
await sendControlCommand({command: {command: "deploy", releaseId: "v4", releasePath: v4Path, revision: "v4"}, path: fixture.socketPath})
|
|
472
|
-
|
|
463
|
+
// new work must progress while old generations remain retained
|
|
464
|
+
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v4")
|
|
473
465
|
|
|
474
466
|
await Promise.all(["v1", "v2"].map((releaseId) => fs.writeFile(path.join(fixture.root, releaseId, "worker.fifo"), "drained\n")))
|
|
475
467
|
const afterDrain = await waitForState(fixture.statePath, (state) => state.releaseReferences?.map((/** @type {{releaseId: string}} */ reference) => reference.releaseId).join(",") === "v3,v4")
|
|
476
|
-
|
|
468
|
+
expect(afterDrain.releaseReferences.map((/** @type {{releaseId: string}} */ reference) => reference.releaseId)).toEqual(["v3", "v4"])
|
|
477
469
|
|
|
478
470
|
await fs.writeFile(path.join(fixture.root, "v3", "worker.fifo"), "drained\n")
|
|
479
471
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
@@ -516,9 +508,9 @@ test("guardian restarts an abruptly exited daemon without replacing managed proc
|
|
|
516
508
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
517
509
|
|
|
518
510
|
recoveredDaemonPid = recovered.daemonPid
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
511
|
+
expect(recovered.daemonPid).not.toBe(before.daemonPid)
|
|
512
|
+
expect(recovered.activeReleaseId).toBe("v1")
|
|
513
|
+
expect(releaseProcessPid(recovered, "v1", "worker")).toBe(workerPid)
|
|
522
514
|
|
|
523
515
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
524
516
|
|
|
@@ -541,7 +533,7 @@ test("guardian recovers a persistent service after its final active release stop
|
|
|
541
533
|
const jobs = processes.find((processConfig) => processConfig.id === "jobs")
|
|
542
534
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
543
535
|
|
|
544
|
-
|
|
536
|
+
if (!jobs || !worker) throw new Error("Missing fixture jobs service or worker")
|
|
545
537
|
jobs.port = {from: 17000, to: 17001}
|
|
546
538
|
worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
547
539
|
fixture.config.processes = processes.filter((processConfig) => processConfig.id !== "singleton")
|
|
@@ -558,11 +550,11 @@ test("guardian recovers a persistent service after its final active release stop
|
|
|
558
550
|
const before = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
559
551
|
const servicePid = before.services[0]?.process.pid
|
|
560
552
|
|
|
561
|
-
|
|
562
|
-
|
|
553
|
+
expect(before.activeReleaseId).toBe(null)
|
|
554
|
+
expect(typeof servicePid).toBe("number")
|
|
563
555
|
const persisted = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
|
|
564
556
|
|
|
565
|
-
|
|
557
|
+
expect(persisted.serviceReleaseIds?.beacon).toBe("v1")
|
|
566
558
|
const recoveredListenerLog = waitForLog(owner, "control socket listening", {allowChildExit: true})
|
|
567
559
|
|
|
568
560
|
owner.kill("SIGKILL")
|
|
@@ -571,16 +563,16 @@ test("guardian recovers a persistent service after its final active release stop
|
|
|
571
563
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
572
564
|
|
|
573
565
|
recoveredDaemonPid = recovered.daemonPid
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
566
|
+
expect(recovered.activeReleaseId).toBe(null)
|
|
567
|
+
expect(recovered.services[0]?.process.pid).toBe(servicePid)
|
|
568
|
+
expect(recovered.releaseReferences).toEqual([{releaseId: "v1", releasePath}])
|
|
577
569
|
const nextReleasePath = await prepareRelease(fixture.root, "v2")
|
|
578
570
|
|
|
579
571
|
await sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: nextReleasePath, revision: "v2"}, path: fixture.socketPath})
|
|
580
572
|
const finalReleasePath = await prepareRelease(fixture.root, "v3")
|
|
581
573
|
|
|
582
574
|
await sendControlCommand({command: {command: "deploy", releaseId: "v3", releasePath: finalReleasePath, revision: "v3"}, path: fixture.socketPath})
|
|
583
|
-
|
|
575
|
+
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v3")
|
|
584
576
|
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
585
577
|
} finally {
|
|
586
578
|
await killChild(owner)
|
|
@@ -607,17 +599,17 @@ test("owner state omits a new persistent service until its defining release is r
|
|
|
607
599
|
const guardianState = /** @type {Record<string, import("../src/json.js").JsonValue> | undefined} */ (await owner.guardian?.ownerState())
|
|
608
600
|
const guardianSnapshot = /** @type {RecoveryState | undefined} */ (guardianState?.snapshot)
|
|
609
601
|
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
602
|
+
expect(persisted.releases).toEqual([])
|
|
603
|
+
expect(persisted.services).toEqual([])
|
|
604
|
+
expect(persisted.serviceReleaseIds).toEqual({})
|
|
605
|
+
expect(guardianSnapshot?.releases).toEqual([])
|
|
606
|
+
expect(guardianSnapshot?.services).toEqual([])
|
|
607
|
+
expect(guardianState?.serviceReleaseIds).toEqual({})
|
|
616
608
|
|
|
617
609
|
await fs.writeFile(path.join(releasePath, "jobs.bind"), "ready\n")
|
|
618
610
|
await deploy
|
|
619
|
-
|
|
620
|
-
|
|
611
|
+
expect(owner.status().services.find(({id}) => id === "beacon")?.process.state).toBe("running")
|
|
612
|
+
expect(owner.serviceReleaseIds.get("beacon")).toBe("v1")
|
|
621
613
|
} finally {
|
|
622
614
|
await fs.writeFile(path.join(releasePath, "jobs.bind"), "ready\n").catch(() => {})
|
|
623
615
|
await deploy?.catch(() => {})
|
|
@@ -636,7 +628,7 @@ test("owner recovery accepts released format-2 state without journal revision or
|
|
|
636
628
|
const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
637
629
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
638
630
|
|
|
639
|
-
|
|
631
|
+
if (!worker) throw new Error("Missing fixture worker")
|
|
640
632
|
worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
641
633
|
await writeConfig(fixture.configPath, fixture.config)
|
|
642
634
|
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -666,9 +658,9 @@ test("owner recovery accepts released format-2 state without journal revision or
|
|
|
666
658
|
recovered = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
667
659
|
await recovered.start()
|
|
668
660
|
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
661
|
+
expect(recovered.status().activeReleaseId).toBe("v1")
|
|
662
|
+
expect(recovered.status().generationTransition?.journalRevision).toBe(undefined)
|
|
663
|
+
expect(recovered.serviceReleaseIds.get("beacon")).toBe("v1")
|
|
672
664
|
} finally {
|
|
673
665
|
if (recovered) await recovered.shutdown().catch(() => {})
|
|
674
666
|
else await owner.shutdown().catch(() => {})
|
|
@@ -688,7 +680,7 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
|
|
|
688
680
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
689
681
|
const lifecycle = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobs?.lifecycle)
|
|
690
682
|
|
|
691
|
-
|
|
683
|
+
if (!jobs || !worker) throw new Error("Missing fixture jobs service or worker")
|
|
692
684
|
jobs.gracefulStopMs = 5000
|
|
693
685
|
lifecycle.quietCommand = `printf 'waiting\n' >> ${JSON.stringify(retirementWaitingPath)}; while [ ! -f ${JSON.stringify(retirementGatePath)} ]; do sleep 0.02; done; printf 'retire:%s\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(fixture.lifecycleLogPath)}`
|
|
694
686
|
worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
@@ -716,28 +708,19 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
|
|
|
716
708
|
const recovering = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
717
709
|
const guardianPid = JSON.parse(await fs.readFile(fixture.statePath, "utf8")).recovery?.guardian?.pid
|
|
718
710
|
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
await
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
)
|
|
727
|
-
await assert.rejects(
|
|
728
|
-
sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}),
|
|
729
|
-
/Cannot shut down while generation transition recovery is in progress/
|
|
730
|
-
)
|
|
731
|
-
await assert.rejects(
|
|
732
|
-
sendControlCommand({command: {attestation: `sha256:${"a".repeat(64)}`, command: "retire-owner"}, path: fixture.socketPath}),
|
|
733
|
-
/Cannot retire owner while generation transition recovery is in progress/
|
|
734
|
-
)
|
|
711
|
+
expect(recovering.daemonPid).toBe(recoveredDaemonPid)
|
|
712
|
+
expect(recovering.ownerRecovery?.ready).toBe(true)
|
|
713
|
+
expect(recovering.generationTransition?.phase).toBe("retiring_previous")
|
|
714
|
+
expect(typeof guardianPid).toBe("number")
|
|
715
|
+
await expect(() => sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: fixture.socketPath})).toThrow(/Another owner mutation/)
|
|
716
|
+
await expect(() => sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})).toThrow(/Cannot shut down while generation transition recovery is in progress/)
|
|
717
|
+
await expect(() => sendControlCommand({command: {attestation: `sha256:${"a".repeat(64)}`, command: "retire-owner"}, path: fixture.socketPath})).toThrow(/Cannot retire owner while generation transition recovery is in progress/)
|
|
735
718
|
|
|
736
719
|
process.kill(recoveredDaemonPid, "SIGTERM")
|
|
737
720
|
await fs.writeFile(retirementGatePath, "release retirement\n")
|
|
738
721
|
await waitForProcessExit(recoveredDaemonPid, 5000)
|
|
739
722
|
await waitForProcessExit(guardianPid, 5000)
|
|
740
|
-
|
|
723
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v1", "activate:v2", "retire:v2"])
|
|
741
724
|
} finally {
|
|
742
725
|
await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
|
|
743
726
|
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
|
|
@@ -758,11 +741,8 @@ test("owner recovery preserves a completed activation compensation without repla
|
|
|
758
741
|
const v2Path = await prepareRelease(fixture.root, "v2")
|
|
759
742
|
|
|
760
743
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})
|
|
761
|
-
await
|
|
762
|
-
|
|
763
|
-
/activate command exited non-zero/
|
|
764
|
-
)
|
|
765
|
-
assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
|
|
744
|
+
await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})).toThrow(/activate command exited non-zero/)
|
|
745
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
|
|
766
746
|
|
|
767
747
|
owner.kill("SIGKILL")
|
|
768
748
|
await once(owner, "exit")
|
|
@@ -771,9 +751,10 @@ test("owner recovery preserves a completed activation compensation without repla
|
|
|
771
751
|
|
|
772
752
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
773
753
|
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
754
|
+
expect(recovered.activeReleaseId).toBe("v1")
|
|
755
|
+
expect(recovered.generationTransition).toBe(undefined)
|
|
756
|
+
// owner recovery must not replay completed compensation hooks
|
|
757
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
|
|
777
758
|
|
|
778
759
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
779
760
|
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
|
|
@@ -792,7 +773,7 @@ test("owner recovery replays one journaled ambiguous first-generation activation
|
|
|
792
773
|
try {
|
|
793
774
|
await waitForLog(owner, "control socket listening")
|
|
794
775
|
const v1Path = await prepareRelease(fixture.root, "v1")
|
|
795
|
-
await
|
|
776
|
+
await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})).toThrow()
|
|
796
777
|
owner.kill("SIGKILL")
|
|
797
778
|
await once(owner, "exit")
|
|
798
779
|
|
|
@@ -811,9 +792,9 @@ test("owner recovery replays one journaled ambiguous first-generation activation
|
|
|
811
792
|
|
|
812
793
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
813
794
|
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
795
|
+
expect(recovered.activeReleaseId).toBe("v1")
|
|
796
|
+
expect(recovered.generationTransition?.phase).toBe("committed")
|
|
797
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1"])
|
|
817
798
|
|
|
818
799
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
819
800
|
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
@@ -830,7 +811,7 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
830
811
|
const initialProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
831
812
|
const initialWorker = initialProcesses.find((processConfig) => processConfig.id === "worker")
|
|
832
813
|
|
|
833
|
-
|
|
814
|
+
if (!initialWorker) throw new Error("Missing initial fixture worker")
|
|
834
815
|
initialWorker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
835
816
|
await writeConfig(fixture.configPath, fixture.config)
|
|
836
817
|
const initialConfig = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -847,7 +828,7 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
847
828
|
const processes = Array.isArray(changedConfig.processes) ? changedConfig.processes : []
|
|
848
829
|
const jobsValue = processes.find((processConfig) => processConfig && typeof processConfig === "object" && !Array.isArray(processConfig) && processConfig.id === "jobs")
|
|
849
830
|
|
|
850
|
-
|
|
831
|
+
expect(jobsValue && typeof jobsValue === "object" && !Array.isArray(jobsValue)).toBeTruthy()
|
|
851
832
|
const jobs = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobsValue)
|
|
852
833
|
|
|
853
834
|
jobs.env = {.../** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobs.env), RELEASE_CONFIG_AUTHORITY: "v2"}
|
|
@@ -857,7 +838,7 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
857
838
|
// retirement refreshes the previous generation's guardian definition.
|
|
858
839
|
owner.resumeGenerationTransition = async () => ({pausedAt: "candidate_ready"})
|
|
859
840
|
await owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
860
|
-
|
|
841
|
+
expect(owner.status().generationTransition?.phase).toBe("candidate_ready")
|
|
861
842
|
|
|
862
843
|
await owner.retireCommittedOwner(undefined)
|
|
863
844
|
owner.guardian?.disconnect()
|
|
@@ -871,9 +852,9 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
871
852
|
recovered = new RollbridgeDaemon({config: normalizeConfig(changedConfig, fixture.configPath), configPath: fixture.configPath, logger: () => {}})
|
|
872
853
|
await recovered.start()
|
|
873
854
|
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
855
|
+
expect(recovered.status().activeReleaseId).toBe("v2")
|
|
856
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
857
|
+
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2"])
|
|
877
858
|
} finally {
|
|
878
859
|
if (recovered) await recovered.shutdown().catch(() => {})
|
|
879
860
|
await stopFixtureGuardian(fixture.statePath)
|
|
@@ -909,25 +890,26 @@ test("owner recovery reconstructs a stopped release that still owns a committed-
|
|
|
909
890
|
if (owner.pendingWrite) await owner.pendingWrite
|
|
910
891
|
const pending = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
|
|
911
892
|
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
893
|
+
expect(pending.activeReleaseId).toBe("v2")
|
|
894
|
+
expect(pending.generationTransition?.phase).toBe("committed_pending")
|
|
895
|
+
expect(pending.serviceReleaseIds?.beacon).toBe("v2")
|
|
896
|
+
expect(pending.singletonReleaseIds?.singleton).toBe("v1")
|
|
916
897
|
const stoppedRelease = pending.releases.find((release) => release.releaseId === "v1" && release.state === "stopped")
|
|
917
898
|
|
|
918
|
-
|
|
919
|
-
|
|
899
|
+
if (!stoppedRelease) throw new Error("Missing stopped singleton owner release")
|
|
900
|
+
expect(pending.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2"])
|
|
920
901
|
await owner.retireCommittedOwner(undefined)
|
|
921
902
|
owner.guardian?.disconnect()
|
|
922
903
|
|
|
923
904
|
recovered = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
924
905
|
await recovered.start()
|
|
925
906
|
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
907
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
908
|
+
expect(recovered.singletonReleaseIds.get("singleton")).toBe("v2")
|
|
909
|
+
// the stopped singleton owner may be pruned after replacement commits
|
|
910
|
+
expect(!recovered.releases.has("v1")).toBeTruthy()
|
|
911
|
+
expect(recovered.portReservations.has(stoppedRelease.ports.jobs)).toBe(false)
|
|
912
|
+
expect(recovered.portReservations.has(stoppedRelease.ports.web)).toBe(false)
|
|
931
913
|
} finally {
|
|
932
914
|
replacementPause.continue()
|
|
933
915
|
await deployPromise?.catch(() => {})
|
|
@@ -957,7 +939,7 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
957
939
|
const initialProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
958
940
|
const initialSingleton = initialProcesses.find((processConfig) => processConfig.id === "singleton")
|
|
959
941
|
|
|
960
|
-
|
|
942
|
+
if (!initialSingleton) throw new Error("Missing initial singleton")
|
|
961
943
|
initialSingleton.env = {SINGLETON_CONFIG_AUTHORITY: "v1"}
|
|
962
944
|
await writeConfig(fixture.configPath, fixture.config)
|
|
963
945
|
const initialConfig = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -977,7 +959,7 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
977
959
|
const changedProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (changedConfig.processes)
|
|
978
960
|
const changedSingleton = changedProcesses.find((processConfig) => processConfig.id === "singleton")
|
|
979
961
|
|
|
980
|
-
|
|
962
|
+
if (!changedSingleton) throw new Error("Missing changed singleton")
|
|
981
963
|
changedSingleton.env = {SINGLETON_CONFIG_AUTHORITY: "v2"}
|
|
982
964
|
await writeConfig(fixture.configPath, changedConfig)
|
|
983
965
|
deployPromise = owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
@@ -985,9 +967,10 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
985
967
|
await replacementPause.started
|
|
986
968
|
const pending = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
|
|
987
969
|
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
970
|
+
expect(pending.activeReleaseId).toBe("v2")
|
|
971
|
+
expect(pending.generationTransition?.phase).toBe("committed_pending")
|
|
972
|
+
expect(pending.singletonReleaseIds?.singleton).toBe("v1")
|
|
973
|
+
await owner.persistState({throwOnError: true})
|
|
991
974
|
await owner.retireCommittedOwner(undefined)
|
|
992
975
|
owner.guardian?.disconnect()
|
|
993
976
|
|
|
@@ -996,9 +979,9 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
996
979
|
recovered = new RollbridgeDaemon({config: nextConfig, configPath: fixture.configPath, logger: () => {}})
|
|
997
980
|
await recovered.start()
|
|
998
981
|
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
982
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
983
|
+
expect(recovered.singletonReleaseIds.get("singleton")).toBe("v2")
|
|
984
|
+
expect(recovered.singletons.get("singleton")?.env.SINGLETON_CONFIG_AUTHORITY).toBe("v2")
|
|
1002
985
|
} finally {
|
|
1003
986
|
replacementPause.continue()
|
|
1004
987
|
await deployPromise?.catch(() => {})
|
|
@@ -1025,7 +1008,7 @@ test("owner recovery retains a stopped previous release until committed-pending
|
|
|
1025
1008
|
const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
1026
1009
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
1027
1010
|
|
|
1028
|
-
|
|
1011
|
+
if (!worker) throw new Error("Missing fixture worker")
|
|
1029
1012
|
worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
1030
1013
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1031
1014
|
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -1060,16 +1043,16 @@ test("owner recovery retains a stopped previous release until committed-pending
|
|
|
1060
1043
|
await replacementComplete
|
|
1061
1044
|
const pending = await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed_pending" && state.releases?.some((release) => release.releaseId === "v1" && release.state === "stopped"), AbortSignal.timeout(5000))
|
|
1062
1045
|
|
|
1063
|
-
|
|
1064
|
-
|
|
1046
|
+
expect(pending.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2"])
|
|
1047
|
+
expect(pending.singletonReleaseIds?.singleton).toBe("v2")
|
|
1065
1048
|
await owner.retireCommittedOwner(undefined)
|
|
1066
1049
|
owner.guardian?.disconnect()
|
|
1067
1050
|
|
|
1068
1051
|
recovered = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
1069
1052
|
await recovered.start()
|
|
1070
1053
|
|
|
1071
|
-
|
|
1072
|
-
|
|
1054
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
1055
|
+
expect(recovered.status().activeReleaseId).toBe("v2")
|
|
1073
1056
|
} finally {
|
|
1074
1057
|
continueReplacement()
|
|
1075
1058
|
await deployPromise?.catch(() => {})
|
|
@@ -1103,7 +1086,7 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
|
|
|
1103
1086
|
const changedProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (changedConfig.processes)
|
|
1104
1087
|
const changedJobs = changedProcesses.find((processConfig) => processConfig.id === "jobs")
|
|
1105
1088
|
|
|
1106
|
-
|
|
1089
|
+
if (!changedJobs) throw new Error("Missing changed jobs service")
|
|
1107
1090
|
changedJobs.env = {.../** @type {Record<string, import("../src/json.js").JsonValue>} */ (changedJobs.env), RELEASE_CONFIG_AUTHORITY: "v2"}
|
|
1108
1091
|
await writeConfig(fixture.configPath, changedConfig)
|
|
1109
1092
|
const interruptedDeploy = sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})
|
|
@@ -1120,9 +1103,10 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
|
|
|
1120
1103
|
await waitForLog(owner, "control socket listening")
|
|
1121
1104
|
await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed", AbortSignal.timeout(5000))
|
|
1122
1105
|
await recoverySettled
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1106
|
+
// ambiguous retirement must replay exactly once
|
|
1107
|
+
expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(2)
|
|
1108
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v1", "activate:v2"])
|
|
1109
|
+
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v2")
|
|
1126
1110
|
|
|
1127
1111
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
1128
1112
|
|
|
@@ -1146,7 +1130,7 @@ test("owner recovery rejects config identity mismatch without changing the valid
|
|
|
1146
1130
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
|
|
1147
1131
|
const status = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1148
1132
|
workerPid = status.releases[0]?.processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
1149
|
-
|
|
1133
|
+
expect(workerPid).toBeTruthy()
|
|
1150
1134
|
const validState = await fs.readFile(fixture.statePath, "utf8")
|
|
1151
1135
|
|
|
1152
1136
|
owner.kill("SIGKILL")
|
|
@@ -1155,9 +1139,9 @@ test("owner recovery rejects config identity mismatch without changing the valid
|
|
|
1155
1139
|
|
|
1156
1140
|
const rejected = await runDaemon(fixture.configPath)
|
|
1157
1141
|
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1142
|
+
expect(rejected.code).not.toBe(0)
|
|
1143
|
+
expect(rejected.output).toMatch(/authority does not match/)
|
|
1144
|
+
expect(await fs.readFile(fixture.statePath, "utf8")).toBe(validState)
|
|
1161
1145
|
|
|
1162
1146
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1163
1147
|
owner = spawnDaemon(fixture.configPath)
|
|
@@ -1187,7 +1171,7 @@ test("failed recovery bootstrap keeps the reconstructed active generation servin
|
|
|
1187
1171
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
|
|
1188
1172
|
const status = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1189
1173
|
workerPid = status.releases[0]?.processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
1190
|
-
|
|
1174
|
+
expect(workerPid).toBeTruthy()
|
|
1191
1175
|
|
|
1192
1176
|
owner.kill("SIGKILL")
|
|
1193
1177
|
await once(owner, "exit")
|
|
@@ -1197,9 +1181,9 @@ test("failed recovery bootstrap keeps the reconstructed active generation servin
|
|
|
1197
1181
|
const preserved = await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})
|
|
1198
1182
|
const events = await sendControlCommand({command: {command: "events"}, path: fixture.socketPath})
|
|
1199
1183
|
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1184
|
+
expect(preserved.activeReleaseId).toBe("v1")
|
|
1185
|
+
expect(preserved.releaseReferences).toEqual([{releaseId: "v1", releasePath}])
|
|
1186
|
+
expect(Array.isArray(events.events) && events.events.some((event) => event && typeof event === "object" && "message" in event && event.message === "bootstrap activation failed")).toBeTruthy()
|
|
1203
1187
|
|
|
1204
1188
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
1205
1189
|
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
@@ -1227,7 +1211,7 @@ test("owner recovery repairs a partial public snapshot from committed guardian s
|
|
|
1227
1211
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
|
|
1228
1212
|
const status = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1229
1213
|
workerPid = status.releases[0]?.processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
1230
|
-
|
|
1214
|
+
expect(workerPid).toBeTruthy()
|
|
1231
1215
|
const validState = JSON.parse(await fs.readFile(fixture.statePath, "utf8"))
|
|
1232
1216
|
|
|
1233
1217
|
owner.kill("SIGKILL")
|
|
@@ -1243,9 +1227,9 @@ test("owner recovery repairs a partial public snapshot from committed guardian s
|
|
|
1243
1227
|
recoveredDaemonPid = repairedState.daemonPid
|
|
1244
1228
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1245
1229
|
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1230
|
+
expect(recovered.activeReleaseId).toBe("v1")
|
|
1231
|
+
expect(releaseProcessPid(recovered, "v1", "worker")).toBe(workerPid)
|
|
1232
|
+
expect(JSON.parse(await fs.readFile(fixture.statePath, "utf8"))).not.toEqual(partialState)
|
|
1249
1233
|
|
|
1250
1234
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
1251
1235
|
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
@@ -1274,7 +1258,7 @@ test("concurrent same-authority replacements converge on one fenced owner", asyn
|
|
|
1274
1258
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
|
|
1275
1259
|
const status = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1276
1260
|
workerPid = status.releases[0]?.processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
1277
|
-
|
|
1261
|
+
expect(workerPid).toBeTruthy()
|
|
1278
1262
|
|
|
1279
1263
|
owner.kill("SIGKILL")
|
|
1280
1264
|
await once(owner, "exit")
|
|
@@ -1289,8 +1273,9 @@ test("concurrent same-authority replacements converge on one fenced owner", asyn
|
|
|
1289
1273
|
const loser = winner === owner ? secondContender : owner
|
|
1290
1274
|
const [loserCode] = loser.exitCode === null && loser.signalCode === null ? await once(loser, "exit") : [loser.exitCode]
|
|
1291
1275
|
|
|
1292
|
-
|
|
1293
|
-
|
|
1276
|
+
// fenced loser must attest the matching winner and exit successfully
|
|
1277
|
+
expect(loserCode).toBe(0)
|
|
1278
|
+
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v1")
|
|
1294
1279
|
|
|
1295
1280
|
owner = winner
|
|
1296
1281
|
contender = undefined
|
|
@@ -1321,20 +1306,20 @@ test("fresh guardian identity remains recoverable when proxy startup fails", asy
|
|
|
1321
1306
|
try {
|
|
1322
1307
|
await new Promise((resolve, reject) => blocker.listen(0, "127.0.0.1", () => resolve(undefined)).once("error", reject))
|
|
1323
1308
|
const address = blocker.address()
|
|
1324
|
-
|
|
1309
|
+
if (!address || typeof address === "string") throw new Error("Expected a TCP listener address")
|
|
1325
1310
|
config.proxy.port = address.port
|
|
1326
1311
|
const startupAttempt = new RollbridgeDaemon({config, logger: () => {}})
|
|
1327
1312
|
failedOwner = startupAttempt
|
|
1328
|
-
await
|
|
1313
|
+
await expect(() => startupAttempt.start()).toThrow(/EADDRINUSE/)
|
|
1329
1314
|
|
|
1330
1315
|
const state = JSON.parse(await fs.readFile(fixture.statePath, "utf8"))
|
|
1331
|
-
|
|
1316
|
+
expect(typeof state.recovery?.guardian?.token).toBe("string")
|
|
1332
1317
|
failedOwner.abandonOwnerRecoveryAttempt()
|
|
1333
1318
|
await new Promise((resolve) => blocker.close(() => resolve(undefined)))
|
|
1334
1319
|
|
|
1335
1320
|
replacement = new RollbridgeDaemon({config, logger: () => {}})
|
|
1336
1321
|
await replacement.start()
|
|
1337
|
-
|
|
1322
|
+
expect(replacement.status().activeReleaseId).toBe(null)
|
|
1338
1323
|
await replacement.shutdown()
|
|
1339
1324
|
} finally {
|
|
1340
1325
|
if (blocker.listening) await new Promise((resolve) => blocker.close(() => resolve(undefined)))
|
|
@@ -1364,8 +1349,8 @@ test("replacement reconstructs retained draining generations without an active r
|
|
|
1364
1349
|
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
1365
1350
|
await stopActive
|
|
1366
1351
|
const drainOnly = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1367
|
-
|
|
1368
|
-
|
|
1352
|
+
expect(drainOnly.activeReleaseId).toBe(null)
|
|
1353
|
+
expect(drainOnly.releaseReferences).toEqual([{releaseId: "v1", releasePath: v1Path}])
|
|
1369
1354
|
await waitForState(fixture.statePath, (state) => state.activeReleaseId === null && state.releaseReferences?.length === 1 && state.releaseReferences[0]?.releaseId === "v1")
|
|
1370
1355
|
|
|
1371
1356
|
owner.kill("SIGKILL")
|
|
@@ -1373,12 +1358,13 @@ test("replacement reconstructs retained draining generations without an active r
|
|
|
1373
1358
|
owner = spawnDaemon(fixture.configPath)
|
|
1374
1359
|
await waitForLog(owner, "control socket listening")
|
|
1375
1360
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1376
|
-
|
|
1377
|
-
|
|
1361
|
+
expect(recovered.activeReleaseId).toBe(null)
|
|
1362
|
+
expect(recovered.releaseReferences).toEqual([{releaseId: "v1", releasePath: v1Path}])
|
|
1378
1363
|
|
|
1379
1364
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
1380
1365
|
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
1381
|
-
|
|
1366
|
+
// replacement shutdown must acknowledge after the recovered drain settles
|
|
1367
|
+
await expect(() => shutdown).not.toThrow()
|
|
1382
1368
|
await once(owner, "exit")
|
|
1383
1369
|
} finally {
|
|
1384
1370
|
await killChild(owner)
|
|
@@ -1399,10 +1385,7 @@ test("deploy rejects a live ownerRecovery mode change", async () => {
|
|
|
1399
1385
|
for (const processConfig of changedConfig.processes) if (processConfig.lifecycle) delete processConfig.lifecycle.activateCommand
|
|
1400
1386
|
await writeConfig(fixture.configPath, changedConfig)
|
|
1401
1387
|
|
|
1402
|
-
await
|
|
1403
|
-
sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath}),
|
|
1404
|
-
/ownerRecovery.*cannot be applied live/
|
|
1405
|
-
)
|
|
1388
|
+
await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})).toThrow(/ownerRecovery.*cannot be applied live/)
|
|
1406
1389
|
|
|
1407
1390
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1408
1391
|
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
@@ -1429,10 +1412,7 @@ test("deploy rejects a live activation lifecycle mode change", async () => {
|
|
|
1429
1412
|
|
|
1430
1413
|
delete lifecycle.activateCommand
|
|
1431
1414
|
await writeConfig(fixture.configPath, changedConfig)
|
|
1432
|
-
await
|
|
1433
|
-
sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath}),
|
|
1434
|
-
/lifecycle\.activateCommand.*cannot be applied live/
|
|
1435
|
-
)
|
|
1415
|
+
await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})).toThrow(/lifecycle\.activateCommand.*cannot be applied live/)
|
|
1436
1416
|
|
|
1437
1417
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1438
1418
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
@@ -1451,7 +1431,7 @@ test("public state does not advance when private guardian publication fails", as
|
|
|
1451
1431
|
const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
1452
1432
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
1453
1433
|
|
|
1454
|
-
|
|
1434
|
+
if (!worker) throw new Error("Missing fixture worker")
|
|
1455
1435
|
worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
1456
1436
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1457
1437
|
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -1469,9 +1449,9 @@ test("public state does not advance when private guardian publication fails", as
|
|
|
1469
1449
|
owner.publishOwnerState = async () => { throw new Error("injected guardian publication failure") }
|
|
1470
1450
|
const write = owner.persistState({throwOnError: true})
|
|
1471
1451
|
|
|
1472
|
-
|
|
1473
|
-
await
|
|
1474
|
-
|
|
1452
|
+
expect(write).toBeTruthy()
|
|
1453
|
+
await expect(() => write).toThrow(/injected guardian publication failure/)
|
|
1454
|
+
expect(await fs.readFile(fixture.statePath, "utf8")).toBe(before)
|
|
1475
1455
|
owner.publishOwnerState = publishOwnerState
|
|
1476
1456
|
await owner.persistState({throwOnError: true})
|
|
1477
1457
|
await owner.shutdown()
|
|
@@ -1498,7 +1478,7 @@ test("replacement removes only guardian-owned candidate inventory left before de
|
|
|
1498
1478
|
const worker = candidateConfig.processes.find((processConfig) => processConfig.id === "worker")
|
|
1499
1479
|
const web = candidateConfig.processes.find((processConfig) => processConfig.id === "web")
|
|
1500
1480
|
|
|
1501
|
-
|
|
1481
|
+
if (!worker || !web) throw new Error("Missing fixture worker or web process")
|
|
1502
1482
|
worker.command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("const fs = require('node:fs'); const target = process.env.ROLLBRIDGE_RELEASE_PATH + '/candidate.pid'; fs.writeFileSync(target + '.tmp', String(process.pid)); fs.renameSync(target + '.tmp', target); setInterval(() => {}, 1000)")}`
|
|
1503
1483
|
worker.lifecycle = {drainTimeoutMs: 0}
|
|
1504
1484
|
web.command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`
|
|
@@ -1509,26 +1489,31 @@ test("replacement removes only guardian-owned candidate inventory left before de
|
|
|
1509
1489
|
|
|
1510
1490
|
await waitForFile(candidatePidPath)
|
|
1511
1491
|
candidatePid = Number(await fs.readFile(candidatePidPath, "utf8"))
|
|
1512
|
-
|
|
1492
|
+
expect(Number.isInteger(candidatePid) && candidatePid > 0).toBeTruthy()
|
|
1513
1493
|
owner.kill("SIGKILL")
|
|
1514
1494
|
await once(owner, "exit")
|
|
1515
1495
|
const stateAfterDeath = JSON.parse(await fs.readFile(fixture.statePath, "utf8"))
|
|
1516
1496
|
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1497
|
+
// owner death must preserve the last committed active release
|
|
1498
|
+
expect(stateAfterDeath.activeReleaseId).toBe(committedState.activeReleaseId)
|
|
1499
|
+
// owner death must preserve committed release references
|
|
1500
|
+
expect(stateAfterDeath.releaseReferences).toEqual(committedState.releaseReferences)
|
|
1501
|
+
// owner death must not commit candidate release metadata
|
|
1502
|
+
expect(stateAfterDeath.releases).toEqual(committedState.releases)
|
|
1520
1503
|
|
|
1521
1504
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1522
1505
|
owner = spawnDaemon(fixture.configPath)
|
|
1523
1506
|
await waitForLog(owner, "control socket listening")
|
|
1524
1507
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1525
1508
|
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1509
|
+
expect(recovered.activeReleaseId).toBe("v1")
|
|
1510
|
+
expect(recovered.releaseReferences).toEqual([{releaseId: "v1", releasePath: v1Path}])
|
|
1511
|
+
// uncommitted candidate must be stopped before replacement becomes healthy
|
|
1512
|
+
expect(isProcessRunning(candidatePid)).toBe(false)
|
|
1529
1513
|
|
|
1530
1514
|
await sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})
|
|
1531
|
-
|
|
1515
|
+
// removed candidate keys must be reusable by a later valid deploy
|
|
1516
|
+
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v2")
|
|
1532
1517
|
|
|
1533
1518
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
1534
1519
|
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
|
|
@@ -1561,18 +1546,18 @@ test("ensure-daemon replaces a same-authority owner whose control socket disappe
|
|
|
1561
1546
|
try {
|
|
1562
1547
|
const first = await runCli(ensureArgs)
|
|
1563
1548
|
|
|
1564
|
-
|
|
1549
|
+
expect({value: first.code, context: first.output}).toMatchObject({value: 0})
|
|
1565
1550
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})
|
|
1566
1551
|
const before = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1567
1552
|
|
|
1568
1553
|
await fs.rm(fixture.socketPath, {force: true})
|
|
1569
1554
|
const replacement = await runCli(ensureArgs)
|
|
1570
1555
|
|
|
1571
|
-
|
|
1556
|
+
expect({value: replacement.code, context: `${replacement.output}\n${await fs.readFile(daemonLogPath, "utf8")}`}).toMatchObject({value: 0})
|
|
1572
1557
|
const after = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1573
1558
|
|
|
1574
|
-
|
|
1575
|
-
|
|
1559
|
+
expect(after.activeReleaseId).toBe("v1")
|
|
1560
|
+
expect(after.daemonPid).not.toBe(before.daemonPid)
|
|
1576
1561
|
await waitForProcessExit(before.daemonPid)
|
|
1577
1562
|
|
|
1578
1563
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
@@ -1620,7 +1605,7 @@ test("ensure-daemon atomically replaces an incompatible owner without losing ret
|
|
|
1620
1605
|
nextConfig.control = {path: newControlPath}
|
|
1621
1606
|
const companionTemplate = nextConfig.processes.find((processConfig) => processConfig.policy === "companion")
|
|
1622
1607
|
|
|
1623
|
-
|
|
1608
|
+
if (!companionTemplate) throw new Error("Missing fixture companion template")
|
|
1624
1609
|
nextConfig.processes.splice(2, 0, {
|
|
1625
1610
|
...structuredClone(companionTemplate),
|
|
1626
1611
|
id: "new-topology-process",
|
|
@@ -1636,36 +1621,38 @@ test("ensure-daemon atomically replaces an incompatible owner without losing ret
|
|
|
1636
1621
|
"--daemon-runtime-path", runtimePath,
|
|
1637
1622
|
"--daemon-start-timeout-ms", "5000"
|
|
1638
1623
|
])
|
|
1639
|
-
|
|
1624
|
+
expect({value: replacement.code, context: `${replacement.output}\n${await fs.readFile(daemonLogPath, "utf8")}`}).toMatchObject({value: 0})
|
|
1640
1625
|
cleanupControlPath = newControlPath
|
|
1641
1626
|
|
|
1642
1627
|
const after = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: newControlPath}))
|
|
1643
1628
|
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1629
|
+
expect(after.activeReleaseId).toBe("v2")
|
|
1630
|
+
expect(after.releaseReferences).toEqual(before.releaseReferences)
|
|
1631
|
+
expect(after.releases.map((release) => release.processes.map((processStatus) => processStatus.pid))).toEqual(before.releases.map((release) => release.processes.map((processStatus) => processStatus.pid)))
|
|
1632
|
+
expect(after.services[0]?.process.pid).toBe(before.services[0]?.process.pid)
|
|
1633
|
+
expect(after.singletons[0]?.process.pid).toBe(before.singletons[0]?.process.pid)
|
|
1634
|
+
// listener-owned WebSocket must remain supervised across replacement
|
|
1635
|
+
expect(retainedConnectionClosed).toBe(false)
|
|
1636
|
+
expect(after.daemonPid).toBeTruthy()
|
|
1651
1637
|
await fs.writeFile(daemonLogPath, "")
|
|
1652
1638
|
process.kill(after.daemonPid, "SIGKILL")
|
|
1653
1639
|
const restartedState = await waitForState(fixture.statePath, (state) => state.daemonPid !== after.daemonPid, AbortSignal.timeout(5000))
|
|
1654
1640
|
const restarted = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: newControlPath}))
|
|
1655
1641
|
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1642
|
+
expect(restarted.daemonPid).toBe(restartedState.daemonPid)
|
|
1643
|
+
expect(Number((await fs.readFile(daemonPidPath, "utf8")).trim())).toBe(restarted.daemonPid)
|
|
1644
|
+
expect(restarted.releases.map((release) => release.processes.map((processStatus) => processStatus.pid))).toEqual(before.releases.map((release) => release.processes.map((processStatus) => processStatus.pid)))
|
|
1645
|
+
expect(await fs.readFile(daemonLogPath, "utf8")).toMatch(/owner state recovered/)
|
|
1660
1646
|
|
|
1661
1647
|
const v3Path = await prepareRelease(fixture.root, "v3")
|
|
1662
1648
|
await sendControlCommand({command: {command: "deploy", releaseId: "v3", releasePath: v3Path, revision: "v3"}, path: newControlPath})
|
|
1663
1649
|
const deployed = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: newControlPath}))
|
|
1664
1650
|
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1651
|
+
expect(deployed.activeReleaseId).toBe("v3")
|
|
1652
|
+
expect(deployed.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2", "v3"])
|
|
1653
|
+
expect(releaseProcessPid(deployed, "v1", "web")).toBe(releaseProcessPid(before, "v1", "web"))
|
|
1654
|
+
// a later deploy must not stop a process with a transferred live connection
|
|
1655
|
+
expect(retainedConnectionClosed).toBe(false)
|
|
1669
1656
|
|
|
1670
1657
|
retainedConnection.destroy()
|
|
1671
1658
|
await retainedConnectionClose
|
|
@@ -1773,6 +1760,33 @@ async function lifecycleEvents(lifecycleLogPath) {
|
|
|
1773
1760
|
return (await fs.readFile(lifecycleLogPath, "utf8")).trim().split("\n").filter(Boolean)
|
|
1774
1761
|
}
|
|
1775
1762
|
|
|
1763
|
+
/**
|
|
1764
|
+
* @param {string} lifecycleLogPath - Fixture lifecycle log.
|
|
1765
|
+
* @param {string[]} expected - Exact lifecycle events to await.
|
|
1766
|
+
* @returns {Promise<string[]>} The matching ordered events.
|
|
1767
|
+
*/
|
|
1768
|
+
async function waitForLifecycleEvents(lifecycleLogPath, expected) {
|
|
1769
|
+
const watcher = fs.watch(lifecycleLogPath, {signal: AbortSignal.timeout(3000)})
|
|
1770
|
+
|
|
1771
|
+
try {
|
|
1772
|
+
const events = await lifecycleEvents(lifecycleLogPath)
|
|
1773
|
+
|
|
1774
|
+
if (JSON.stringify(events) === JSON.stringify(expected)) return events
|
|
1775
|
+
for await (const _event of watcher) {
|
|
1776
|
+
const changedEvents = await lifecycleEvents(lifecycleLogPath)
|
|
1777
|
+
|
|
1778
|
+
if (JSON.stringify(changedEvents) === JSON.stringify(expected)) return changedEvents
|
|
1779
|
+
}
|
|
1780
|
+
} catch (error) {
|
|
1781
|
+
if (error instanceof Error && error.name === "AbortError") throw new Error(`Timed out waiting for lifecycle events: ${JSON.stringify(expected)}`, {cause: error})
|
|
1782
|
+
throw error
|
|
1783
|
+
} finally {
|
|
1784
|
+
await watcher.return?.()
|
|
1785
|
+
}
|
|
1786
|
+
|
|
1787
|
+
throw new Error(`Timed out waiting for lifecycle events: ${JSON.stringify(expected)}`)
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1776
1790
|
/**
|
|
1777
1791
|
* @param {string} root - Fixture root.
|
|
1778
1792
|
* @param {string} releaseId - Release id.
|
|
@@ -1785,7 +1799,7 @@ async function prepareRelease(root, releaseId, {holdJobsBind = false} = {}) {
|
|
|
1785
1799
|
await fs.mkdir(releasePath)
|
|
1786
1800
|
if (!holdJobsBind) await fs.writeFile(path.join(releasePath, "jobs.bind"), "ready\n")
|
|
1787
1801
|
const gate = spawn("mkfifo", [path.join(releasePath, "worker.fifo")])
|
|
1788
|
-
|
|
1802
|
+
expect((await once(gate, "exit"))[0]).toBe(0)
|
|
1789
1803
|
return releasePath
|
|
1790
1804
|
}
|
|
1791
1805
|
|
|
@@ -1978,7 +1992,7 @@ async function openWebSocket(port) {
|
|
|
1978
1992
|
].join("\r\n"))
|
|
1979
1993
|
const [response] = await once(socket, "data")
|
|
1980
1994
|
|
|
1981
|
-
|
|
1995
|
+
expect(String(response)).toMatch(/^HTTP\/1\.1 101 /)
|
|
1982
1996
|
return socket
|
|
1983
1997
|
}
|
|
1984
1998
|
|
|
@@ -2056,7 +2070,7 @@ async function runCli(args) {
|
|
|
2056
2070
|
* @param {{allowChildExit?: boolean}} [options] - Whether inherited descriptors may outlive the original child.
|
|
2057
2071
|
*/
|
|
2058
2072
|
async function waitForLog(child, message, {allowChildExit = false} = {}) {
|
|
2059
|
-
|
|
2073
|
+
if (!child.stdout) throw new Error("Missing CLI stdout stream")
|
|
2060
2074
|
child.stdout.setEncoding("utf8")
|
|
2061
2075
|
|
|
2062
2076
|
await new Promise((resolve, reject) => {
|
|
@@ -2093,3 +2107,4 @@ async function waitForLog(child, message, {allowChildExit = false} = {}) {
|
|
|
2093
2107
|
child.stderr?.setEncoding("utf8").on("data", onErrorData)
|
|
2094
2108
|
})
|
|
2095
2109
|
}
|
|
2110
|
+
})
|