rollbridge 0.1.54 → 0.1.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -0
- package/changelog.d/20260909120000-velocious-testing.md +1 -0
- package/eslint.config.js +8 -0
- package/package.json +3 -2
- package/test/completion.test.js +18 -16
- package/test/config-examples.test.js +16 -17
- package/test/config-path.test.js +10 -11
- package/test/config-validation.test.js +163 -167
- package/test/control-protocol.test.js +22 -21
- package/test/daemon-bootstrap.test.js +104 -104
- package/test/daemon-runtime.test.js +17 -26
- package/test/doctor.test.js +51 -49
- package/test/event-log.test.js +13 -11
- package/test/guardian-client.test.js +140 -148
- package/test/health.test.js +6 -4
- package/test/logs.test.js +17 -18
- package/test/managed-process.test.js +96 -91
- package/test/owner-recovery.test.js +226 -239
- package/test/owner-replacement.test.js +227 -222
- package/test/package-metadata.test.js +48 -39
- package/test/port-allocator.test.js +13 -16
- package/test/predeploy-cleanup.test.js +12 -10
- package/test/process-memory.test.js +17 -15
- package/test/proxy.test.js +10 -8
- package/test/recover.test.js +30 -23
- package/test/release-group.test.js +16 -17
- package/test/release-retention.test.js +10 -8
- package/test/release-runtime-retention.test.js +31 -39
- package/test/rollbridge.test.js +377 -396
- package/test/shutdown-completion.test.js +51 -51
- package/test/state-store.test.js +10 -8
- package/test/system-ids.test.js +15 -13
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
-
import assert from "node:assert/strict"
|
|
4
3
|
import {spawn} from "node:child_process"
|
|
5
4
|
import {once} from "node:events"
|
|
6
5
|
import fs from "node:fs/promises"
|
|
7
6
|
import net from "node:net"
|
|
8
7
|
import os from "node:os"
|
|
9
8
|
import path from "node:path"
|
|
10
|
-
import test from "
|
|
9
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
11
10
|
import {fileURLToPath} from "node:url"
|
|
12
11
|
import {normalizeConfig} from "../src/config.js"
|
|
13
12
|
import {sendControlCommand} from "../src/control-client.js"
|
|
@@ -15,6 +14,8 @@ import RollbridgeDaemon from "../src/daemon.js"
|
|
|
15
14
|
import GuardianClient from "../src/guardian-client.js"
|
|
16
15
|
import {isProcessRunning, waitForProcessExit} from "./support/process.js"
|
|
17
16
|
|
|
17
|
+
describe("owner-recovery", () => {
|
|
18
|
+
|
|
18
19
|
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
|
19
20
|
const binPath = path.join(currentDir, "..", "bin", "rollbridge")
|
|
20
21
|
const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
|
|
@@ -72,21 +73,23 @@ test("external owner retirement releases guardian authority without losing its g
|
|
|
72
73
|
const v1 = recovered.releases.find(({releaseId}) => releaseId === "v1")
|
|
73
74
|
const v2 = recovered.releases.find(({releaseId}) => releaseId === "v2")
|
|
74
75
|
|
|
75
|
-
|
|
76
|
-
|
|
76
|
+
// the prestarted candidate must remain active
|
|
77
|
+
expect(recovered.activeReleaseId).toBe("v2")
|
|
78
|
+
expect(recovered.releaseReferences.sort((a, b) => a.releaseId.localeCompare(b.releaseId))).toEqual([
|
|
77
79
|
{releaseId: "v1", releasePath: v1Path},
|
|
78
80
|
{releaseId: "v2", releasePath: v2Path}
|
|
79
81
|
])
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
82
|
+
expect(v1?.state).toBe("draining")
|
|
83
|
+
expect(v1?.processes.find(({id}) => id === "worker")?.pid).toBe(v1WorkerPid)
|
|
84
|
+
expect(v1?.processes.find(({id}) => id === "worker")?.state).toBe("quiesced")
|
|
85
|
+
expect(v2?.state).toBe("active")
|
|
86
|
+
expect(v2?.processes.find(({id}) => id === "worker")?.pid).toBe(v2WorkerPid)
|
|
87
|
+
expect(v2?.processes.find(({id}) => id === "worker")?.state).toBe("running")
|
|
86
88
|
await waitForFile(path.join(v1Path, "drain-started"), 1000)
|
|
87
89
|
await fs.writeFile(path.join(v1Path, "drained"), "done\n")
|
|
88
90
|
await waitForProcessExit(v1WorkerPid, 1000)
|
|
89
|
-
|
|
91
|
+
// the active candidate worker must remain usable while the old generation drains
|
|
92
|
+
expect(isProcessRunning(v2WorkerPid)).toBe(true)
|
|
90
93
|
} finally {
|
|
91
94
|
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "drained"), "done\n").catch(() => {})))
|
|
92
95
|
await replacement?.shutdown().catch(() => {})
|
|
@@ -117,8 +120,8 @@ test("exact bootstrap restores the committed generation after external owner ret
|
|
|
117
120
|
...committed.singletons.map(({process}) => process.pid)
|
|
118
121
|
].filter((pid) => typeof pid === "number")
|
|
119
122
|
|
|
120
|
-
|
|
121
|
-
|
|
123
|
+
expect(committed.activeReleaseId).toBe("v2")
|
|
124
|
+
expect(committed.generationTransition?.phase).toBe("committed")
|
|
122
125
|
await retired.retireOwner({attestation: `sha256:${"a".repeat(64)}`})
|
|
123
126
|
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
124
127
|
await Promise.all(retiredPids.map((pid) => waitForProcessExit(pid, 3000)))
|
|
@@ -134,12 +137,10 @@ test("exact bootstrap restores the committed generation after external owner ret
|
|
|
134
137
|
const recoveredCandidate = recovered.releases.get("v2")
|
|
135
138
|
const recoveredService = recovered.services.get("beacon")
|
|
136
139
|
|
|
137
|
-
|
|
140
|
+
if (!recoveredCandidate || !recoveredService) throw new Error("Missing recovered candidate or service")
|
|
141
|
+
expect(recovered.singletons.get("singleton")).toBeTruthy()
|
|
138
142
|
recovered.serviceReleaseIds.set("beacon", "v1")
|
|
139
|
-
|
|
140
|
-
() => recovered?.assertCommittedBootstrapRecoveryReady(),
|
|
141
|
-
/service beacon belongs to retained release v1/
|
|
142
|
-
)
|
|
143
|
+
await expect(() => recovered?.assertCommittedBootstrapRecoveryReady()).toThrow(/service beacon belongs to retained release v1/)
|
|
143
144
|
recovered.serviceReleaseIds.set("beacon", "v2")
|
|
144
145
|
const activateGeneration = recoveredCandidate.activateGeneration.bind(recoveredCandidate)
|
|
145
146
|
const startService = recoveredService.start.bind(recoveredService)
|
|
@@ -150,7 +151,7 @@ test("exact bootstrap restores the committed generation after external owner ret
|
|
|
150
151
|
await activateGeneration()
|
|
151
152
|
}
|
|
152
153
|
recoveredService.start = async (...args) => {
|
|
153
|
-
|
|
154
|
+
expect(recovered?.generationTransition?.phase).toBe("restoring_committed")
|
|
154
155
|
recoveryOrder.push("service")
|
|
155
156
|
await startService(...args)
|
|
156
157
|
}
|
|
@@ -161,17 +162,19 @@ test("exact bootstrap restores the committed generation after external owner ret
|
|
|
161
162
|
await recovered.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
162
163
|
const active = recovered.status()
|
|
163
164
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
165
|
+
expect(active.activeReleaseId).toBe("v2")
|
|
166
|
+
expect(active.generationTransition?.phase).toBe("committed")
|
|
167
|
+
expect(active.releases.find(({releaseId}) => releaseId === "v1")?.state).toBe("draining")
|
|
168
|
+
expect(releaseProcessPid(active, "v1", "worker")).toBe(v1WorkerPid)
|
|
169
|
+
// the retained previous generation must keep draining
|
|
170
|
+
expect(isProcessRunning(v1WorkerPid)).toBe(true)
|
|
171
|
+
expect(active.releases.find(({releaseId}) => releaseId === "v2")?.state).toBe("active")
|
|
172
|
+
expect(active.releases.find(({releaseId}) => releaseId === "v2")?.processes.every(({pid, state}) => typeof pid === "number" && state === "running")).toBeTruthy()
|
|
173
|
+
expect(active.services.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
|
|
174
|
+
expect(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
|
|
175
|
+
// candidate activation must precede post-commit singleton completion
|
|
176
|
+
expect(recoveryOrder).toEqual(["service", "activate", "singleton"])
|
|
177
|
+
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
|
|
175
178
|
} finally {
|
|
176
179
|
if (recovered) {
|
|
177
180
|
const activeRecovery = recovered.status().activeReleaseId === "v2"
|
|
@@ -195,7 +198,7 @@ test("exact bootstrap restores a committed generation after its previous release
|
|
|
195
198
|
const worker = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
196
199
|
.find((processConfig) => processConfig.id === "worker")
|
|
197
200
|
|
|
198
|
-
|
|
201
|
+
if (!worker) throw new Error("Missing fixture worker")
|
|
199
202
|
const workerLifecycle = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (worker.lifecycle)
|
|
200
203
|
|
|
201
204
|
workerLifecycle.drainTimeoutMs = 500
|
|
@@ -219,7 +222,7 @@ test("exact bootstrap restores a committed generation after its previous release
|
|
|
219
222
|
.map(({pid}) => pid)
|
|
220
223
|
.filter((pid) => typeof pid === "number") || []
|
|
221
224
|
|
|
222
|
-
|
|
225
|
+
expect(committed.releases.some(({releaseId}) => releaseId === "v1")).toBe(false)
|
|
223
226
|
await retired.retireOwner({attestation: `sha256:${"d".repeat(64)}`})
|
|
224
227
|
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
225
228
|
await Promise.all(retiredPids.map((pid) => waitForProcessExit(pid, 3000)))
|
|
@@ -233,8 +236,8 @@ test("exact bootstrap restores a committed generation after its previous release
|
|
|
233
236
|
await recovered.start({exposeControl: false})
|
|
234
237
|
await recovered.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
235
238
|
|
|
236
|
-
|
|
237
|
-
|
|
239
|
+
expect(recovered.status().activeReleaseId).toBe("v2")
|
|
240
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
238
241
|
} finally {
|
|
239
242
|
if (recovered) {
|
|
240
243
|
const activeRecovery = recovered.status().activeReleaseId === "v2"
|
|
@@ -283,7 +286,7 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
|
|
|
283
286
|
await interrupted.updateGenerationTransition("restoring_committed")
|
|
284
287
|
const candidate = interrupted.releases.get("v2")
|
|
285
288
|
|
|
286
|
-
|
|
289
|
+
if (!candidate) throw new Error("Missing recovered candidate")
|
|
287
290
|
for (const processInstance of interrupted.services.values()) await processInstance.start("deploy")
|
|
288
291
|
await candidate.restartCommittedGeneration()
|
|
289
292
|
await interrupted.checkpointGenerationTransition()
|
|
@@ -291,8 +294,8 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
|
|
|
291
294
|
const restartedCandidatePids = restarted.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid)
|
|
292
295
|
const restartedServicePids = restarted.services.map(({process}) => process.pid)
|
|
293
296
|
|
|
294
|
-
|
|
295
|
-
|
|
297
|
+
expect(restarted.generationTransition?.phase).toBe("restoring_committed")
|
|
298
|
+
expect(restartedCandidatePids?.every((pid) => typeof pid === "number")).toBeTruthy()
|
|
296
299
|
await interrupted.retireCommittedOwner(undefined)
|
|
297
300
|
interrupted.guardian?.disconnect()
|
|
298
301
|
|
|
@@ -300,14 +303,14 @@ test("journaled committed bootstrap recovery resumes after a restart begins", as
|
|
|
300
303
|
await recovered.start({exposeControl: false})
|
|
301
304
|
const active = recovered.status()
|
|
302
305
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
306
|
+
expect(active.activeReleaseId).toBe("v2")
|
|
307
|
+
expect(active.generationTransition?.phase).toBe("committed")
|
|
308
|
+
expect(active.releases.find(({releaseId}) => releaseId === "v2")?.processes.map(({pid}) => pid)).toEqual(restartedCandidatePids)
|
|
309
|
+
expect(active.services.map(({process}) => process.pid)).toEqual(restartedServicePids)
|
|
310
|
+
expect(active.singletons.every(({process}) => typeof process.pid === "number" && process.state === "running")).toBeTruthy()
|
|
311
|
+
expect(releaseProcessPid(active, "v1", "worker")).toBe(v1WorkerPid)
|
|
312
|
+
expect(isProcessRunning(v1WorkerPid)).toBe(true)
|
|
313
|
+
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2", "retire:v2", "activate:v2"])
|
|
311
314
|
} finally {
|
|
312
315
|
if (recovered) {
|
|
313
316
|
const shutdown = recovered.shutdown()
|
|
@@ -358,39 +361,27 @@ test("committed bootstrap tuple mismatches fail closed without singletons", asyn
|
|
|
358
361
|
await recovered.start({exposeControl: false})
|
|
359
362
|
const owner = recovered
|
|
360
363
|
|
|
361
|
-
await
|
|
362
|
-
|
|
363
|
-
/only the exact same release, path, revision, and config authority/u
|
|
364
|
-
)
|
|
365
|
-
await assert.rejects(
|
|
366
|
-
() => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "wrong"}),
|
|
367
|
-
/only the exact same release, path, revision, and config authority/u
|
|
368
|
-
)
|
|
364
|
+
await expect(() => owner.deploy({releaseId: "v2", releasePath: wrongPath, revision: "v2"})).toThrow(/only the exact same release, path, revision, and config authority/u)
|
|
365
|
+
await expect(() => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "wrong"})).toThrow(/only the exact same release, path, revision, and config authority/u)
|
|
369
366
|
const changedConfig = structuredClone(fixture.config)
|
|
370
367
|
const jobs = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (changedConfig.processes)
|
|
371
368
|
.find((processConfig) => processConfig.id === "jobs")
|
|
372
369
|
|
|
373
|
-
|
|
370
|
+
if (!jobs) throw new Error("Missing fixture jobs service")
|
|
374
371
|
jobs.env = {.../** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobs.env), MISMATCHED_AUTHORITY: "true"}
|
|
375
372
|
await writeConfig(fixture.configPath, changedConfig)
|
|
376
|
-
await
|
|
377
|
-
() => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"}),
|
|
378
|
-
/only the exact same release, path, revision, and config authority/u
|
|
379
|
-
)
|
|
373
|
+
await expect(() => owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})).toThrow(/only the exact same release, path, revision, and config authority/u)
|
|
380
374
|
await writeConfig(fixture.configPath, fixture.config)
|
|
381
|
-
await
|
|
382
|
-
() => owner.deploy({releaseId: "wrong", releasePath: wrongPath, revision: "wrong"}),
|
|
383
|
-
/only the exact same release, path, revision, and config authority/u
|
|
384
|
-
)
|
|
375
|
+
await expect(() => owner.deploy({releaseId: "wrong", releasePath: wrongPath, revision: "wrong"})).toThrow(/only the exact same release, path, revision, and config authority/u)
|
|
385
376
|
const preserved = owner.status()
|
|
386
377
|
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
378
|
+
expect(preserved.activeReleaseId).toBe(null)
|
|
379
|
+
expect(preserved.generationTransition?.candidateReleaseId).toBe("v2")
|
|
380
|
+
expect(preserved.generationTransition?.phase).toBe("committed")
|
|
381
|
+
expect(preserved.releases.find(({releaseId}) => releaseId === "v2")?.state).toBe("draining")
|
|
382
|
+
expect(releaseProcessPid(preserved, "v1", "worker")).toBe(v1WorkerPid)
|
|
383
|
+
expect(isProcessRunning(v1WorkerPid)).toBe(true)
|
|
384
|
+
expect(preserved.releases.some(({releaseId}) => releaseId === "wrong")).toBe(false)
|
|
394
385
|
|
|
395
386
|
await owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
396
387
|
await Promise.all([
|
|
@@ -399,8 +390,8 @@ test("committed bootstrap tuple mismatches fail closed without singletons", asyn
|
|
|
399
390
|
])
|
|
400
391
|
const afterIntentionalStop = await owner.deploy({releaseId: "wrong", releasePath: wrongPath, revision: "wrong"})
|
|
401
392
|
|
|
402
|
-
|
|
403
|
-
|
|
393
|
+
expect(afterIntentionalStop.activeReleaseId).toBe("wrong")
|
|
394
|
+
expect(owner.status().activeReleaseId).toBe("wrong")
|
|
404
395
|
} finally {
|
|
405
396
|
if (recovered) {
|
|
406
397
|
const activeReleaseId = recovered.status().activeReleaseId
|
|
@@ -439,8 +430,8 @@ test("replacement owner reconstructs one active and two draining generations aft
|
|
|
439
430
|
|
|
440
431
|
const before = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
441
432
|
|
|
442
|
-
|
|
443
|
-
|
|
433
|
+
expect(before.activeReleaseId).toBe("v3")
|
|
434
|
+
expect(before.releaseReferences.map((/** @type {{releaseId: string}} */ reference) => reference.releaseId)).toEqual(["v1", "v2", "v3"])
|
|
444
435
|
const generationEndpoints = before.releases.map((release) => ({
|
|
445
436
|
jobsPort: release.ports.jobs,
|
|
446
437
|
jobsState: release.processes.find((processStatus) => processStatus.id === "jobs")?.state,
|
|
@@ -448,7 +439,7 @@ test("replacement owner reconstructs one active and two draining generations aft
|
|
|
448
439
|
state: release.state
|
|
449
440
|
}))
|
|
450
441
|
|
|
451
|
-
|
|
442
|
+
expect({value: new Set(generationEndpoints.map(({jobsPort}) => jobsPort)).size, context: JSON.stringify(generationEndpoints)}).toMatchObject({value: 3})
|
|
452
443
|
owner.kill("SIGKILL")
|
|
453
444
|
await once(owner, "exit")
|
|
454
445
|
|
|
@@ -457,23 +448,24 @@ test("replacement owner reconstructs one active and two draining generations aft
|
|
|
457
448
|
|
|
458
449
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
459
450
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
451
|
+
expect(recovered.activeReleaseId).toBe("v3")
|
|
452
|
+
expect(recovered.releases.map((/** @type {{state: string}} */ release) => release.state)).toEqual(["draining", "draining", "active"])
|
|
453
|
+
expect(recovered.releaseReferences.map((/** @type {{releaseId: string}} */ reference) => reference.releaseId)).toEqual(["v1", "v2", "v3"])
|
|
454
|
+
expect(recovered.releases.map((release) => release.ports.jobs)).toEqual(before.releases.map((release) => release.ports.jobs))
|
|
455
|
+
expect(recovered.services[0]?.process.pid).toBe(before.services[0]?.process.pid)
|
|
456
|
+
expect(recovered.singletons[0]?.process.pid).toBe(before.singletons[0]?.process.pid)
|
|
466
457
|
|
|
467
458
|
const v4Path = path.join(fixture.root, "v4")
|
|
468
459
|
await fs.mkdir(v4Path)
|
|
469
460
|
const v4Gate = spawn("mkfifo", [path.join(v4Path, "worker.fifo")])
|
|
470
|
-
|
|
461
|
+
expect((await once(v4Gate, "exit"))[0]).toBe(0)
|
|
471
462
|
await sendControlCommand({command: {command: "deploy", releaseId: "v4", releasePath: v4Path, revision: "v4"}, path: fixture.socketPath})
|
|
472
|
-
|
|
463
|
+
// new work must progress while old generations remain retained
|
|
464
|
+
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v4")
|
|
473
465
|
|
|
474
466
|
await Promise.all(["v1", "v2"].map((releaseId) => fs.writeFile(path.join(fixture.root, releaseId, "worker.fifo"), "drained\n")))
|
|
475
467
|
const afterDrain = await waitForState(fixture.statePath, (state) => state.releaseReferences?.map((/** @type {{releaseId: string}} */ reference) => reference.releaseId).join(",") === "v3,v4")
|
|
476
|
-
|
|
468
|
+
expect(afterDrain.releaseReferences.map((/** @type {{releaseId: string}} */ reference) => reference.releaseId)).toEqual(["v3", "v4"])
|
|
477
469
|
|
|
478
470
|
await fs.writeFile(path.join(fixture.root, "v3", "worker.fifo"), "drained\n")
|
|
479
471
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
@@ -516,9 +508,9 @@ test("guardian restarts an abruptly exited daemon without replacing managed proc
|
|
|
516
508
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
517
509
|
|
|
518
510
|
recoveredDaemonPid = recovered.daemonPid
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
511
|
+
expect(recovered.daemonPid).not.toBe(before.daemonPid)
|
|
512
|
+
expect(recovered.activeReleaseId).toBe("v1")
|
|
513
|
+
expect(releaseProcessPid(recovered, "v1", "worker")).toBe(workerPid)
|
|
522
514
|
|
|
523
515
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
524
516
|
|
|
@@ -541,7 +533,7 @@ test("guardian recovers a persistent service after its final active release stop
|
|
|
541
533
|
const jobs = processes.find((processConfig) => processConfig.id === "jobs")
|
|
542
534
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
543
535
|
|
|
544
|
-
|
|
536
|
+
if (!jobs || !worker) throw new Error("Missing fixture jobs service or worker")
|
|
545
537
|
jobs.port = {from: 17000, to: 17001}
|
|
546
538
|
worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
547
539
|
fixture.config.processes = processes.filter((processConfig) => processConfig.id !== "singleton")
|
|
@@ -558,11 +550,11 @@ test("guardian recovers a persistent service after its final active release stop
|
|
|
558
550
|
const before = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
559
551
|
const servicePid = before.services[0]?.process.pid
|
|
560
552
|
|
|
561
|
-
|
|
562
|
-
|
|
553
|
+
expect(before.activeReleaseId).toBe(null)
|
|
554
|
+
expect(typeof servicePid).toBe("number")
|
|
563
555
|
const persisted = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
|
|
564
556
|
|
|
565
|
-
|
|
557
|
+
expect(persisted.serviceReleaseIds?.beacon).toBe("v1")
|
|
566
558
|
const recoveredListenerLog = waitForLog(owner, "control socket listening", {allowChildExit: true})
|
|
567
559
|
|
|
568
560
|
owner.kill("SIGKILL")
|
|
@@ -571,16 +563,16 @@ test("guardian recovers a persistent service after its final active release stop
|
|
|
571
563
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
572
564
|
|
|
573
565
|
recoveredDaemonPid = recovered.daemonPid
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
566
|
+
expect(recovered.activeReleaseId).toBe(null)
|
|
567
|
+
expect(recovered.services[0]?.process.pid).toBe(servicePid)
|
|
568
|
+
expect(recovered.releaseReferences).toEqual([{releaseId: "v1", releasePath}])
|
|
577
569
|
const nextReleasePath = await prepareRelease(fixture.root, "v2")
|
|
578
570
|
|
|
579
571
|
await sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: nextReleasePath, revision: "v2"}, path: fixture.socketPath})
|
|
580
572
|
const finalReleasePath = await prepareRelease(fixture.root, "v3")
|
|
581
573
|
|
|
582
574
|
await sendControlCommand({command: {command: "deploy", releaseId: "v3", releasePath: finalReleasePath, revision: "v3"}, path: fixture.socketPath})
|
|
583
|
-
|
|
575
|
+
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v3")
|
|
584
576
|
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
585
577
|
} finally {
|
|
586
578
|
await killChild(owner)
|
|
@@ -607,17 +599,17 @@ test("owner state omits a new persistent service until its defining release is r
|
|
|
607
599
|
const guardianState = /** @type {Record<string, import("../src/json.js").JsonValue> | undefined} */ (await owner.guardian?.ownerState())
|
|
608
600
|
const guardianSnapshot = /** @type {RecoveryState | undefined} */ (guardianState?.snapshot)
|
|
609
601
|
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
602
|
+
expect(persisted.releases).toEqual([])
|
|
603
|
+
expect(persisted.services).toEqual([])
|
|
604
|
+
expect(persisted.serviceReleaseIds).toEqual({})
|
|
605
|
+
expect(guardianSnapshot?.releases).toEqual([])
|
|
606
|
+
expect(guardianSnapshot?.services).toEqual([])
|
|
607
|
+
expect(guardianState?.serviceReleaseIds).toEqual({})
|
|
616
608
|
|
|
617
609
|
await fs.writeFile(path.join(releasePath, "jobs.bind"), "ready\n")
|
|
618
610
|
await deploy
|
|
619
|
-
|
|
620
|
-
|
|
611
|
+
expect(owner.status().services.find(({id}) => id === "beacon")?.process.state).toBe("running")
|
|
612
|
+
expect(owner.serviceReleaseIds.get("beacon")).toBe("v1")
|
|
621
613
|
} finally {
|
|
622
614
|
await fs.writeFile(path.join(releasePath, "jobs.bind"), "ready\n").catch(() => {})
|
|
623
615
|
await deploy?.catch(() => {})
|
|
@@ -636,7 +628,7 @@ test("owner recovery accepts released format-2 state without journal revision or
|
|
|
636
628
|
const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
637
629
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
638
630
|
|
|
639
|
-
|
|
631
|
+
if (!worker) throw new Error("Missing fixture worker")
|
|
640
632
|
worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
641
633
|
await writeConfig(fixture.configPath, fixture.config)
|
|
642
634
|
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -666,9 +658,9 @@ test("owner recovery accepts released format-2 state without journal revision or
|
|
|
666
658
|
recovered = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
667
659
|
await recovered.start()
|
|
668
660
|
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
661
|
+
expect(recovered.status().activeReleaseId).toBe("v1")
|
|
662
|
+
expect(recovered.status().generationTransition?.journalRevision).toBe(undefined)
|
|
663
|
+
expect(recovered.serviceReleaseIds.get("beacon")).toBe("v1")
|
|
672
664
|
} finally {
|
|
673
665
|
if (recovered) await recovered.shutdown().catch(() => {})
|
|
674
666
|
else await owner.shutdown().catch(() => {})
|
|
@@ -688,7 +680,7 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
|
|
|
688
680
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
689
681
|
const lifecycle = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobs?.lifecycle)
|
|
690
682
|
|
|
691
|
-
|
|
683
|
+
if (!jobs || !worker) throw new Error("Missing fixture jobs service or worker")
|
|
692
684
|
jobs.gracefulStopMs = 5000
|
|
693
685
|
lifecycle.quietCommand = `printf 'waiting\n' >> ${JSON.stringify(retirementWaitingPath)}; while [ ! -f ${JSON.stringify(retirementGatePath)} ]; do sleep 0.02; done; printf 'retire:%s\n' "$ROLLBRIDGE_RELEASE_ID" >> ${JSON.stringify(fixture.lifecycleLogPath)}`
|
|
694
686
|
worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
@@ -716,28 +708,19 @@ test("guardian recovery becomes ready before replaying a gated generation hook",
|
|
|
716
708
|
const recovering = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
717
709
|
const guardianPid = JSON.parse(await fs.readFile(fixture.statePath, "utf8")).recovery?.guardian?.pid
|
|
718
710
|
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
await
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
)
|
|
727
|
-
await assert.rejects(
|
|
728
|
-
sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}),
|
|
729
|
-
/Cannot shut down while generation transition recovery is in progress/
|
|
730
|
-
)
|
|
731
|
-
await assert.rejects(
|
|
732
|
-
sendControlCommand({command: {attestation: `sha256:${"a".repeat(64)}`, command: "retire-owner"}, path: fixture.socketPath}),
|
|
733
|
-
/Cannot retire owner while generation transition recovery is in progress/
|
|
734
|
-
)
|
|
711
|
+
expect(recovering.daemonPid).toBe(recoveredDaemonPid)
|
|
712
|
+
expect(recovering.ownerRecovery?.ready).toBe(true)
|
|
713
|
+
expect(recovering.generationTransition?.phase).toBe("retiring_previous")
|
|
714
|
+
expect(typeof guardianPid).toBe("number")
|
|
715
|
+
await expect(() => sendControlCommand({command: {command: "stop", releaseId: "v1"}, path: fixture.socketPath})).toThrow(/Another owner mutation/)
|
|
716
|
+
await expect(() => sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})).toThrow(/Cannot shut down while generation transition recovery is in progress/)
|
|
717
|
+
await expect(() => sendControlCommand({command: {attestation: `sha256:${"a".repeat(64)}`, command: "retire-owner"}, path: fixture.socketPath})).toThrow(/Cannot retire owner while generation transition recovery is in progress/)
|
|
735
718
|
|
|
736
719
|
process.kill(recoveredDaemonPid, "SIGTERM")
|
|
737
720
|
await fs.writeFile(retirementGatePath, "release retirement\n")
|
|
738
721
|
await waitForProcessExit(recoveredDaemonPid, 5000)
|
|
739
722
|
await waitForProcessExit(guardianPid, 5000)
|
|
740
|
-
|
|
723
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v1", "activate:v2", "retire:v2"])
|
|
741
724
|
} finally {
|
|
742
725
|
await fs.writeFile(retirementGatePath, "release retirement\n").catch(() => undefined)
|
|
743
726
|
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath}).catch(() => undefined)
|
|
@@ -758,11 +741,8 @@ test("owner recovery preserves a completed activation compensation without repla
|
|
|
758
741
|
const v2Path = await prepareRelease(fixture.root, "v2")
|
|
759
742
|
|
|
760
743
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})
|
|
761
|
-
await
|
|
762
|
-
|
|
763
|
-
/activate command exited non-zero/
|
|
764
|
-
)
|
|
765
|
-
assert.deepEqual(await lifecycleEvents(fixture.lifecycleLogPath), ["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
|
|
744
|
+
await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})).toThrow(/activate command exited non-zero/)
|
|
745
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
|
|
766
746
|
|
|
767
747
|
owner.kill("SIGKILL")
|
|
768
748
|
await once(owner, "exit")
|
|
@@ -771,9 +751,10 @@ test("owner recovery preserves a completed activation compensation without repla
|
|
|
771
751
|
|
|
772
752
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
773
753
|
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
754
|
+
expect(recovered.activeReleaseId).toBe("v1")
|
|
755
|
+
expect(recovered.generationTransition).toBe(undefined)
|
|
756
|
+
// owner recovery must not replay completed compensation hooks
|
|
757
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
|
|
777
758
|
|
|
778
759
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
779
760
|
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
|
|
@@ -792,7 +773,7 @@ test("owner recovery replays one journaled ambiguous first-generation activation
|
|
|
792
773
|
try {
|
|
793
774
|
await waitForLog(owner, "control socket listening")
|
|
794
775
|
const v1Path = await prepareRelease(fixture.root, "v1")
|
|
795
|
-
await
|
|
776
|
+
await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})).toThrow()
|
|
796
777
|
owner.kill("SIGKILL")
|
|
797
778
|
await once(owner, "exit")
|
|
798
779
|
|
|
@@ -811,9 +792,9 @@ test("owner recovery replays one journaled ambiguous first-generation activation
|
|
|
811
792
|
|
|
812
793
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
813
794
|
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
795
|
+
expect(recovered.activeReleaseId).toBe("v1")
|
|
796
|
+
expect(recovered.generationTransition?.phase).toBe("committed")
|
|
797
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1"])
|
|
817
798
|
|
|
818
799
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
819
800
|
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
@@ -830,7 +811,7 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
830
811
|
const initialProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
831
812
|
const initialWorker = initialProcesses.find((processConfig) => processConfig.id === "worker")
|
|
832
813
|
|
|
833
|
-
|
|
814
|
+
if (!initialWorker) throw new Error("Missing initial fixture worker")
|
|
834
815
|
initialWorker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
835
816
|
await writeConfig(fixture.configPath, fixture.config)
|
|
836
817
|
const initialConfig = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -847,7 +828,7 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
847
828
|
const processes = Array.isArray(changedConfig.processes) ? changedConfig.processes : []
|
|
848
829
|
const jobsValue = processes.find((processConfig) => processConfig && typeof processConfig === "object" && !Array.isArray(processConfig) && processConfig.id === "jobs")
|
|
849
830
|
|
|
850
|
-
|
|
831
|
+
expect(jobsValue && typeof jobsValue === "object" && !Array.isArray(jobsValue)).toBeTruthy()
|
|
851
832
|
const jobs = /** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobsValue)
|
|
852
833
|
|
|
853
834
|
jobs.env = {.../** @type {Record<string, import("../src/json.js").JsonValue>} */ (jobs.env), RELEASE_CONFIG_AUTHORITY: "v2"}
|
|
@@ -857,7 +838,7 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
857
838
|
// retirement refreshes the previous generation's guardian definition.
|
|
858
839
|
owner.resumeGenerationTransition = async () => ({pausedAt: "candidate_ready"})
|
|
859
840
|
await owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
860
|
-
|
|
841
|
+
expect(owner.status().generationTransition?.phase).toBe("candidate_ready")
|
|
861
842
|
|
|
862
843
|
await owner.retireCommittedOwner(undefined)
|
|
863
844
|
owner.guardian?.disconnect()
|
|
@@ -871,9 +852,9 @@ test("owner recovery preserves complete private transition authority across a ca
|
|
|
871
852
|
recovered = new RollbridgeDaemon({config: normalizeConfig(changedConfig, fixture.configPath), configPath: fixture.configPath, logger: () => {}})
|
|
872
853
|
await recovered.start()
|
|
873
854
|
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
855
|
+
expect(recovered.status().activeReleaseId).toBe("v2")
|
|
856
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
857
|
+
expect(await waitForLifecycleEvents(fixture.lifecycleLogPath, ["activate:v1", "retire:v1", "activate:v2"])).toEqual(["activate:v1", "retire:v1", "activate:v2"])
|
|
877
858
|
} finally {
|
|
878
859
|
if (recovered) await recovered.shutdown().catch(() => {})
|
|
879
860
|
await stopFixtureGuardian(fixture.statePath)
|
|
@@ -909,25 +890,26 @@ test("owner recovery reconstructs a stopped release that still owns a committed-
|
|
|
909
890
|
if (owner.pendingWrite) await owner.pendingWrite
|
|
910
891
|
const pending = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
|
|
911
892
|
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
893
|
+
expect(pending.activeReleaseId).toBe("v2")
|
|
894
|
+
expect(pending.generationTransition?.phase).toBe("committed_pending")
|
|
895
|
+
expect(pending.serviceReleaseIds?.beacon).toBe("v2")
|
|
896
|
+
expect(pending.singletonReleaseIds?.singleton).toBe("v1")
|
|
916
897
|
const stoppedRelease = pending.releases.find((release) => release.releaseId === "v1" && release.state === "stopped")
|
|
917
898
|
|
|
918
|
-
|
|
919
|
-
|
|
899
|
+
if (!stoppedRelease) throw new Error("Missing stopped singleton owner release")
|
|
900
|
+
expect(pending.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2"])
|
|
920
901
|
await owner.retireCommittedOwner(undefined)
|
|
921
902
|
owner.guardian?.disconnect()
|
|
922
903
|
|
|
923
904
|
recovered = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
924
905
|
await recovered.start()
|
|
925
906
|
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
907
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
908
|
+
expect(recovered.singletonReleaseIds.get("singleton")).toBe("v2")
|
|
909
|
+
// the stopped singleton owner may be pruned after replacement commits
|
|
910
|
+
expect(!recovered.releases.has("v1")).toBeTruthy()
|
|
911
|
+
expect(recovered.portReservations.has(stoppedRelease.ports.jobs)).toBe(false)
|
|
912
|
+
expect(recovered.portReservations.has(stoppedRelease.ports.web)).toBe(false)
|
|
931
913
|
} finally {
|
|
932
914
|
replacementPause.continue()
|
|
933
915
|
await deployPromise?.catch(() => {})
|
|
@@ -957,7 +939,7 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
957
939
|
const initialProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
958
940
|
const initialSingleton = initialProcesses.find((processConfig) => processConfig.id === "singleton")
|
|
959
941
|
|
|
960
|
-
|
|
942
|
+
if (!initialSingleton) throw new Error("Missing initial singleton")
|
|
961
943
|
initialSingleton.env = {SINGLETON_CONFIG_AUTHORITY: "v1"}
|
|
962
944
|
await writeConfig(fixture.configPath, fixture.config)
|
|
963
945
|
const initialConfig = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -977,7 +959,7 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
977
959
|
const changedProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (changedConfig.processes)
|
|
978
960
|
const changedSingleton = changedProcesses.find((processConfig) => processConfig.id === "singleton")
|
|
979
961
|
|
|
980
|
-
|
|
962
|
+
if (!changedSingleton) throw new Error("Missing changed singleton")
|
|
981
963
|
changedSingleton.env = {SINGLETON_CONFIG_AUTHORITY: "v2"}
|
|
982
964
|
await writeConfig(fixture.configPath, changedConfig)
|
|
983
965
|
deployPromise = owner.deploy({releaseId: "v2", releasePath: v2Path, revision: "v2"})
|
|
@@ -985,9 +967,9 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
985
967
|
await replacementPause.started
|
|
986
968
|
const pending = /** @type {RecoveryState} */ (JSON.parse(await fs.readFile(fixture.statePath, "utf8")))
|
|
987
969
|
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
970
|
+
expect(pending.activeReleaseId).toBe("v2")
|
|
971
|
+
expect(pending.generationTransition?.phase).toBe("committed_pending")
|
|
972
|
+
expect(pending.singletonReleaseIds?.singleton).toBe("v1")
|
|
991
973
|
await owner.persistState({throwOnError: true})
|
|
992
974
|
await owner.retireCommittedOwner(undefined)
|
|
993
975
|
owner.guardian?.disconnect()
|
|
@@ -997,9 +979,9 @@ test("owner recovery uses the owning release singleton definition during a commi
|
|
|
997
979
|
recovered = new RollbridgeDaemon({config: nextConfig, configPath: fixture.configPath, logger: () => {}})
|
|
998
980
|
await recovered.start()
|
|
999
981
|
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
982
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
983
|
+
expect(recovered.singletonReleaseIds.get("singleton")).toBe("v2")
|
|
984
|
+
expect(recovered.singletons.get("singleton")?.env.SINGLETON_CONFIG_AUTHORITY).toBe("v2")
|
|
1003
985
|
} finally {
|
|
1004
986
|
replacementPause.continue()
|
|
1005
987
|
await deployPromise?.catch(() => {})
|
|
@@ -1026,7 +1008,7 @@ test("owner recovery retains a stopped previous release until committed-pending
|
|
|
1026
1008
|
const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
1027
1009
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
1028
1010
|
|
|
1029
|
-
|
|
1011
|
+
if (!worker) throw new Error("Missing fixture worker")
|
|
1030
1012
|
worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
1031
1013
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1032
1014
|
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -1061,16 +1043,16 @@ test("owner recovery retains a stopped previous release until committed-pending
|
|
|
1061
1043
|
await replacementComplete
|
|
1062
1044
|
const pending = await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed_pending" && state.releases?.some((release) => release.releaseId === "v1" && release.state === "stopped"), AbortSignal.timeout(5000))
|
|
1063
1045
|
|
|
1064
|
-
|
|
1065
|
-
|
|
1046
|
+
expect(pending.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2"])
|
|
1047
|
+
expect(pending.singletonReleaseIds?.singleton).toBe("v2")
|
|
1066
1048
|
await owner.retireCommittedOwner(undefined)
|
|
1067
1049
|
owner.guardian?.disconnect()
|
|
1068
1050
|
|
|
1069
1051
|
recovered = new RollbridgeDaemon({config, configPath: fixture.configPath, logger: () => {}})
|
|
1070
1052
|
await recovered.start()
|
|
1071
1053
|
|
|
1072
|
-
|
|
1073
|
-
|
|
1054
|
+
expect(recovered.status().generationTransition?.phase).toBe("committed")
|
|
1055
|
+
expect(recovered.status().activeReleaseId).toBe("v2")
|
|
1074
1056
|
} finally {
|
|
1075
1057
|
continueReplacement()
|
|
1076
1058
|
await deployPromise?.catch(() => {})
|
|
@@ -1104,7 +1086,7 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
|
|
|
1104
1086
|
const changedProcesses = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (changedConfig.processes)
|
|
1105
1087
|
const changedJobs = changedProcesses.find((processConfig) => processConfig.id === "jobs")
|
|
1106
1088
|
|
|
1107
|
-
|
|
1089
|
+
if (!changedJobs) throw new Error("Missing changed jobs service")
|
|
1108
1090
|
changedJobs.env = {.../** @type {Record<string, import("../src/json.js").JsonValue>} */ (changedJobs.env), RELEASE_CONFIG_AUTHORITY: "v2"}
|
|
1109
1091
|
await writeConfig(fixture.configPath, changedConfig)
|
|
1110
1092
|
const interruptedDeploy = sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})
|
|
@@ -1121,9 +1103,10 @@ test("owner recovery replays ambiguous retirement with the previous release's ex
|
|
|
1121
1103
|
await waitForLog(owner, "control socket listening")
|
|
1122
1104
|
await waitForState(fixture.statePath, (state) => state.generationTransition?.phase === "committed", AbortSignal.timeout(5000))
|
|
1123
1105
|
await recoverySettled
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1106
|
+
// ambiguous retirement must replay exactly once
|
|
1107
|
+
expect((await fs.readFile(retirementWaitingPath, "utf8")).trim().split("\n").length).toBe(2)
|
|
1108
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v1", "activate:v2"])
|
|
1109
|
+
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v2")
|
|
1127
1110
|
|
|
1128
1111
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
1129
1112
|
|
|
@@ -1147,7 +1130,7 @@ test("owner recovery rejects config identity mismatch without changing the valid
|
|
|
1147
1130
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
|
|
1148
1131
|
const status = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1149
1132
|
workerPid = status.releases[0]?.processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
1150
|
-
|
|
1133
|
+
expect(workerPid).toBeTruthy()
|
|
1151
1134
|
const validState = await fs.readFile(fixture.statePath, "utf8")
|
|
1152
1135
|
|
|
1153
1136
|
owner.kill("SIGKILL")
|
|
@@ -1156,9 +1139,9 @@ test("owner recovery rejects config identity mismatch without changing the valid
|
|
|
1156
1139
|
|
|
1157
1140
|
const rejected = await runDaemon(fixture.configPath)
|
|
1158
1141
|
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1142
|
+
expect(rejected.code).not.toBe(0)
|
|
1143
|
+
expect(rejected.output).toMatch(/authority does not match/)
|
|
1144
|
+
expect(await fs.readFile(fixture.statePath, "utf8")).toBe(validState)
|
|
1162
1145
|
|
|
1163
1146
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1164
1147
|
owner = spawnDaemon(fixture.configPath)
|
|
@@ -1188,7 +1171,7 @@ test("failed recovery bootstrap keeps the reconstructed active generation servin
|
|
|
1188
1171
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
|
|
1189
1172
|
const status = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1190
1173
|
workerPid = status.releases[0]?.processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
1191
|
-
|
|
1174
|
+
expect(workerPid).toBeTruthy()
|
|
1192
1175
|
|
|
1193
1176
|
owner.kill("SIGKILL")
|
|
1194
1177
|
await once(owner, "exit")
|
|
@@ -1198,9 +1181,9 @@ test("failed recovery bootstrap keeps the reconstructed active generation servin
|
|
|
1198
1181
|
const preserved = await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})
|
|
1199
1182
|
const events = await sendControlCommand({command: {command: "events"}, path: fixture.socketPath})
|
|
1200
1183
|
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1184
|
+
expect(preserved.activeReleaseId).toBe("v1")
|
|
1185
|
+
expect(preserved.releaseReferences).toEqual([{releaseId: "v1", releasePath}])
|
|
1186
|
+
expect(Array.isArray(events.events) && events.events.some((event) => event && typeof event === "object" && "message" in event && event.message === "bootstrap activation failed")).toBeTruthy()
|
|
1204
1187
|
|
|
1205
1188
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
1206
1189
|
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
@@ -1228,7 +1211,7 @@ test("owner recovery repairs a partial public snapshot from committed guardian s
|
|
|
1228
1211
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
|
|
1229
1212
|
const status = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1230
1213
|
workerPid = status.releases[0]?.processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
1231
|
-
|
|
1214
|
+
expect(workerPid).toBeTruthy()
|
|
1232
1215
|
const validState = JSON.parse(await fs.readFile(fixture.statePath, "utf8"))
|
|
1233
1216
|
|
|
1234
1217
|
owner.kill("SIGKILL")
|
|
@@ -1244,9 +1227,9 @@ test("owner recovery repairs a partial public snapshot from committed guardian s
|
|
|
1244
1227
|
recoveredDaemonPid = repairedState.daemonPid
|
|
1245
1228
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1246
1229
|
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1230
|
+
expect(recovered.activeReleaseId).toBe("v1")
|
|
1231
|
+
expect(releaseProcessPid(recovered, "v1", "worker")).toBe(workerPid)
|
|
1232
|
+
expect(JSON.parse(await fs.readFile(fixture.statePath, "utf8"))).not.toEqual(partialState)
|
|
1250
1233
|
|
|
1251
1234
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
1252
1235
|
await fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")
|
|
@@ -1275,7 +1258,7 @@ test("concurrent same-authority replacements converge on one fenced owner", asyn
|
|
|
1275
1258
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})
|
|
1276
1259
|
const status = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1277
1260
|
workerPid = status.releases[0]?.processes.find((processStatus) => processStatus.id === "worker")?.pid
|
|
1278
|
-
|
|
1261
|
+
expect(workerPid).toBeTruthy()
|
|
1279
1262
|
|
|
1280
1263
|
owner.kill("SIGKILL")
|
|
1281
1264
|
await once(owner, "exit")
|
|
@@ -1290,8 +1273,9 @@ test("concurrent same-authority replacements converge on one fenced owner", asyn
|
|
|
1290
1273
|
const loser = winner === owner ? secondContender : owner
|
|
1291
1274
|
const [loserCode] = loser.exitCode === null && loser.signalCode === null ? await once(loser, "exit") : [loser.exitCode]
|
|
1292
1275
|
|
|
1293
|
-
|
|
1294
|
-
|
|
1276
|
+
// fenced loser must attest the matching winner and exit successfully
|
|
1277
|
+
expect(loserCode).toBe(0)
|
|
1278
|
+
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v1")
|
|
1295
1279
|
|
|
1296
1280
|
owner = winner
|
|
1297
1281
|
contender = undefined
|
|
@@ -1322,20 +1306,20 @@ test("fresh guardian identity remains recoverable when proxy startup fails", asy
|
|
|
1322
1306
|
try {
|
|
1323
1307
|
await new Promise((resolve, reject) => blocker.listen(0, "127.0.0.1", () => resolve(undefined)).once("error", reject))
|
|
1324
1308
|
const address = blocker.address()
|
|
1325
|
-
|
|
1309
|
+
if (!address || typeof address === "string") throw new Error("Expected a TCP listener address")
|
|
1326
1310
|
config.proxy.port = address.port
|
|
1327
1311
|
const startupAttempt = new RollbridgeDaemon({config, logger: () => {}})
|
|
1328
1312
|
failedOwner = startupAttempt
|
|
1329
|
-
await
|
|
1313
|
+
await expect(() => startupAttempt.start()).toThrow(/EADDRINUSE/)
|
|
1330
1314
|
|
|
1331
1315
|
const state = JSON.parse(await fs.readFile(fixture.statePath, "utf8"))
|
|
1332
|
-
|
|
1316
|
+
expect(typeof state.recovery?.guardian?.token).toBe("string")
|
|
1333
1317
|
failedOwner.abandonOwnerRecoveryAttempt()
|
|
1334
1318
|
await new Promise((resolve) => blocker.close(() => resolve(undefined)))
|
|
1335
1319
|
|
|
1336
1320
|
replacement = new RollbridgeDaemon({config, logger: () => {}})
|
|
1337
1321
|
await replacement.start()
|
|
1338
|
-
|
|
1322
|
+
expect(replacement.status().activeReleaseId).toBe(null)
|
|
1339
1323
|
await replacement.shutdown()
|
|
1340
1324
|
} finally {
|
|
1341
1325
|
if (blocker.listening) await new Promise((resolve) => blocker.close(() => resolve(undefined)))
|
|
@@ -1365,8 +1349,8 @@ test("replacement reconstructs retained draining generations without an active r
|
|
|
1365
1349
|
await fs.writeFile(path.join(v2Path, "worker.fifo"), "drained\n")
|
|
1366
1350
|
await stopActive
|
|
1367
1351
|
const drainOnly = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1368
|
-
|
|
1369
|
-
|
|
1352
|
+
expect(drainOnly.activeReleaseId).toBe(null)
|
|
1353
|
+
expect(drainOnly.releaseReferences).toEqual([{releaseId: "v1", releasePath: v1Path}])
|
|
1370
1354
|
await waitForState(fixture.statePath, (state) => state.activeReleaseId === null && state.releaseReferences?.length === 1 && state.releaseReferences[0]?.releaseId === "v1")
|
|
1371
1355
|
|
|
1372
1356
|
owner.kill("SIGKILL")
|
|
@@ -1374,12 +1358,13 @@ test("replacement reconstructs retained draining generations without an active r
|
|
|
1374
1358
|
owner = spawnDaemon(fixture.configPath)
|
|
1375
1359
|
await waitForLog(owner, "control socket listening")
|
|
1376
1360
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1377
|
-
|
|
1378
|
-
|
|
1361
|
+
expect(recovered.activeReleaseId).toBe(null)
|
|
1362
|
+
expect(recovered.releaseReferences).toEqual([{releaseId: "v1", releasePath: v1Path}])
|
|
1379
1363
|
|
|
1380
1364
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
1381
1365
|
await fs.writeFile(path.join(v1Path, "worker.fifo"), "drained\n")
|
|
1382
|
-
|
|
1366
|
+
// replacement shutdown must acknowledge after the recovered drain settles
|
|
1367
|
+
await expect(() => shutdown).not.toThrow()
|
|
1383
1368
|
await once(owner, "exit")
|
|
1384
1369
|
} finally {
|
|
1385
1370
|
await killChild(owner)
|
|
@@ -1400,10 +1385,7 @@ test("deploy rejects a live ownerRecovery mode change", async () => {
|
|
|
1400
1385
|
for (const processConfig of changedConfig.processes) if (processConfig.lifecycle) delete processConfig.lifecycle.activateCommand
|
|
1401
1386
|
await writeConfig(fixture.configPath, changedConfig)
|
|
1402
1387
|
|
|
1403
|
-
await
|
|
1404
|
-
sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath}),
|
|
1405
|
-
/ownerRecovery.*cannot be applied live/
|
|
1406
|
-
)
|
|
1388
|
+
await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath, revision: "v1"}, path: fixture.socketPath})).toThrow(/ownerRecovery.*cannot be applied live/)
|
|
1407
1389
|
|
|
1408
1390
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1409
1391
|
await sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
@@ -1430,10 +1412,7 @@ test("deploy rejects a live activation lifecycle mode change", async () => {
|
|
|
1430
1412
|
|
|
1431
1413
|
delete lifecycle.activateCommand
|
|
1432
1414
|
await writeConfig(fixture.configPath, changedConfig)
|
|
1433
|
-
await
|
|
1434
|
-
sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath}),
|
|
1435
|
-
/lifecycle\.activateCommand.*cannot be applied live/
|
|
1436
|
-
)
|
|
1415
|
+
await expect(() => sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})).toThrow(/lifecycle\.activateCommand.*cannot be applied live/)
|
|
1437
1416
|
|
|
1438
1417
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1439
1418
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
@@ -1452,7 +1431,7 @@ test("public state does not advance when private guardian publication fails", as
|
|
|
1452
1431
|
const processes = /** @type {Record<string, import("../src/json.js").JsonValue>[]} */ (fixture.config.processes)
|
|
1453
1432
|
const worker = processes.find((processConfig) => processConfig.id === "worker")
|
|
1454
1433
|
|
|
1455
|
-
|
|
1434
|
+
if (!worker) throw new Error("Missing fixture worker")
|
|
1456
1435
|
worker.lifecycle = {drainCommand: "true", drainTimeoutMs: 1000}
|
|
1457
1436
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1458
1437
|
const config = normalizeConfig(fixture.config, fixture.configPath)
|
|
@@ -1470,9 +1449,9 @@ test("public state does not advance when private guardian publication fails", as
|
|
|
1470
1449
|
owner.publishOwnerState = async () => { throw new Error("injected guardian publication failure") }
|
|
1471
1450
|
const write = owner.persistState({throwOnError: true})
|
|
1472
1451
|
|
|
1473
|
-
|
|
1474
|
-
await
|
|
1475
|
-
|
|
1452
|
+
expect(write).toBeTruthy()
|
|
1453
|
+
await expect(() => write).toThrow(/injected guardian publication failure/)
|
|
1454
|
+
expect(await fs.readFile(fixture.statePath, "utf8")).toBe(before)
|
|
1476
1455
|
owner.publishOwnerState = publishOwnerState
|
|
1477
1456
|
await owner.persistState({throwOnError: true})
|
|
1478
1457
|
await owner.shutdown()
|
|
@@ -1499,7 +1478,7 @@ test("replacement removes only guardian-owned candidate inventory left before de
|
|
|
1499
1478
|
const worker = candidateConfig.processes.find((processConfig) => processConfig.id === "worker")
|
|
1500
1479
|
const web = candidateConfig.processes.find((processConfig) => processConfig.id === "web")
|
|
1501
1480
|
|
|
1502
|
-
|
|
1481
|
+
if (!worker || !web) throw new Error("Missing fixture worker or web process")
|
|
1503
1482
|
worker.command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("const fs = require('node:fs'); const target = process.env.ROLLBRIDGE_RELEASE_PATH + '/candidate.pid'; fs.writeFileSync(target + '.tmp', String(process.pid)); fs.renameSync(target + '.tmp', target); setInterval(() => {}, 1000)")}`
|
|
1504
1483
|
worker.lifecycle = {drainTimeoutMs: 0}
|
|
1505
1484
|
web.command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`
|
|
@@ -1510,26 +1489,31 @@ test("replacement removes only guardian-owned candidate inventory left before de
|
|
|
1510
1489
|
|
|
1511
1490
|
await waitForFile(candidatePidPath)
|
|
1512
1491
|
candidatePid = Number(await fs.readFile(candidatePidPath, "utf8"))
|
|
1513
|
-
|
|
1492
|
+
expect(Number.isInteger(candidatePid) && candidatePid > 0).toBeTruthy()
|
|
1514
1493
|
owner.kill("SIGKILL")
|
|
1515
1494
|
await once(owner, "exit")
|
|
1516
1495
|
const stateAfterDeath = JSON.parse(await fs.readFile(fixture.statePath, "utf8"))
|
|
1517
1496
|
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1497
|
+
// owner death must preserve the last committed active release
|
|
1498
|
+
expect(stateAfterDeath.activeReleaseId).toBe(committedState.activeReleaseId)
|
|
1499
|
+
// owner death must preserve committed release references
|
|
1500
|
+
expect(stateAfterDeath.releaseReferences).toEqual(committedState.releaseReferences)
|
|
1501
|
+
// owner death must not commit candidate release metadata
|
|
1502
|
+
expect(stateAfterDeath.releases).toEqual(committedState.releases)
|
|
1521
1503
|
|
|
1522
1504
|
await writeConfig(fixture.configPath, fixture.config)
|
|
1523
1505
|
owner = spawnDaemon(fixture.configPath)
|
|
1524
1506
|
await waitForLog(owner, "control socket listening")
|
|
1525
1507
|
const recovered = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1526
1508
|
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1509
|
+
expect(recovered.activeReleaseId).toBe("v1")
|
|
1510
|
+
expect(recovered.releaseReferences).toEqual([{releaseId: "v1", releasePath: v1Path}])
|
|
1511
|
+
// uncommitted candidate must be stopped before replacement becomes healthy
|
|
1512
|
+
expect(isProcessRunning(candidatePid)).toBe(false)
|
|
1530
1513
|
|
|
1531
1514
|
await sendControlCommand({command: {command: "deploy", releaseId: "v2", releasePath: v2Path, revision: "v2"}, path: fixture.socketPath})
|
|
1532
|
-
|
|
1515
|
+
// removed candidate keys must be reusable by a later valid deploy
|
|
1516
|
+
expect((await sendControlCommand({command: {command: "status"}, path: fixture.socketPath})).activeReleaseId).toBe("v2")
|
|
1533
1517
|
|
|
1534
1518
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
1535
1519
|
await Promise.all([v1Path, v2Path].map((releasePath) => fs.writeFile(path.join(releasePath, "worker.fifo"), "drained\n")))
|
|
@@ -1562,18 +1546,18 @@ test("ensure-daemon replaces a same-authority owner whose control socket disappe
|
|
|
1562
1546
|
try {
|
|
1563
1547
|
const first = await runCli(ensureArgs)
|
|
1564
1548
|
|
|
1565
|
-
|
|
1549
|
+
expect({value: first.code, context: first.output}).toMatchObject({value: 0})
|
|
1566
1550
|
await sendControlCommand({command: {command: "deploy", releaseId: "v1", releasePath: v1Path, revision: "v1"}, path: fixture.socketPath})
|
|
1567
1551
|
const before = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1568
1552
|
|
|
1569
1553
|
await fs.rm(fixture.socketPath, {force: true})
|
|
1570
1554
|
const replacement = await runCli(ensureArgs)
|
|
1571
1555
|
|
|
1572
|
-
|
|
1556
|
+
expect({value: replacement.code, context: `${replacement.output}\n${await fs.readFile(daemonLogPath, "utf8")}`}).toMatchObject({value: 0})
|
|
1573
1557
|
const after = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: fixture.socketPath}))
|
|
1574
1558
|
|
|
1575
|
-
|
|
1576
|
-
|
|
1559
|
+
expect(after.activeReleaseId).toBe("v1")
|
|
1560
|
+
expect(after.daemonPid).not.toBe(before.daemonPid)
|
|
1577
1561
|
await waitForProcessExit(before.daemonPid)
|
|
1578
1562
|
|
|
1579
1563
|
const shutdown = sendControlCommand({command: {command: "shutdown"}, path: fixture.socketPath})
|
|
@@ -1621,7 +1605,7 @@ test("ensure-daemon atomically replaces an incompatible owner without losing ret
|
|
|
1621
1605
|
nextConfig.control = {path: newControlPath}
|
|
1622
1606
|
const companionTemplate = nextConfig.processes.find((processConfig) => processConfig.policy === "companion")
|
|
1623
1607
|
|
|
1624
|
-
|
|
1608
|
+
if (!companionTemplate) throw new Error("Missing fixture companion template")
|
|
1625
1609
|
nextConfig.processes.splice(2, 0, {
|
|
1626
1610
|
...structuredClone(companionTemplate),
|
|
1627
1611
|
id: "new-topology-process",
|
|
@@ -1637,36 +1621,38 @@ test("ensure-daemon atomically replaces an incompatible owner without losing ret
|
|
|
1637
1621
|
"--daemon-runtime-path", runtimePath,
|
|
1638
1622
|
"--daemon-start-timeout-ms", "5000"
|
|
1639
1623
|
])
|
|
1640
|
-
|
|
1624
|
+
expect({value: replacement.code, context: `${replacement.output}\n${await fs.readFile(daemonLogPath, "utf8")}`}).toMatchObject({value: 0})
|
|
1641
1625
|
cleanupControlPath = newControlPath
|
|
1642
1626
|
|
|
1643
1627
|
const after = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: newControlPath}))
|
|
1644
1628
|
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1629
|
+
expect(after.activeReleaseId).toBe("v2")
|
|
1630
|
+
expect(after.releaseReferences).toEqual(before.releaseReferences)
|
|
1631
|
+
expect(after.releases.map((release) => release.processes.map((processStatus) => processStatus.pid))).toEqual(before.releases.map((release) => release.processes.map((processStatus) => processStatus.pid)))
|
|
1632
|
+
expect(after.services[0]?.process.pid).toBe(before.services[0]?.process.pid)
|
|
1633
|
+
expect(after.singletons[0]?.process.pid).toBe(before.singletons[0]?.process.pid)
|
|
1634
|
+
// listener-owned WebSocket must remain supervised across replacement
|
|
1635
|
+
expect(retainedConnectionClosed).toBe(false)
|
|
1636
|
+
expect(after.daemonPid).toBeTruthy()
|
|
1652
1637
|
await fs.writeFile(daemonLogPath, "")
|
|
1653
1638
|
process.kill(after.daemonPid, "SIGKILL")
|
|
1654
1639
|
const restartedState = await waitForState(fixture.statePath, (state) => state.daemonPid !== after.daemonPid, AbortSignal.timeout(5000))
|
|
1655
1640
|
const restarted = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: newControlPath}))
|
|
1656
1641
|
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1642
|
+
expect(restarted.daemonPid).toBe(restartedState.daemonPid)
|
|
1643
|
+
expect(Number((await fs.readFile(daemonPidPath, "utf8")).trim())).toBe(restarted.daemonPid)
|
|
1644
|
+
expect(restarted.releases.map((release) => release.processes.map((processStatus) => processStatus.pid))).toEqual(before.releases.map((release) => release.processes.map((processStatus) => processStatus.pid)))
|
|
1645
|
+
expect(await fs.readFile(daemonLogPath, "utf8")).toMatch(/owner state recovered/)
|
|
1661
1646
|
|
|
1662
1647
|
const v3Path = await prepareRelease(fixture.root, "v3")
|
|
1663
1648
|
await sendControlCommand({command: {command: "deploy", releaseId: "v3", releasePath: v3Path, revision: "v3"}, path: newControlPath})
|
|
1664
1649
|
const deployed = /** @type {DaemonStatus} */ (await sendControlCommand({command: {command: "status"}, path: newControlPath}))
|
|
1665
1650
|
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1651
|
+
expect(deployed.activeReleaseId).toBe("v3")
|
|
1652
|
+
expect(deployed.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2", "v3"])
|
|
1653
|
+
expect(releaseProcessPid(deployed, "v1", "web")).toBe(releaseProcessPid(before, "v1", "web"))
|
|
1654
|
+
// a later deploy must not stop a process with a transferred live connection
|
|
1655
|
+
expect(retainedConnectionClosed).toBe(false)
|
|
1670
1656
|
|
|
1671
1657
|
retainedConnection.destroy()
|
|
1672
1658
|
await retainedConnectionClose
|
|
@@ -1813,7 +1799,7 @@ async function prepareRelease(root, releaseId, {holdJobsBind = false} = {}) {
|
|
|
1813
1799
|
await fs.mkdir(releasePath)
|
|
1814
1800
|
if (!holdJobsBind) await fs.writeFile(path.join(releasePath, "jobs.bind"), "ready\n")
|
|
1815
1801
|
const gate = spawn("mkfifo", [path.join(releasePath, "worker.fifo")])
|
|
1816
|
-
|
|
1802
|
+
expect((await once(gate, "exit"))[0]).toBe(0)
|
|
1817
1803
|
return releasePath
|
|
1818
1804
|
}
|
|
1819
1805
|
|
|
@@ -2006,7 +1992,7 @@ async function openWebSocket(port) {
|
|
|
2006
1992
|
].join("\r\n"))
|
|
2007
1993
|
const [response] = await once(socket, "data")
|
|
2008
1994
|
|
|
2009
|
-
|
|
1995
|
+
expect(String(response)).toMatch(/^HTTP\/1\.1 101 /)
|
|
2010
1996
|
return socket
|
|
2011
1997
|
}
|
|
2012
1998
|
|
|
@@ -2084,7 +2070,7 @@ async function runCli(args) {
|
|
|
2084
2070
|
* @param {{allowChildExit?: boolean}} [options] - Whether inherited descriptors may outlive the original child.
|
|
2085
2071
|
*/
|
|
2086
2072
|
async function waitForLog(child, message, {allowChildExit = false} = {}) {
|
|
2087
|
-
|
|
2073
|
+
if (!child.stdout) throw new Error("Missing CLI stdout stream")
|
|
2088
2074
|
child.stdout.setEncoding("utf8")
|
|
2089
2075
|
|
|
2090
2076
|
await new Promise((resolve, reject) => {
|
|
@@ -2121,3 +2107,4 @@ async function waitForLog(child, message, {allowChildExit = false} = {}) {
|
|
|
2121
2107
|
child.stderr?.setEncoding("utf8").on("data", onErrorData)
|
|
2122
2108
|
})
|
|
2123
2109
|
}
|
|
2110
|
+
})
|